nacos服务注册之服务器端Distro
一致性协议算法Distro阿里自己的创的算法吧，网上能找到的资料很少。Distro用于处理ephemeral类型数据
- Distro协议算法看代码大体流程是：
- nacos启动首先从其他远程节点同步全部数据
- nacos每个节点是平等的都可以处理写入请求，同时把新数据同步到其他节点
- 每个节点只负责部分数据，定时发送自己负责数据校验值到其他节点来保持数据一致性
- Distro代码分析， nacos负责数据一致性的服务是ConsistencyService接口，DistroConsistencyServiceImpl是EphemeralConsistencyService和ConsistencyService实现类。
Distro协议算法数据存储实现笔记简单，DataStore是存储实现类临时节点的数据都存储在ConcurrentHashMap里面
/**
* Store of data
*
* @author nkorange
* @since 1.0.0
*/
@Component
public class DataStore {
private Map<String, Datum> dataMap = new ConcurrentHashMap<>(1024);
public void put(String key, Datum value) {
dataMap.put(key, value);
}
public Datum remove(String key) {
return dataMap.remove(key);
}
public Set<String> keys() {
return dataMap.keySet();
}
public Datum get(String key) {
return dataMap.get(key);
}
public boolean contains(String key) {
return dataMap.containsKey(key);
}
public Map<String, Datum> batchGet(List<String> keys) {
Map<String, Datum> map = new HashMap<>(128);
for (String key : keys) {
Datum datum = dataMap.get(key);
if (datum == null) {
continue;
}
map.put(key, datum);
}
return map;
}
public int getInstanceCount() {
int count = 0;
for (Map.Entry<String, Datum> entry : dataMap.entrySet()) {
try {
Datum instancesDatum = entry.getValue();
if (instancesDatum.value instanceof Instances) {
count += ((Instances) instancesDatum.value).getInstanceList().size();
}
} catch (Exception ignore) {
}
}
return count;
}
public Map<String, Datum> getDataMap() {
return dataMap;
}
}
nacos启动首先从其他节点同步全部数据
DistroConsistencyServiceImpl类构造完成后启动同步线程直到首次同步成功
@org.springframework.stereotype.Service("distroConsistencyService")
public class DistroConsistencyServiceImpl implements EphemeralConsistencyService {
private volatile Notifier notifier = new Notifier();
private LoadDataTask loadDataTask = new LoadDataTask();
@PostConstruct
public void init() {
GlobalExecutor.submit(loadDataTask);
GlobalExecutor.submitDistroNotifyTask(notifier);
}
/**
* 从其它节点同步数据任务线程
*/
private class LoadDataTask implements Runnable {
@Override
public void run() {
try {
load();
if (!initialized) {
GlobalExecutor.submit(this, globalConfig.getLoadDataRetryDelayMillis());
}
} catch (Exception e) {
Loggers.DISTRO.error("load data failed.", e);
}
}
}
/**
* 初始化时从其它节点同步数据
* @throws Exception
*/
public void load() throws Exception {
//如果单列模式则返回
if (SystemUtils.STANDALONE_MODE) {
initialized = true;
return;
}
// size = 1 means only myself in the list, we need at least one another server alive:
while (serverListManager.getHealthyServers().size() <= 1) {
Thread.sleep(1000L);
Loggers.DISTRO.info("waiting server list init...");
}
//只要有一个节点同步成功则返回
for (Server server : serverListManager.getHealthyServers()) {
if (NetUtils.localServer().equals(server.getKey())) {
continue;
}
// try sync data from remote server:
if (syncAllDataFromRemote(server)) {
initialized = true;
return;
}
}
}
/**
* 从指定nacos节点同步数据
* @param server 目标nacos节点
* @return 是否同步成功
*/
public boolean syncAllDataFromRemote(Server server) {
try {
byte[] data = NamingProxy.getAllData(server.getKey());
processData(data);
return true;
} catch (Exception e) {
return false;
}
}
/**
* 处理同步报文信息
* 更新数据存储和通知监听器
* @param data
* @param data
* @throws Exception
*/
public void processData(byte[] data) throws Exception {
if (data.length > 0) {
Map<String, Datum<Instances>> datumMap = serializer.deserializeMap(data, Instances.class);
for (Map.Entry<String, Datum<Instances>> entry : datumMap.entrySet()) {
dataStore.put(entry.getKey(), entry.getValue());
if (!listeners.containsKey(entry.getKey())) {
// pretty sure the service not exist:
if (switchDomain.isDefaultInstanceEphemeral()) {
// create empty service
Loggers.DISTRO.info("creating service {}", entry.getKey());
Service service = new Service();
String serviceName = KeyBuilder.getServiceName(entry.getKey());
String namespaceId = KeyBuilder.getNamespace(entry.getKey());
service.setName(serviceName);
service.setNamespaceId(namespaceId);
service.setGroupName(Constants.DEFAULT_GROUP);
// now validate the service. if failed, exception will be thrown
service.setLastModifiedMillis(System.currentTimeMillis());
service.recalculateChecksum();
listeners.get(KeyBuilder.SERVICE_META_KEY_PREFIX).get(0).onChange(KeyBuilder.buildServiceMetaKey(namespaceId, serviceName), service);
}
}
}
for (Map.Entry<String, Datum<Instances>> entry : datumMap.entrySet()) {
if (!listeners.containsKey(entry.getKey())) {
// Should not happen:
continue;
}
try {
for (RecordListener listener : listeners.get(entry.getKey())) {
listener.onChange(entry.getKey(), entry.getValue().value);
}
} catch (Exception e) {
continue;
}
// Update data store if listener executed successfully:
dataStore.put(entry.getKey(), entry.getValue());
}
}
}
}
public class NamingProxy {
/**
* 获取指定nacos节点所有数据
* @param server 目标nacos节点
* @return 返回到数据
* @throws Exception
*/
public static byte[] getAllData(String server) throws Exception {
Map<String, String> params = new HashMap<>(8);
//http://120.0.0.1:8848/nacos/v1/ns/distro/datums
HttpClient.HttpResult result = HttpClient.httpGet("http://" + server + RunningConfig.getContextPath() + UtilsAndCommons.NACOS_NAMING_CONTEXT + ALL_DATA_GET_URL, new ArrayList<>(), params);
if (HttpURLConnection.HTTP_OK == result.code) {
return result.content.getBytes();
}
throw new IOException("failed to req API: " + "http://" + server + RunningConfig.getContextPath() + UtilsAndCommons.NACOS_NAMING_CONTEXT + DATA_GET_URL + ". code: " + result.code + " msg: " + result.content);
}
}
Distro两种数据的同步方式：1.定时同步 2.本节点接收到服务变更同步
DataSyncer启动定时同步线程和负责数据同步
TaskDispatcher负责接受服务变更后分发到DataSyncer即时同步
@Component
@DependsOn("serverListManager")
public class DataSyncer {
@PostConstruct
//启动定时同步的定时任务
public void init() {
startTimedSync();
}
public void startTimedSync() {
//每5秒一次
//dataSyncExecutor.scheduleWithFixedDelay(runnable, PARTITION_DATA_TIMED_SYNC_INTERVAL=5000, PARTITION_DATA_TIMED_SYNC_INTERVAL=5000, TimeUnit.MILLISECONDS);
GlobalExecutor.schedulePartitionDataTimedSync(new TimedSync());
}
public class TimedSync implements Runnable {
@Override
public void run() {
try {
// send local timestamps to other servers:
Map<String, String> keyChecksums = new HashMap<>(64);
for (String key : dataStore.keys()) {
//只同步自己负责的那部分数据
// int index = healthyList.indexOf(NetUtils.localServer());
// int target = distroHash(serviceName) % healthyList.size();
if (!distroMapper.responsible(KeyBuilder.getServiceName(key))) {
continue;
}
Datum datum = dataStore.get(key);
if (datum == null) {
continue;
}
keyChecksums.put(key, datum.value.getChecksum());
}
if (keyChecksums.isEmpty()) {
return;
}
for (Server member : getServers()) {
if (NetUtils.localServer().equals(member.getKey())) {
continue;
}
//请求路径/nacos/v1/ns/distro/checksum
NamingProxy.syncCheckSums(keyChecksums, member.getKey());
}
} catch (Exception e) {
Loggers.DISTRO.error("timed sync task failed.", e);
}
}
}
}
TaskDispatcher负责接受服务变更后分发到DataSyncer即时同步
/**
* Data sync task dispatcher
*
* @author nkorange
* @since 1.0.0
*/
@Component
public class TaskDispatcher {
@Autowired
private GlobalConfig partitionConfig;
@Autowired
private DataSyncer dataSyncer;
private List<TaskScheduler> taskSchedulerList = new ArrayList<>();
private final int cpuCoreCount = Runtime.getRuntime().availableProcessors();
@PostConstruct
public void init() {
for (int i = 0; i < cpuCoreCount; i++) {
TaskScheduler taskScheduler = new TaskScheduler(i);
taskSchedulerList.add(taskScheduler);
GlobalExecutor.submitTaskDispatch(taskScheduler);
}
}
/**
* 接收服务变更后同步任务
* @param key 服务唯一标识
*/
public void addTask(String key) {
taskSchedulerList.get(UtilsAndCommons.shakeUp(key, cpuCoreCount)).addTask(key);
}
public class TaskScheduler implements Runnable {
private int index;
private int dataSize = 0;
private long lastDispatchTime = 0L;
private BlockingQueue<String> queue = new LinkedBlockingQueue<>(128 * 1024);
public TaskScheduler(int index) {
this.index = index;
}
public void addTask(String key) {
queue.offer(key);
}
public int getIndex() {
return index;
}
@Override
public void run() {
List<String> keys = new ArrayList<>();
while (true) {
try {
String key = queue.poll(partitionConfig.getTaskDispatchPeriod(), TimeUnit.MILLISECONDS);
if (dataSyncer.getServers() == null || dataSyncer.getServers().isEmpty()) {
continue;
}
if (StringUtils.isBlank(key)) {
continue;
}
if (dataSize == 0) {
keys = new ArrayList<>();
}
keys.add(key);
dataSize++;
//批量同步； 1 数量达到batchSyncKeyCount=1000，2：距上次同步事件大于taskDispatchPeriod=2000
if (dataSize == partitionConfig.getBatchSyncKeyCount() ||
(System.currentTimeMillis() - lastDispatchTime) > partitionConfig.getTaskDispatchPeriod()) {
for (Server member : dataSyncer.getServers()) {
if (NetUtils.localServer().equals(member.getKey())) {
continue;
}
SyncTask syncTask = new SyncTask();
syncTask.setKeys(keys);
syncTask.setTargetServer(member.getKey());
//分发任务到dataSyncer同步数据
dataSyncer.submit(syncTask, 0);
}
lastDispatchTime = System.currentTimeMillis();
dataSize = 0;
}
} catch (Exception e) {
Loggers.DISTRO.error("dispatch sync task failed.", e);
}
}
}
}
}
@Component
@DependsOn("serverListManager")
public class DataSyncer {
/**
*
* @param task 同步数据
* @param delay 延迟时间
*/
public void submit(SyncTask task, long delay) {
//各种监测
..................
GlobalExecutor.submitDataSync(() -> {
// 1. check the server
..................
// 2. get the datums by keys and check the datum is empty or not
..................
byte[] data = serializer.serialize(datumMap);
long timestamp = System.currentTimeMillis();
// DATA_ON_SYNC_URL = "/nacos/v1/ns/distro/datum"
boolean success = NamingProxy.syncData(data, task.getTargetServer());
..............
//不成功重试
}, delay);
}
}
