【源码阅读】VictoriaLogs v1.22.2: vlinsert的转发功能还非常原始

作者:张富春(ahfuzhang)，转载时请注明作者和引用链接，谢谢！

• cnblogs博客
• zhihu
• Github
• 公众号:一本正经的瞎扯
  

---

群集版目前看起来还比较早期，值得再等等。

1. 群集版只有单一的一个二进制文件。
   不像 VictoriaMetrics，区分了vmselect/vminsert/vmstorage；vmlog 使用参数来区分，这个节点是 vlstorage 或是 vlinsert+vlselect.
   是的，vlinsert 和 vlselect 会同时开启：

//  app/vlstorage/main.go
func initNetworkStorage() {
	if netstorageInsert != nil || netstorageSelect != nil {
		logger.Panicf("BUG: initNetworkStorage() has been already called")
	}

	authCfgs := make([]*promauth.Config, len(*storageNodeAddrs))
	isTLSs := make([]bool, len(*storageNodeAddrs))
	for i := range authCfgs {
		authCfgs[i] = newAuthConfigForStorageNode(i)
		isTLSs[i] = storageNodeTLS.GetOptionalArg(i)
	}
    // todo: 明显有问题，没理由同时初始化 insert 和 select
	logger.Infof("starting insert service for nodes %s", *storageNodeAddrs)
	netstorageInsert = netinsert.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *insertConcurrency, *insertDisableCompression)

	logger.Infof("initializing select service for nodes %s", *storageNodeAddrs)
	netstorageSelect = netselect.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *selectDisableCompression)

	logger.Infof("initialized all the network services")
}

2. vlinsert 选取 vlstorage 后端，使用了简单粗暴的取模算法，而不是一致性 hash

// app/vlstorage/netinsert/netinsert.go
func (srt *streamRowsTracker) getNodeIdx(streamHash uint64) uint64 {
	if srt.nodesCount == 1 {
		// Fast path for a single node.
		return 0
	}

	srt.mu.Lock()
	defer srt.mu.Unlock()

	streamRows := srt.rowsPerStream[streamHash] + 1
	srt.rowsPerStream[streamHash] = streamRows

	if streamRows <= 1000 {
		// Write the initial rows for the stream to a single storage node for better locality.
		// This should work great for log streams containing small number of logs, since will be distributed
		// evenly among available storage nodes because they have different streamHash.
		return streamHash % uint64(srt.nodesCount)
	}

	// The log stream contains more than 1000 rows. Distribute them among storage nodes at random
	// in order to improve query performance over this stream (the data for the log stream
	// can be processed in parallel on all the storage nodes).
	//
	// The random distribution is preferred over round-robin distribution in order to avoid possible
	// dependency between the order of the ingested logs and the number of storage nodes,
	// which may lead to non-uniform distribution of logs among storage nodes.
	return uint64(fastrand.Uint32n(uint32(srt.nodesCount)))
}

3. vlinsert 还不支持 duplicate factor 的选项，无法做到 vlstorage 层的冗余存储，影响了高可用。

// AddRow adds the given log row into s.
func (s *Storage) AddRow(streamHash uint64, r *logstorage.InsertRow) {
	idx := s.srt.getNodeIdx(streamHash)
	sn := s.sns[idx]
	sn.addRow(r)  // 仅在 n 个存储节点的其中一个节点上存储
}

4. 转发层的监控不全，上线有风险。
   数据丢了都不知道为什么丢的。

func (sn *storageNode) mustSendInsertRequest(pendingData *bytesutil.ByteBuffer) {
	defer func() {
		pendingData.Reset()
		sn.s.pendingDataBuffers <- pendingData
	}()

	err := sn.sendInsertRequest(pendingData)
	if err == nil {
		return
	}
    // todo: 这里应该增加 metrics 上报
	if !errors.Is(err, errTemporarilyDisabled) {
		logger.Warnf("%s; re-routing the data block to the remaining nodes", err)
	}
	for !sn.s.sendInsertRequestToAnyNode(pendingData) {
		logger.Errorf("cannot send pending data to all storage nodes, since all of them are unavailable; re-trying to send the data in a second")

		t := timerpool.Get(time.Second)
		select {
		case <-sn.s.stopCh:
			timerpool.Put(t)
			logger.Errorf("dropping %d bytes of data, since there are no available storage nodes", pendingData.Len())
			return
		case <-t.C:
			timerpool.Put(t)
		}
	}
}