08.CAT-埋点数据解析

8.1 服务数据接收

消息数据从客户度`被(生产)发送`到服务端，服务端进行数据`消费`。

8.1.1 数据接收器

MessageConsumer

public interface MessageConsumer {
    //消费数据
    public void consume(MessageTree tree);
    // 检查检查点
    public void doCheckpoint();
}

RealtimeConsumer

public class RealtimeConsumer extends ContainerHolder implements MessageConsumer, Initializable, LogEnabled {

  @Inject
    private MessageAnalyzerManager m_analyzerManager;

    @Inject
    private ServerStatisticManager m_serverStateManager;
    
    private PeriodManager m_periodManager;

    public static final long MINUTE = 60 * 1000L;

    public static final long HOUR = 60 * MINUTE;

    ......
    
    @Override
    public void initialize() throws InitializationException {
        m_periodManager = new PeriodManager(HOUR, m_analyzerManager, m_serverStateManager, m_logger);
        m_periodManager.init();

        Threads.forGroup("cat").start(m_periodManager);
    }

}

• 消费实例的初始化
• 通过 lookup 实例化，在实例化时，调用 Initializable 接口的 initialize 资源，在 initialize 进行 PeriodManager 实例

8.2 数据分发处理

public class RealtimeConsumer extends ContainerHolder implements MessageConsumer, Initializable, LogEnabled {

    @Inject
    private MessageAnalyzerManager m_analyzerManager;

    @Inject
    private ServerStatisticManager m_serverStateManager;

    @Inject
    private BlackListManager m_blackListManager;

    private PeriodManager m_periodManager;
    @Override
    public void consume(MessageTree tree) {
        String domain = tree.getDomain();
        String ip = tree.getIpAddress();
      // 黑名单 过滤
        if (!m_blackListManager.isBlack(domain, ip)) {
            long timestamp = tree.getMessage().getTimestamp();
            // 周期 获取当前小时的周期
            Period period = m_periodManager.findPeriod(timestamp);

            if (period != null) {
                period.distribute(tree);
            } else {
                m_serverStateManager.addNetworkTimeError(1);
            }
        } else {
            m_black++;

            if (m_black % CatConstants.SUCCESS_COUNT == 0) {
                Cat.logEvent("Discard", domain);
            }
        }
    }

• 通过上报的数据，拿到消息的 Timestamp 的，通过 m_periodManager 缓存中获取 Period
• Period 的周期时长是 1 小时

8.3 数据分析检查

doCheckpoint

public class RealtimeConsumer extends ContainerHolder implements MessageConsumer, Initializable, LogEnabled {

    public void doCheckpoint() {
        m_logger.info("starting do checkpoint.");
        MessageProducer cat = Cat.getProducer();
        Transaction t = cat.newTransaction("Checkpoint", getClass().getSimpleName());

        try {
            long currentStartTime = getCurrentStartTime();
            Period period = m_periodManager.findPeriod(currentStartTime);

            for (MessageAnalyzer analyzer : period.getAnalzyers()) {
                try {
                    analyzer.doCheckpoint(false);
                } catch (Exception e) {
                    Cat.logError(e);
                }
            }

            try {
                // wait dump analyzer store completed
                Thread.sleep(10 * 1000);
            } catch (InterruptedException e) {
                // ignore
            }
            t.setStatus(Message.SUCCESS);
        } catch (RuntimeException e) {
            cat.logError(e);
            t.setStatus(e);
        } finally {
            t.complete();
        }
        m_logger.info("end do checkpoint.");
    }

• MessageAnalyzer 指控消息 检查状态

8.4 DefaultMessageAnalyzerManager

public class DefaultMessageAnalyzerManager extends ContainerHolder implements MessageAnalyzerManager, Initializable,
      LogEnabled {
   // 1 分钟   
   private static final long MINUTE = 60 * 1000L;

    private long m_duration = 60 * MINUTE;
  // 上一期和本期的交叉的时间3分
    private long m_extraTime = 3 * MINUTE;

    private List<String> m_analyzerNames;

    private Map<Long, Map<String, List<MessageAnalyzer>>> m_analyzers = new HashMap<Long, Map<String, List<MessageAnalyzer>>>();
    
@Override
    public List<MessageAnalyzer> getAnalyzer(String name, long startTime) {
        // remove last two hour analyzer
        try {
          //移除上2期的 MessageAnalyzer
            Map<String, List<MessageAnalyzer>> temp = m_analyzers.remove(startTime - m_duration * 2);

            if (temp != null) {
                for (List<MessageAnalyzer> anlyzers : temp.values()) {
                    for (MessageAnalyzer analyzer : anlyzers) {
                        analyzer.destroy();
                    }
                }
            }
        } catch (Exception e) {
            Cat.logError(e);
        }
        // 获取本期，需要提前初始化
        Map<String, List<MessageAnalyzer>> map = m_analyzers.get(startTime);

        if (map == null) {
            synchronized (m_analyzers) {
                map = m_analyzers.get(startTime);

                if (map == null) {
                    map = new HashMap<String, List<MessageAnalyzer>>();
                    m_analyzers.put(startTime, map);
                }
            }
        }

        List<MessageAnalyzer> analyzers = map.get(name);

        if (analyzers == null) {
            synchronized (map) {
                analyzers = map.get(name);

                if (analyzers == null) {
                    analyzers = new ArrayList<MessageAnalyzer>();

                    MessageAnalyzer analyzer = lookup(MessageAnalyzer.class, name);

                    analyzer.setIndex(0);
                    analyzer.initialize(startTime, m_duration, m_extraTime);
                    analyzers.add(analyzer);

                    int count = analyzer.getAnanlyzerCount();

                    for (int i = 1; i < count; i++) {
                        MessageAnalyzer tempAnalyzer = lookup(MessageAnalyzer.class, name);

                        tempAnalyzer.setIndex(i);
                        tempAnalyzer.initialize(startTime, m_duration, m_extraTime);
                        analyzers.add(tempAnalyzer);
                    }
                    map.put(name, analyzers);
                }
            }
        }

        return analyzers;
    }

8.5 小结

• RealtimeConsumer 的消费管理
• PeriodManager 在 RealtimeConsumer 中被实例，进入后台资管理状态