AWS CloudWatch Logs 到 OpenSearch 数据传输

概述

本文档旨在指导您配置 AWS Lambda 函数，用于从 AWS CloudWatch Logs 获取日志数据，并将其转发至 AWS OpenSearch 服务进行存储和分析。

架构说明

该解决方案使用以下 AWS 组件：

• CloudWatch Logs：源日志数据
• Lambda 函数：处理和转发日志数据
• OpenSearch Service：日志的存储和搜索

架构图

graph TD A[应用/服务] -->|生成日志| B[CloudWatch Log Group] B -->|Subscription Filter| C[Lambda Function] C -->|推送数据| D[OpenSearch Domain] D -->|索引数据| E[OpenSearch Index] F[Grafana/Kibana] -->|查询/可视化| E

Lambda 函数配置

基础配置

• 运行时(Runtime): Node.js
• 处理器(Handler): index.handler
• 架构(Architecture): ARM64
• 超时时间(Timeout): 1 分钟
• 内存(Memory): 128 MB

网络配置

Lambda 函数需要部署在 VPC 内以便访问 OpenSearch 服务：

1. VPC: 选择包含 OpenSearch 域的 VPC
2. 子网(Subnets): 选择至少两个私有子网（推荐可用区分布）
3. 安全组(Security Group): 配置允许出站访问 OpenSearch 端点的安全组

IAM 角色(Role)

创建具有以下权限的 IAM 角色：

{
	"Version": "2012-10-17",
	"Statement": [
		{
			"Sid": "AWSLambdaVPCAccessExecutionPermissions",
			"Effect": "Allow",
			"Action": [
				"logs:CreateLogGroup",
				"logs:CreateLogStream",
				"logs:PutLogEvents",
				"ec2:CreateNetworkInterface",
				"ec2:DescribeNetworkInterfaces",
				"ec2:DescribeSubnets",
				"ec2:DeleteNetworkInterface",
				"ec2:AssignPrivateIpAddresses",
				"ec2:UnassignPrivateIpAddresses"
			],
			"Resource": "*"
		},
		{
			"Effect": "Allow",
			"Action": [
				"es:ESHttpPut",
				"es:ESHttpPost"
			],
			"Resource": "arn:aws-cn:es:cn-northwest-1:{account_id}:domain/logcenter/*"
		}
	]
}

环境变量

配置以下环境变量供 Lambda 函数使用：

变量名	描述	示例值
ENDPOINT	OpenSearch 域端点	https://search-domain.region.es.amazonaws.com
INDEX	OpenSearch 索引名称	cloudwatch-logs

Lambda 函数实现

// v1.1.2
var https = require('https');
var zlib = require('zlib');
var crypto = require('crypto');

var endpoint = process.env.ENDPOINT;
var index = process.env.INDEX;

// Set this to true if you want to debug why data isn't making it to
// your Elasticsearch cluster. This will enable logging of failed items
// to CloudWatch Logs.
var logFailedResponses = false;

exports.handler = function(input, context) {
    // decode input from base64
    var zippedInput = new Buffer.from(input.awslogs.data, 'base64');

    // decompress the input
    zlib.gunzip(zippedInput, function(error, buffer) {
        if (error) { context.fail(error); return; }

        // parse the input from JSON
        var awslogsData = JSON.parse(buffer.toString('utf8'));

        // transform the input to Elasticsearch documents
        var elasticsearchBulkData = transform(awslogsData);

        // skip control messages
        if (!elasticsearchBulkData) {
            console.log('Received a control message');
            context.succeed('Control message handled successfully');
            return;
        }

        // post documents to the Amazon Elasticsearch Service
        post(elasticsearchBulkData, function(error, success, statusCode, failedItems) {
            console.log('Response: ' + JSON.stringify({
                "statusCode": statusCode
            }));

            if (error) {
                logFailure(error, failedItems);
                context.fail(JSON.stringify(error));
            } else {
                console.log('Success: ' + JSON.stringify(success));
                context.succeed('Success');
            }
        });
    });
};

function transform(payload) {
    if (payload.messageType === 'CONTROL_MESSAGE') {
        return null;
    }

    var bulkRequestBody = '';

    payload.logEvents.forEach(function(logEvent) {
        var timestamp = new Date(1 * logEvent.timestamp);

        // index name format: cwl-YYYY.MM.DD
        // Calculate the year and week number
        var startOfYear = new Date(timestamp.getUTCFullYear(), 0, 1);
        var diff = timestamp - startOfYear;
        var oneDay = 86400000; // milliseconds in a day
        var dayOfYear = Math.floor(diff / oneDay);
        var weekNumber = Math.ceil((dayOfYear + startOfYear.getUTCDay() + 1) / 7);

        // index name format: yyyy.w
        var indexName = [
            index + '-' + timestamp.getUTCFullYear(), // year
            'w' + ('0' + weekNumber).slice(-2) // week
        ].join('.');

        var source = buildSource(logEvent.message, logEvent.extractedFields);
        source['@id'] = logEvent.id;
        source['@timestamp'] = new Date(1 * logEvent.timestamp).toISOString();
        source['@message'] = logEvent.message;
        source['@owner'] = payload.owner;
        source['@log_group'] = payload.logGroup;
        source['@log_stream'] = payload.logStream;

        var action = { "index": {} };
        action.index._index = indexName;
        action.index._id = logEvent.id;

        bulkRequestBody += [
            JSON.stringify(action),
            JSON.stringify(source),
        ].join('\n') + '\n';
    });
    return bulkRequestBody;
}

function buildSource(message, extractedFields) {
    if (extractedFields) {
        var source = {};

        for (var key in extractedFields) {
            if (extractedFields.hasOwnProperty(key) && extractedFields[key]) {
                var value = extractedFields[key];

                if (isNumeric(value)) {
                    source[key] = 1 * value;
                    continue;
                }

                var jsonSubString = extractJson(value);
                if (jsonSubString !== null) {
                    source['$' + key] = JSON.parse(jsonSubString);
                }

                source[key] = value;
            }
        }
        return source;
    }

    var jsonSubString = extractJson(message);
    if (jsonSubString !== null) {
        return JSON.parse(jsonSubString);
    }

    return {};
}

function extractJson(message) {
    var jsonStart = message.indexOf('{');
    if (jsonStart < 0) return null;
    var jsonSubString = message.substring(jsonStart);
    return isValidJson(jsonSubString) ? jsonSubString : null;
}

function isValidJson(message) {
    try {
        JSON.parse(message);
    } catch (e) { return false; }
    return true;
}

function isNumeric(n) {
    return !isNaN(parseFloat(n)) && isFinite(n);
}

function post(body, callback) {
    var requestParams = buildRequest(endpoint, body);

    var request = https.request(requestParams, function(response) {
        var responseBody = '';
        response.on('data', function(chunk) {
            responseBody += chunk;
        });

        response.on('end', function() {
            var info = JSON.parse(responseBody);
            var failedItems;
            var success;
            var error;

            if (response.statusCode >= 200 && response.statusCode < 299) {
                failedItems = info.items.filter(function(x) {
                    return x.index.status >= 300;
                });

                success = {
                    "attemptedItems": info.items.length,
                    "successfulItems": info.items.length - failedItems.length,
                    "failedItems": failedItems.length
                };
            }

            if (response.statusCode !== 200 || info.errors === true) {
                // prevents logging of failed entries, but allows logging
                // of other errors such as access restrictions
                delete info.items;
                error = {
                    statusCode: response.statusCode,
                    responseBody: info
                };
            }

            callback(error, success, response.statusCode, failedItems);
        });
    }).on('error', function(e) {
        callback(e);
    });
    request.end(requestParams.body);
}

function buildRequest(endpoint, body) {
    var endpointParts = endpoint.match(/^([^\.]+)\.?([^\.]*)\.?([^\.]*)\.amazonaws\.com.cn$/);
    var region = endpointParts[2];
    var service = endpointParts[3];
    var datetime = (new Date()).toISOString().replace(/[:\-]|\.\d{3}/g, '');
    var date = datetime.substr(0, 8);
    var kDate = hmac('AWS4' + process.env.AWS_SECRET_ACCESS_KEY, date);
    var kRegion = hmac(kDate, region);
    var kService = hmac(kRegion, service);
    var kSigning = hmac(kService, 'aws4_request');

    var request = {
        host: endpoint,
        method: 'POST',
        path: '/_bulk',
        body: body,
        headers: {
            'Content-Type': 'application/json',
            'Host': endpoint,
            'Content-Length': Buffer.byteLength(body),
            'X-Amz-Security-Token': process.env.AWS_SESSION_TOKEN,
            'X-Amz-Date': datetime
        }
    };

    var canonicalHeaders = Object.keys(request.headers)
        .sort(function(a, b) { return a.toLowerCase() < b.toLowerCase() ? -1 : 1; })
        .map(function(k) { return k.toLowerCase() + ':' + request.headers[k]; })
        .join('\n');

    var signedHeaders = Object.keys(request.headers)
        .map(function(k) { return k.toLowerCase(); })
        .sort()
        .join(';');

    var canonicalString = [
        request.method,
        request.path, '',
        canonicalHeaders, '',
        signedHeaders,
        hash(request.body, 'hex'),
    ].join('\n');

    var credentialString = [ date, region, service, 'aws4_request' ].join('/');

    var stringToSign = [
        'AWS4-HMAC-SHA256',
        datetime,
        credentialString,
        hash(canonicalString, 'hex')
    ] .join('\n');

    request.headers.Authorization = [
        'AWS4-HMAC-SHA256 Credential=' + process.env.AWS_ACCESS_KEY_ID + '/' + credentialString,
        'SignedHeaders=' + signedHeaders,
        'Signature=' + hmac(kSigning, stringToSign, 'hex')
    ].join(', ');

    return request;
}

function hmac(key, str, encoding) {
    return crypto.createHmac('sha256', key).update(str, 'utf8').digest(encoding);
}

function hash(str, encoding) {
    return crypto.createHash('sha256').update(str, 'utf8').digest(encoding);
}

function logFailure(error, failedItems) {
    if (logFailedResponses) {
        console.log('Error: ' + JSON.stringify(error, null, 2));

        if (failedItems && failedItems.length > 0) {
            console.log("Failed Items: " +
                JSON.stringify(failedItems, null, 2));
        }
    }
}

CloudWatch Logs 订阅过滤器配置

创建订阅过滤器步骤

1. 登录 AWS 控制台，导航到 CloudWatch 服务
2. 在左侧边栏选择 "Logs" -> "Log groups"
3. 选择目标日志组
4. 点击 "Actions" -> "Create Lambda subscription filter"
5. 配置如下参数：
   • Lambda Function: 选择之前创建的 Lambda 函数
   • Log format: 选择 "JSON"
   • Filter pattern (可选): 例如 { $.level = "*" } 或其他适用的日志过滤模式
   • Subscription filter name: 输入过滤器名称

Filter Pattern 示例

• 所有日志: [timestamp=*Z, request_id="*", level=*, *] 或 { $.level = "*" }
• 特定级别的日志: { $.level = "ERROR" }
• 包含特定字段的日志: { $.message = "*error*" }

安全注意事项

1. 确保 Lambda 函数的角色权限最小化，仅授予必要的权限
2. 将 Lambda 函数部署在私有子网中以提高安全性
3. 配置适当的安全组规则，限制对 OpenSearch 域的访问
4. 考虑启用 OpenSearch 域的加密功能
5. 需要在OpenSearch 资源访问策略中添加Lambda执行role

故障排除

常见问题及解决方法

1. Lambda 函数无法访问 OpenSearch:

   • 检查 VPC、子网和安全组配置
   • 确认 Lambda 角色具有适当的权限

2. 日志未正确解析:

   • 检查订阅过滤器的 Filter Pattern 设置
   • 验证日志格式是否与解析逻辑匹配

3. 性能问题:

   • 调整 Lambda 函数的内存和超时设置
   • 检查批处理大小和并发执行情况

监控和日志

建议为 Lambda 函数启用 CloudWatch Logs，以便监控其执行情况和调试问题。

成本考虑

• Lambda 执行时间和请求数量会产生费用
• OpenSearch 域的实例小时数和存储会产生费用
• 网络传输可能会产生额外费用