《60天AI学习计划启动 | Day 14: AI 应用优化 - 持续改进与迭代》
Day 14: AI 应用优化 - 持续改进与迭代
学习目标
核心学习内容
1. AI 应用优化策略
优化维度：
- 提示词优化：A/B 测试不同 Prompt
- 参数调优：Temperature、Top-p 等
- 模型选择：不同模型对比
- 用户体验：响应速度、准确性
- 成本优化：Token 使用、缓存策略
2. A/B 测试
什么是 A/B 测试？
- 对比不同方案效果
- 数据驱动决策
- 持续优化改进
测试场景：
- 不同提示词
- 不同模型
- 不同参数
- 不同 UI 设计
3. 用户反馈系统
反馈类型：
- 点赞/点踩
- 评分
- 文本反馈
- 使用行为数据
实践作业
作业1：实现 A/B 测试系统
src/services/ab-test.js：
import { chatWithAI } from './openai.js';
import { logger } from '../utils/logger.js';
/**
* A/B 测试服务
*/
export class ABTestService {
constructor() {
this.tests = new Map();
this.results = new Map();
}
/**
* 创建 A/B 测试
*/
createTest(testId, variants) {
this.tests.set(testId, {
id: testId,
variants: variants,
createdAt: Date.now()
});
this.results.set(testId, {
variants: variants.map(v => ({
id: v.id,
name: v.name,
impressions: 0,
conversions: 0,
ratings: [],
avgRating: 0
}))
});
logger.info(`创建 A/B 测试: ${testId}`);
}
/**
* 获取测试变体（随机分配）
*/
getVariant(testId, userId = null) {
const test = this.tests.get(testId);
if (!test) {
return null;
}
// 基于用户ID的稳定分配（同一用户总是看到同一变体）
if (userId) {
const hash = this.hashUserId(userId);
const index = hash % test.variants.length;
return test.variants[index];
}
// 随机分配
const index = Math.floor(Math.random() * test.variants.length);
return test.variants[index];
}
/**
* 记录展示
*/
recordImpression(testId, variantId) {
const result = this.results.get(testId);
if (!result) return;
const variant = result.variants.find(v => v.id === variantId);
if (variant) {
variant.impressions++;
}
}
/**
* 记录转化（用户满意）
*/
recordConversion(testId, variantId, rating = null) {
const result = this.results.get(testId);
if (!result) return;
const variant = result.variants.find(v => v.id === variantId);
if (variant) {
variant.conversions++;
if (rating !== null) {
variant.ratings.push(rating);
variant.avgRating =
variant.ratings.reduce((a, b) => a + b, 0) / variant.ratings.length;
}
}
}
/**
* 获取测试结果
*/
getResults(testId) {
const result = this.results.get(testId);
if (!result) return null;
return {
testId: testId,
variants: result.variants.map(v => ({
id: v.id,
name: v.name,
impressions: v.impressions,
conversions: v.conversions,
conversionRate: v.impressions > 0
? (v.conversions / v.impressions * 100).toFixed(2) + '%'
: '0%',
avgRating: v.avgRating.toFixed(2),
totalRatings: v.ratings.length
})),
winner: this.determineWinner(result.variants)
};
}
/**
* 确定获胜者
*/
determineWinner(variants) {
if (variants.length === 0) return null;
// 基于转化率和评分
const scored = variants.map(v => {
const conversionRate = v.impressions > 0
? v.conversions / v.impressions
: 0;
const score = conversionRate * 0.6 + (v.avgRating / 5) * 0.4;
return { variant: v, score };
});
scored.sort((a, b) => b.score - a.score);
return scored[0].variant.id;
}
/**
* 用户ID哈希（稳定分配）
*/
hashUserId(userId) {
let hash = 0;
for (let i = 0; i < userId.length; i++) {
const char = userId.charCodeAt(i);
hash = ((hash << 5) - hash) + char;
hash = hash & hash; // 转换为32位整数
}
return Math.abs(hash);
}
}
export const abTestService = new ABTestService();
作业2：实现提示词 A/B 测试
src/services/prompt-ab-test.js：
import { abTestService } from './ab-test.js';
import { chatWithAI } from './openai.js';
import { logger } from '../utils/logger.js';
/**
* 提示词 A/B 测试
*/
export class PromptABTest {
constructor() {
this.testId = 'prompt_optimization';
this.initializeTest();
}
/**
* 初始化测试
*/
initializeTest() {
abTestService.createTest(this.testId, [
{
id: 'variant_a',
name: '基础提示词',
prompt: `你是一个友好的AI助手，回答问题要简洁明了。`
},
{
id: 'variant_b',
name: '详细提示词',
prompt: `你是一个专业的AI助手。
回答要求：
1. 详细解释
2. 提供示例
3. 给出最佳实践
4. 使用 Markdown 格式化`
},
{
id: 'variant_c',
name: '结构化提示词',
prompt: `你是一个AI助手。
回答格式：
## 简要回答
[核心答案]
## 详细说明
[详细解释]
## 示例
[代码示例]
## 注意事项
[重要提示]`
}
]);
}
/**
* 测试聊天（自动选择变体）
*/
async chat(message, userId = null) {
// 获取变体
const variant = abTestService.getVariant(this.testId, userId);
if (!variant) {
// 降级到默认
return await chatWithAI(message);
}
// 记录展示
abTestService.recordImpression(this.testId, variant.id);
// 使用变体的提示词
const systemPrompt = variant.prompt;
const result = await chatWithAI(message, [], {
systemPrompt: systemPrompt
});
return {
...result,
variantId: variant.id,
variantName: variant.name
};
}
/**
* 记录用户反馈
*/
recordFeedback(variantId, rating, feedback = null) {
abTestService.recordConversion(this.testId, variantId, rating);
if (feedback) {
logger.info(`用户反馈 [${variantId}]: ${feedback}`);
}
}
/**
* 获取测试结果
*/
getResults() {
return abTestService.getResults(this.testId);
}
}
export const promptABTest = new PromptABTest();
作业3：实现用户反馈系统
src/services/feedback.js：
import fs from 'fs/promises';
import path from 'path';
import { logger } from '../utils/logger.js';
/**
* 用户反馈服务
*/
export class FeedbackService {
constructor() {
this.feedbackPath = './data/feedback.json';
this.feedbacks = [];
}
/**
* 初始化
*/
async initialize() {
try {
await this.loadFeedbacks();
} catch (error) {
logger.warn('加载反馈失败，创建新文件');
this.feedbacks = [];
}
}
/**
* 提交反馈
*/
async submitFeedback(feedback) {
const feedbackRecord = {
id: this.generateId(),
...feedback,
timestamp: Date.now(),
createdAt: new Date().toISOString()
};
this.feedbacks.push(feedbackRecord);
await this.saveFeedbacks();
logger.info(`收到反馈: ${feedbackRecord.id}`);
return feedbackRecord;
}
/**
* 获取反馈统计
*/
getStatistics(timeRange = 'all') {
let filtered = this.feedbacks;
if (timeRange !== 'all') {
const now = Date.now();
const ranges = {
'24h': 24 * 60 * 60 * 1000,
'7d': 7 * 24 * 60 * 60 * 1000,
'30d': 30 * 24 * 60 * 60 * 1000
};
const range = ranges[timeRange];
if (range) {
filtered = this.feedbacks.filter(f =>
now - f.timestamp < range
);
}
}
const stats = {
total: filtered.length,
ratings: {
average: 0,
distribution: { 1: 0, 2: 0, 3: 0, 4: 0, 5: 0 }
},
types: {
positive: 0,
negative: 0,
neutral: 0
},
commonIssues: []
};
// 计算评分
const ratings = filtered.filter(f => f.rating).map(f => f.rating);
if (ratings.length > 0) {
stats.ratings.average =
ratings.reduce((a, b) => a + b, 0) / ratings.length;
ratings.forEach(r => {
stats.ratings.distribution[r] =
(stats.ratings.distribution[r] || 0) + 1;
});
}
// 分析反馈类型
filtered.forEach(f => {
if (f.rating >= 4) {
stats.types.positive++;
} else if (f.rating <= 2) {
stats.types.negative++;
} else {
stats.types.neutral++;
}
});
// 提取常见问题（简单关键词分析）
const issues = filtered
.filter(f => f.rating <= 2 && f.comment)
.map(f => f.comment.toLowerCase());
// 简单的关键词统计
const issueCounts = {};
issues.forEach(issue => {
const keywords = ['慢', '错误', '不准确', '不好', '问题'];
keywords.forEach(keyword => {
if (issue.includes(keyword)) {
issueCounts[keyword] = (issueCounts[keyword] || 0) + 1;
}
});
});
stats.commonIssues = Object.entries(issueCounts)
.sort((a, b) => b[1] - a[1])
.slice(0, 5)
.map(([keyword, count]) => ({ keyword, count }));
return stats;
}
/**
* 获取反馈列表
*/
getFeedbacks(filters = {}) {
let filtered = this.feedbacks;
if (filters.rating) {
filtered = filtered.filter(f => f.rating === filters.rating);
}
if (filters.type) {
filtered = filtered.filter(f => f.type === filters.type);
}
if (filters.startDate) {
filtered = filtered.filter(f => f.timestamp >= filters.startDate);
}
if (filters.endDate) {
filtered = filtered.filter(f => f.timestamp <= filters.endDate);
}
// 排序
filtered.sort((a, b) => b.timestamp - a.timestamp);
// 分页
if (filters.limit) {
filtered = filtered.slice(0, filters.limit);
}
return filtered;
}
/**
* 保存反馈
*/
async saveFeedbacks() {
try {
await fs.mkdir(path.dirname(this.feedbackPath), { recursive: true });
await fs.writeFile(
this.feedbackPath,
JSON.stringify(this.feedbacks, null, 2)
);
} catch (error) {
logger.error('保存反馈失败:', error);
}
}
/**
* 加载反馈
*/
async loadFeedbacks() {
try {
const data = await fs.readFile(this.feedbackPath, 'utf-8');
this.feedbacks = JSON.parse(data);
logger.info(`加载了 ${this.feedbacks.length} 条反馈`);
} catch (error) {
if (error.code === 'ENOENT') {
this.feedbacks = [];
} else {
throw error;
}
}
}
/**
* 生成ID
*/
generateId() {
return `feedback_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
}
export const feedbackService = new FeedbackService();
作业4：实现性能监控
src/services/performance-monitor.js：
import { logger } from '../utils/logger.js';
/**
* 性能监控服务
*/
export class PerformanceMonitor {
constructor() {
this.metrics = {
requests: [],
responseTimes: [],
tokenUsage: [],
errors: []
};
this.maxRecords = 10000; // 最多保存10000条记录
}
/**
* 记录请求
*/
recordRequest(request) {
const record = {
id: this.generateId(),
...request,
timestamp: Date.now()
};
this.metrics.requests.push(record);
// 限制记录数量
if (this.metrics.requests.length > this.maxRecords) {
this.metrics.requests.shift();
}
return record.id;
}
/**
* 记录响应时间
*/
recordResponseTime(requestId, duration, details = {}) {
const record = {
requestId,
duration,
...details,
timestamp: Date.now()
};
this.metrics.responseTimes.push(record);
if (this.metrics.responseTimes.length > this.maxRecords) {
this.metrics.responseTimes.shift();
}
}
/**
* 记录 Token 使用
*/
recordTokenUsage(requestId, usage) {
const record = {
requestId,
promptTokens: usage.prompt_tokens || 0,
completionTokens: usage.completion_tokens || 0,
totalTokens: usage.total_tokens || 0,
timestamp: Date.now()
};
this.metrics.tokenUsage.push(record);
if (this.metrics.tokenUsage.length > this.maxRecords) {
this.metrics.tokenUsage.shift();
}
}
/**
* 记录错误
*/
recordError(error, context = {}) {
const record = {
id: this.generateId(),
message: error.message,
stack: error.stack,
...context,
timestamp: Date.now()
};
this.metrics.errors.push(record);
if (this.metrics.errors.length > this.maxRecords) {
this.metrics.errors.shift();
}
logger.error('性能监控 - 错误:', record);
}
/**
* 获取性能统计
*/
getStatistics(timeRange = '24h') {
const now = Date.now();
const ranges = {
'1h': 60 * 60 * 1000,
'24h': 24 * 60 * 60 * 1000,
'7d': 7 * 24 * 60 * 60 * 1000
};
const range = ranges[timeRange] || ranges['24h'];
const cutoff = now - range;
// 过滤时间范围内的数据
const responseTimes = this.metrics.responseTimes.filter(
r => r.timestamp >= cutoff
);
const tokenUsage = this.metrics.tokenUsage.filter(
r => r.timestamp >= cutoff
);
const errors = this.metrics.errors.filter(
r => r.timestamp >= cutoff
);
// 计算统计
const stats = {
timeRange: timeRange,
responseTime: {
average: 0,
min: 0,
max: 0,
p50: 0,
p95: 0,
p99: 0
},
tokenUsage: {
total: 0,
average: 0,
promptAverage: 0,
completionAverage: 0
},
errors: {
count: errors.length,
rate: 0
},
requests: {
total: responseTimes.length,
success: responseTimes.length - errors.length
}
};
// 响应时间统计
if (responseTimes.length > 0) {
const durations = responseTimes.map(r => r.duration).sort((a, b) => a - b);
stats.responseTime.average =
durations.reduce((a, b) => a + b, 0) / durations.length;
stats.responseTime.min = durations[0];
stats.responseTime.max = durations[durations.length - 1];
stats.responseTime.p50 = durations[Math.floor(durations.length * 0.5)];
stats.responseTime.p95 = durations[Math.floor(durations.length * 0.95)];
stats.responseTime.p99 = durations[Math.floor(durations.length * 0.99)];
}
// Token 使用统计
if (tokenUsage.length > 0) {
stats.tokenUsage.total =
tokenUsage.reduce((sum, r) => sum + r.totalTokens, 0);
stats.tokenUsage.average =
stats.tokenUsage.total / tokenUsage.length;
stats.tokenUsage.promptAverage =
tokenUsage.reduce((sum, r) => sum + r.promptTokens, 0) / tokenUsage.length;
stats.tokenUsage.completionAverage =
tokenUsage.reduce((sum, r) => sum + r.completionTokens, 0) / tokenUsage.length;
}
// 错误率
if (responseTimes.length > 0) {
stats.errors.rate = (errors.length / responseTimes.length * 100).toFixed(2) + '%';
}
return stats;
}
/**
* 生成性能报告
*/
generateReport(timeRange = '24h') {
const stats = this.getStatistics(timeRange);
return `
=== 性能监控报告 (${timeRange}) ===
请求统计：
- 总请求数: ${stats.requests.total}
- 成功请求: ${stats.requests.success}
- 错误数: ${stats.errors.count}
- 错误率: ${stats.errors.rate}
响应时间：
- 平均: ${stats.responseTime.average.toFixed(2)}ms
- 最小: ${stats.responseTime.min}ms
- 最大: ${stats.responseTime.max}ms
- P50: ${stats.responseTime.p50}ms
- P95: ${stats.responseTime.p95}ms
- P99: ${stats.responseTime.p99}ms
Token 使用：
- 总计: ${stats.tokenUsage.total}
- 平均: ${stats.tokenUsage.average.toFixed(0)}
- Prompt 平均: ${stats.tokenUsage.promptAverage.toFixed(0)}
- Completion 平均: ${stats.tokenUsage.completionAverage.toFixed(0)}
`;
}
/**
* 生成ID
*/
generateId() {
return `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
}
export const performanceMonitor = new PerformanceMonitor();
作业5：创建优化 API
src/routes/optimization.js：
import express from 'express';
import { promptABTest } from '../services/prompt-ab-test.js';
import { feedbackService } from '../services/feedback.js';
import { performanceMonitor } from '../services/performance-monitor.js';
import { logger } from '../utils/logger.js';
export const optimizationRouter = express.Router();
// 初始化反馈服务
feedbackService.initialize().catch(err => {
logger.error('反馈服务初始化失败:', err);
});
// GET /api/optimization/ab-test/results - A/B 测试结果
optimizationRouter.get('/ab-test/results', (req, res) => {
try {
const results = promptABTest.getResults();
res.json({
success: true,
data: results
});
} catch (error) {
logger.error('获取 A/B 测试结果错误:', error);
res.status(500).json({
success: false,
error: error.message
});
}
});
// POST /api/optimization/feedback - 提交反馈
optimizationRouter.post('/feedback', async (req, res) => {
try {
const { rating, comment, type, metadata } = req.body;
if (!rating || rating < 1 || rating > 5) {
return res.status(400).json({
success: false,
error: '评分必须在 1-5 之间'
});
}
const feedback = await feedbackService.submitFeedback({
rating,
comment,
type: type || 'general',
metadata: metadata || {}
});
res.json({
success: true,
data: feedback
});
} catch (error) {
logger.error('提交反馈错误:', error);
res.status(500).json({
success: false,
error: error.message
});
}
});
// GET /api/optimization/feedback/stats - 反馈统计
optimizationRouter.get('/feedback/stats', async (req, res) => {
try {
const { timeRange = 'all' } = req.query;
const stats = feedbackService.getStatistics(timeRange);
res.json({
success: true,
data: stats
});
} catch (error) {
logger.error('获取反馈统计错误:', error);
res.status(500).json({
success: false,
error: error.message
});
}
});
// GET /api/optimization/performance/stats - 性能统计
optimizationRouter.get('/performance/stats', (req, res) => {
try {
const { timeRange = '24h' } = req.query;
const stats = performanceMonitor.getStatistics(timeRange);
res.json({
success: true,
data: stats
});
} catch (error) {
logger.error('获取性能统计错误:', error);
res.status(500).json({
success: false,
error: error.message
});
}
});
// GET /api/optimization/performance/report - 性能报告
optimizationRouter.get('/performance/report', (req, res) => {
try {
const { timeRange = '24h' } = req.query;
const report = performanceMonitor.generateReport(timeRange);
res.json({
success: true,
data: { report }
});
} catch (error) {
logger.error('生成性能报告错误:', error);
res.status(500).json({
success: false,
error: error.message
});
}
});
作业6：前端反馈组件
FeedbackWidget.vue：
<template>
<div class="feedback-widget">
<button
@click="showFeedback = !showFeedback"
class="feedback-btn">
💬 反馈
</button>
<div v-if="showFeedback" class="feedback-modal" @click.self="showFeedback = false">
<div class="feedback-content">
<h3>您的反馈对我们很重要</h3>
<div class="rating-section">
<label>评分：</label>
<div class="stars">
<span
v-for="star in 5"
:key="star"
:class="['star', { active: star <= rating }]"
@click="rating = star">
⭐
</span>
</div>
</div>
<div class="comment-section">
<label>评论（可选）：</label>
<textarea
v-model="comment"
placeholder="请分享您的想法..."
rows="4" />
</div>
<div class="actions">
<button @click="submitFeedback" :disabled="submitting" class="btn-submit">
{{ submitting ? '提交中...' : '提交' }}
</button>
<button @click="showFeedback = false" class="btn-cancel">
取消
</button>
</div>
</div>
</div>
</div>
</template>
<script>
export default {
name: 'FeedbackWidget',
data() {
return {
showFeedback: false,
rating: 0,
comment: '',
submitting: false
};
},
methods: {
async submitFeedback() {
if (this.rating === 0) {
this.$message.warning('请先评分');
return;
}
this.submitting = true;
try {
const response = await fetch('http://localhost:3000/api/optimization/feedback', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
rating: this.rating,
comment: this.comment,
type: 'user_feedback'
})
});
const result = await response.json();
if (result.success) {
this.$message.success('感谢您的反馈！');
this.showFeedback = false;
this.rating = 0;
this.comment = '';
}
} catch (error) {
this.$message.error('提交失败，请稍后重试');
} finally {
this.submitting = false;
}
}
}
};
</script>
<style scoped>
.feedback-widget {
position: fixed;
bottom: 20px;
right: 20px;
z-index: 1000;
}
.feedback-btn {
padding: 12px 20px;
background: #1976d2;
color: white;
border: none;
border-radius: 24px;
cursor: pointer;
box-shadow: 0 2px 8px rgba(0,0,0,0.2);
}
.feedback-modal {
position: fixed;
top: 0;
left: 0;
right: 0;
bottom: 0;
background: rgba(0,0,0,0.5);
display: flex;
align-items: center;
justify-content: center;
}
.feedback-content {
background: white;
padding: 24px;
border-radius: 8px;
max-width: 500px;
width: 90%;
}
.stars {
display: flex;
gap: 8px;
margin-top: 8px;
}
.star {
font-size: 24px;
cursor: pointer;
opacity: 0.3;
transition: opacity 0.2s;
}
.star.active {
opacity: 1;
}
.comment-section {
margin-top: 20px;
}
.comment-section textarea {
width: 100%;
padding: 8px;
border: 1px solid #ddd;
border-radius: 4px;
margin-top: 8px;
}
.actions {
margin-top: 20px;
display: flex;
gap: 10px;
justify-content: flex-end;
}
.btn-submit {
padding: 10px 20px;
background: #1976d2;
color: white;
border: none;
border-radius: 4px;
cursor: pointer;
}
.btn-cancel {
padding: 10px 20px;
background: #f5f5f5;
border: none;
border-radius: 4px;
cursor: pointer;
}
</style>
遇到的问题
问题1：A/B 测试样本不足
解决方案：
// 设置最小样本量
const minSamples = 100;
if (variant.impressions < minSamples) {
return '样本量不足，继续测试';
}
问题2：性能数据过多
解决方案：
// 定期清理旧数据
if (this.metrics.requests.length > this.maxRecords) {
this.metrics.requests = this.metrics.requests.slice(-this.maxRecords);
}
学习总结
今日收获
- ✅ 理解 AI 应用优化策略
- ✅ 实现 A/B 测试系统
- ✅ 收集用户反馈
- ✅ 性能监控
- ✅ 数据分析
关键知识点
- A/B 测试，数据驱动优化
- 用户反馈，持续改进
- 性能监控，及时发现问题
- 数据分析，指导决策
优化流程
收集数据 → 分析问题 → 提出方案 → A/B 测试 → 评估结果 → 迭代优化
明日计划
明天将学习：
期待明天的学习！ 🚀
参考资源
代码仓库
项目已更新：
- ✅ A/B 测试系统
- ✅ 用户反馈系统
- ✅ 性能监控
- ✅ 优化 API
GitHub 提交： Day 14 - AI 应用优化
标签： #AI学习 #应用优化 #A/B测试 #用户反馈 #性能监控 #学习笔记
写在最后
今天学习了 AI 应用的优化策略，这是持续改进的关键。
通过 A/B 测试、用户反馈和性能监控，可以不断优化应用。
明天将进行综合项目实战，整合所有知识点！
继续加油！ 💪
快速检查清单
完成这些，第十四天就达标了！ ✅
浙公网安备 33010602011771号