云开发-fastGPT接口调用 将日志文件导入到MinIO库

云开发-fastGPT日志导入到MinIO库

传参：

work_flow 名称(name)、 起始时间(dateStart)、 结束时间(dateEnd)

代码实现：

import { createObjectCsvWriter } from 'csv-writer'
import fs from 'fs';
const formatDate = (cellValue) => {
  if (cellValue === null || cellValue === '') return ''
  var date = new Date(cellValue)
  var year = date.getFullYear()   
  var month = date.getMonth() + 1 < 10 ? '0' + (date.getMonth() + 1) : date.getMonth() + 1
  var day = date.getDate() < 10 ? '0' + date.getDate() : date.getDate()
  var hours = date.getHours() < 10 ? '0' + date.getHours() : date.getHours()
  var minutes = date.getMinutes() < 10 ? '0' + date.getMinutes() : date.getMinutes()
  var seconds = date.getSeconds() < 10 ? '0' + date.getSeconds() : date.getSeconds()
  return year + '-' + month + '-' + day + ' ' + hours + ':' + minutes + ':' + seconds
}
// fastGpt导出功能
export default async function (ctx: FunctionContext) {
  // 登陆
  const loginRes = await fetch(`https://fastgpt.geyuandiaoyan.com/api/support/user/account/loginByPassword`, {
	method: 'POST',
	headers: {
	  'Content-Type': 'application/json'
	},
	body: JSON.stringify({
	  password: "edbba7268fc1543b4a72a0cabaf7dc617f6cf8fdef02b5814b731b835911e10d",
	  username: "root"
	}),
  })
  const loginResponse = await loginRes.json();
  let cookieStr = 'fastgpt_token=' + loginResponse.data.token;
  // 获取工作台列表
  const workbenchesRes = await fetch(`https://fastgpt.geyuandiaoyan.com/api/core/app/list`, {
	method: 'POST',
	headers: {
	  'Content-Type': 'application/json',
	  'cookie': cookieStr
	},
	body: JSON.stringify({
	  parentId: null,
	  searchKey: ""
	}),
  })
  const workbenchesResponse = await workbenchesRes.json();
  // console.log(workbenchesResponse.data)
  // 根据前端传入的name
  const name = ctx.query.name;
  // 根据前端传入的dateEnd
  const dateEnd = ctx.query.dateEnd;
  // 根据前端传入的dateStart
  const dateStart = ctx.query.dateStart;
  let appId = "";
  for (let i = 0; i < workbenchesResponse.data.length; i++) {
	if (workbenchesResponse.data[i].name === name) {
	  appId = workbenchesResponse.data[i]._id;
	  break;
	}
  }
  console.log('name:' + name + ' ; appId:' + appId); 
  console.log('dateEnd:' + dateEnd); 
  console.log('dateStart:' + dateStart); 
  if (!appId) {
	return {
	  code: 404,
	  error: '未找到匹配的应用',
	  detail: `找不到名称为 ${name} 的应用`
	};
  }
  const listRes = await fetch(`https://fastgpt.geyuandiaoyan.com/api/core/app/getChatLogs`, {
	method: 'POST',
	headers: {
	  'Content-Type': 'application/json',
	  "cookie": cookieStr
	},
	body: JSON.stringify({
	  "appId": appId,
	  "dateEnd": dateEnd,
	  "dateStart": dateStart,
	  "pageNum": 1,
	  "pageSize": 100
	}),
  })

  const listResult = await listRes.json();
  const csvData = [];
  let listTemp = listResult.data.list
  for (let i = 0; i < listTemp.length; i++) {
	let count = listTemp[i].messageCount;
	let titleStr = listTemp[i].title;
	let time = listTemp[i].time;
	let timeStr = formatDate(time);
	const detailRes = await fetch(`https://fastgpt.geyuandiaoyan.com/api/core/chat/getPaginationRecords`, {
	  method: 'POST',
	  headers: {
		'Content-Type': 'application/json',
		"cookie": cookieStr
	  },
	  body: JSON.stringify({
		"appId": appId,
		"chatId": listTemp[i].id,
		"loadCustomFeedbacks": true,
		"offset": 0,
		"pageSize": 100,
		"type": "normal"
	  }),
	})
	const detailResult = await detailRes.json()
	let contentStr = '';
	for (let j = 0; j < detailResult.data.list.length; j++) {
	  let content = '';
	  if (detailResult.data.list[j].value[0].text?.content != undefined) {
		content = detailResult.data.list[j].value[0].text?.content;
	  } else if (detailResult.data.list[j].value[1].text?.content != undefined) {
		content = detailResult.data.list[j].value[1].text?.content;
	   }
	  contentStr += "<" + (j + 1) + "> " + content + ' \n\n\n\n\n';
	}


	// 处理问答对时保留完整格式
	const lines = contentStr.split('\n\n\n\n\n'); // 改为按换行分割
	const question = lines[0].replace('<1>', '').trim();
	let answer = "";

	for (let j = 1; j < lines.length; j++) {
		  answer += lines[j].replace('<2>', '').trim();
	}
	// console.log('ddd' + question);
	// console.log('aaa' + answer);

	  csvData.push({
		标题: titleStr,
		时间: timeStr,
		问题: question,
		回答: answer.replace(/\n/g, '\\n') // 保留换行符但转义
	  });

  }
  let csvPath = '/tmp/chat_logs.csv';
  // 创建CSV文件
  const csvWriter = createObjectCsvWriter({
	path: csvPath,
	header: [
	  { id: '标题', title: '标题' },
	  { id: '时间', title: '时间' },
	  { id: '问题', title: '问题' },
	  { id: '回答', title: '回答' }
	],
	 encoding: 'utf8',  // 明确指定编码格式
	alwaysQuote: true
  });
  // 写入
  await csvWriter.writeRecords(csvData);
  // 添加UTF-8 BOM头
  const fileContent = fs.readFileSync(csvPath, 'utf8');
  fs.writeFileSync(csvPath, '\uFEFF' + fileContent, 'utf8');

  const fileBuffer = fs.readFileSync(csvPath);
  // 文件上传
	const MinIO = require('minio'); 

  const client = new MinIO.Client({
	 // 内网地址
	endPoint: 'minio-tglmtmiv.ns-a23i8mr4.svc.cluster.local',
	port:9000,
	useSSL:false,
	accessKey: 'TctZ9rVfRxLLi6cmoJMr',
	secretKey: 'B3BzrzbHX9CCULIqsJ0uop84IB8CAVAO5qV5L2YM',
	transportOptions: {
	  connectTimeout: 5000, // 10秒连接超时
	  timeout: 5000 // 30秒操作超时
	}
	});
  console.log('MinIO客户端已初始化'); // 添加调试日志

  // 添加超时处理
  const timeout = setTimeout(() => {
	console.error('连接超时，请检查网络或服务状态');
  }, 5000);

  // 测试连接是否成功
  client.listBuckets(function (err, buckets) {
	clearTimeout(timeout);
	if (err) {
	  console.error('连接失败:', err);
	  console.error('详细错误信息:', err.stack);
	  return;
	}
	console.log('连接成功，存储桶列表:', buckets);
  });
  const bucketName = 'chatlogs';
  // 获取当前时间并调整为北京时间(UTC+8)
  const now = new Date();
  const beijingOffset = 8 * 60 * 60 * 1000; // 8小时的毫秒数
  const beijingTime = new Date(now.getTime() + beijingOffset);

  // 格式化为yyyyMMddHHmmss
  const formattedDate =
	beijingTime.getUTCFullYear() +
	String(beijingTime.getUTCMonth() + 1).padStart(2, '0') +
	String(beijingTime.getUTCDate()).padStart(2, '0') +
	String(beijingTime.getUTCHours()).padStart(2, '0') +
	String(beijingTime.getUTCMinutes()).padStart(2, '0') +
	String(beijingTime.getUTCSeconds()).padStart(2, '0');


  const objectName = name + `_chat_logs_${formattedDate}.csv`.replace(/\//g, '_');

  try {
	// 检查存储桶是否存在，不存在则创建
	const bucketExists = await client.bucketExists(bucketName);
	if (!bucketExists) {
	  await client.makeBucket(bucketName, 'us-east-1');
	  console.log(`创建存储桶: ${bucketName}`);
	}

	// 上传CSV文件
	const metaData = {
	  'Content-Type': 'text/csv; charset=utf-8',
	  'Content-Disposition': 'attachment; filename="chat_logs.csv"',
	  'X-Amz-Meta-Created': new Date().toISOString()
	};

	// 上传文件到MinIO
	await client.putObject(bucketName, objectName, fileBuffer, metaData);
	console.log('CSV文件已成功上传');

	// 获取文件下载URL（有效期7天）
	const downloadUrl = await client.presignedGetObject(
	  bucketName,
	  objectName,
	  7 * 24 * 60 * 60 // 7天有效期
	);

	return {
	  code: 200,
	  data: {
		downloadUrl: downloadUrl,
		objectName: objectName
	  }
	};
  } catch (err) {
	console.error('MinIO上传失败:', err);
	return {
	  code: 500,
	  error: '文件上传失败',
	  detail: err.message
	};
  }
  }