实用指南：第47节：机器学习：3D姿态估计与动画驱动

概述

机器学习正在革命性地改变3D交互方式。本节将探索如何集成TensorFlow.js进行实时3D姿态估计，并使用估计结果驱动Three.js中的骨骼动画，实现从2D视频到3D角色的自然映射。

ML-3D系统架构：

核心原理

姿态估计算法对比

模型	精度	速度	3D支持	适用场景
PoseNet	中等	快	否	实时应用
MoveNet	高	很快	否	运动分析
MediaPipe Pose	很高	快	是	专业应用
BlazePose	极高	中等	是	医疗健身

骨骼映射原理

// 关键点映射配置
class PoseMapper {
static BODY_CONNECTIONS = [
// 身体主干
['left_shoulder', 'right_shoulder'],
['left_shoulder', 'left_hip'],
['right_shoulder', 'right_hip'],
['left_hip', 'right_hip'],
// 左臂
['left_shoulder', 'left_elbow'],
['left_elbow', 'left_wrist'],
// 右臂
['right_shoulder', 'right_elbow'],
['right_elbow', 'right_wrist'],
// 左腿
['left_hip', 'left_knee'],
['left_knee', 'left_ankle'],
// 右腿
['right_hip', 'right_knee'],
['right_knee', 'right_ankle']
];
static mapToBoneRotations(keypoints) {
const rotations = {};
for (const [start, end] of this.BODY_CONNECTIONS) {
const startPoint = keypoints.find(k => k.name === start);
const endPoint = keypoints.find(k => k.name === end);
if (startPoint && endPoint) {
const direction = new THREE.Vector3()
.subVectors(endPoint.position, startPoint.position)
.normalize();
rotations[`${start}_to_${end}`] = this.vectorToQuaternion(direction);
}
}
return rotations;
}
static vectorToQuaternion(direction) {
// 将方向向量转换为四元数
const up = new THREE.Vector3(0, 1, 0);
const quaternion = new THREE.Quaternion();
quaternion.setFromUnitVectors(up, direction);
return quaternion;
}
}

完整代码实现

实时姿态估计系统

<script>
import { onMounted, onUnmounted, ref, reactive, computed } from 'vue';
import * as THREE from 'three';
import * as poseDetection from '@tensorflow-models/pose-detection';
import '@tensorflow/tfjs-backend-webgl';
// 姿态估计管理器
class PoseEstimationManager {
  constructor() {
    this.detector = null;
    this.isInitialized = false;
    this.lastDetectionTime = 0;
  }
  async initialize(modelType = 'movenet', complexity = 'lite') {
    try {
      // 设置TensorFlow.js后端
      await tf.setBackend('webgl');
      // 创建检测器配置
      const detectorConfig = this.getDetectorConfig(modelType, complexity);
      // 加载模型
      this.detector = await poseDetection.createDetector(
        modelType === 'blazepose' ? poseDetection.SupportedModels.BlazePose :
        modelType === 'posenet' ? poseDetection.SupportedModels.PoseNet :
        poseDetection.SupportedModels.MoveNet,
        detectorConfig
      );
      this.isInitialized = true;
      console.log('姿态估计模型加载成功');
    } catch (error) {
      console.error('模型加载失败:', error);
      throw error;
    }
  }
  getDetectorConfig(modelType, complexity) {
    const baseConfig = {
      modelType: complexity.toUpperCase(),
      enableSmoothing: true,
      minPoseScore: 0.25
    };
    switch (modelType) {
      case 'movenet':
        return {
          ...baseConfig,
          modelType: complexity === 'heavy' ? 'THUNDER' : 'LIGHTNING'
        };
      case 'blazepose':
        return {
          ...baseConfig,
          runtime: 'tfjs',
          enableSmoothing: true,
          modelType: complexity === 'heavy' ? 'full' : 'lite'
        };
      case 'posenet':
        return {
          ...baseConfig,
          architecture: 'MobileNetV1',
          outputStride: 16,
          inputResolution: { width: 640, height: 480 },
          multiplier: complexity === 'heavy' ? 1.0 : 0.75
        };
      default:
        return baseConfig;
    }
  }
  async estimatePoses(videoElement, minScore = 0.3) {
    if (!this.detector || !this.isInitialized) {
      throw new Error('检测器未初始化');
    }
    const startTime = performance.now();
    try {
      const poses = await this.detector.estimatePoses(videoElement, {
        maxPoses: 1,
        flipHorizontal: false
      });
      const detectionTime = performance.now() - startTime;
      // 过滤低置信度的姿态
      const validPoses = poses.filter(pose => pose.score >= minScore);
      return {
        poses: validPoses,
        detectionTime,
        keypointCount: validPoses.length > 0 ? validPoses[0].keypoints.length : 0
      };
    } catch (error) {
      console.error('姿态估计失败:', error);
      return { poses: [], detectionTime: 0, keypointCount: 0 };
    }
  }
  dispose() {
    if (this.detector) {
      this.detector.dispose();
    }
    this.isInitialized = false;
  }
}
// 3D角色管理器
class CharacterManager {
  constructor(renderer, scene, camera) {
    this.renderer = renderer;
    this.scene = scene;
    this.camera = camera;
    this.character = null;
    this.bones = new Map();
    this.joints = new Map();
    this.setupScene();
  }
  setupScene() {
    // 基础照明
    const ambientLight = new THREE.AmbientLight(0x404040, 0.6);
    this.scene.add(ambientLight);
    const directionalLight = new THREE.DirectionalLight(0xffffff, 0.8);
    directionalLight.position.set(10, 10, 5);
    this.scene.add(directionalLight);
    // 网格地面
    const gridHelper = new THREE.GridHelper(10, 10);
    this.scene.add(gridHelper);
  }
  // 创建简笔画角色
  createStickFigure() {
    this.clearCharacter();
    const characterGroup = new THREE.Group();
    characterGroup.name = 'stick_figure';
    // 创建骨骼材质
    const boneMaterial = new THREE.MeshBasicMaterial({
      color: 0x00ff00,
      transparent: true,
      opacity: 0.8
    });
    const jointMaterial = new THREE.MeshBasicMaterial({
      color: 0xff0000,
      transparent: true,
      opacity: 0.9
    });
    // 定义骨骼连接
    const boneConnections = [
      // 身体主干
      { name: 'spine', start: 'hip', end: 'shoulder_center', radius: 0.1 },
      // 左臂
      { name: 'left_upper_arm', start: 'shoulder_center', end: 'left_elbow', radius: 0.08 },
      { name: 'left_lower_arm', start: 'left_elbow', end: 'left_wrist', radius: 0.06 },
      // 右臂
      { name: 'right_upper_arm', start: 'shoulder_center', end: 'right_elbow', radius: 0.08 },
      { name: 'right_lower_arm', start: 'right_elbow', end: 'right_wrist', radius: 0.06 },
      // 左腿
      { name: 'left_upper_leg', start: 'hip', end: 'left_knee', radius: 0.1 },
      { name: 'left_lower_leg', start: 'left_knee', end: 'left_ankle', radius: 0.08 },
      // 右腿
      { name: 'right_upper_leg', start: 'hip', end: 'right_knee', radius: 0.1 },
      { name: 'right_lower_leg', start: 'right_knee', end: 'right_ankle', radius: 0.08 }
    ];
    // 创建骨骼
    boneConnections.forEach(connection => {
      const bone = this.createBone(connection.start, connection.end, connection.radius, boneMaterial);
      characterGroup.add(bone);
      this.bones.set(connection.name, bone);
    });
    // 创建关节
    const jointPoints = [
      'hip', 'shoulder_center', 'left_elbow', 'left_wrist',
      'right_elbow', 'right_wrist', 'left_knee', 'left_ankle',
      'right_knee', 'right_ankle'
    ];
    jointPoints.forEach(jointName => {
      const joint = this.createJoint(jointName, 0.15, jointMaterial);
      characterGroup.add(joint);
      this.joints.set(jointName, joint);
    });
    this.scene.add(characterGroup);
    this.character = characterGroup;
    return characterGroup;
  }
  createBone(startJoint, endJoint, radius, material) {
    // 创建圆柱体作为骨骼
    const boneGeometry = new THREE.CylinderGeometry(radius, radius, 1, 8);
    const bone = new THREE.Mesh(boneGeometry, material);
    // 初始位置和方向
    bone.userData = { startJoint, endJoint };
    bone.rotation.order = 'YXZ';
    return bone;
  }
  createJoint(name, radius, material) {
    const jointGeometry = new THREE.SphereGeometry(radius, 8, 6);
    const joint = new THREE.Mesh(jointGeometry, material);
    joint.name = name;
    return joint;
  }
  // 更新角色姿态
  updatePose(keypoints, smoothing = 0.5) {
    if (!this.character || !keypoints || keypoints.length === 0) return;
    // 将2D关键点映射到3D空间
    const jointPositions = this.mapKeypointsTo3D(keypoints);
    // 更新关节位置
    for (const [jointName, position] of Object.entries(jointPositions)) {
      const joint = this.joints.get(jointName);
      if (joint) {
        // 应用平滑
        if (smoothing > 0) {
          position.lerp(joint.position, 1 - smoothing);
        }
        joint.position.copy(position);
      }
    }
    // 更新骨骼方向和长度
    this.updateBones();
  }
  mapKeypointsTo3D(keypoints) {
    const positions = {};
    const scale = 0.1; // 缩放因子
    // 关键点映射表
    const keypointMapping = {
      'nose': 'head',
      'left_shoulder': 'left_shoulder',
      'right_shoulder': 'right_shoulder',
      'left_elbow': 'left_elbow',
      'right_elbow': 'right_elbow',
      'left_wrist': 'left_wrist',
      'right_wrist': 'right_wrist',
      'left_hip': 'left_hip',
      'right_hip': 'right_hip',
      'left_knee': 'left_knee',
      'right_knee': 'right_knee',
      'left_ankle': 'left_ankle',
      'right_ankle': 'right_ankle'
    };
    // 计算中心点（臀部）
    const leftHip = keypoints.find(k => k.name === 'left_hip');
    const rightHip = keypoints.find(k => k.name === 'right_hip');
    if (leftHip && rightHip) {
      const hipCenter = {
        x: (leftHip.x + rightHip.x) / 2,
        y: (leftHip.y + rightHip.y) / 2
      };
      // 设置臀部位置为原点
      positions.hip = new THREE.Vector3(0, 0, 0);
      // 计算其他关节的相对位置
      for (const [keypointName, jointName] of Object.entries(keypointMapping)) {
        const keypoint = keypoints.find(k => k.name === keypointName);
        if (keypoint && keypoint.score >= 0.3) {
          const x = (keypoint.x - hipCenter.x) * scale;
          const y = -(keypoint.y - hipCenter.y) * scale; // Y轴翻转
          const z = 0; // 初始深度为0
          positions[jointName] = new THREE.Vector3(x, y, z);
        }
      }
      // 计算肩部中心
      const leftShoulder = positions.left_shoulder;
      const rightShoulder = positions.right_shoulder;
      if (leftShoulder && rightShoulder) {
        positions.shoulder_center = new THREE.Vector3()
          .addVectors(leftShoulder, rightShoulder)
          .multiplyScalar(0.5);
      }
    }
    return positions;
  }
  updateBones() {
    for (const [boneName, bone] of this.bones) {
      const startJoint = this.joints.get(bone.userData.startJoint);
      const endJoint = this.joints.get(bone.userData.endJoint);
      if (startJoint && endJoint) {
        // 计算骨骼方向
        const direction = new THREE.Vector3()
          .subVectors(endJoint.position, startJoint.position);
        const length = direction.length();
        if (length > 0) {
          // 设置骨骼位置（起点和终点的中点）
          bone.position.copy(startJoint.position)
            .add(endJoint.position)
            .multiplyScalar(0.5);
          // 设置骨骼方向
          bone.lookAt(endJoint.position);
          // 调整圆柱体方向（默认朝向Y轴）
          bone.rotateX(Math.PI / 2);
          // 设置骨骼长度
          bone.scale.set(1, length, 1);
        }
      }
    }
  }
  clearCharacter() {
    if (this.character) {
      this.scene.remove(this.character);
      // 清理几何体和材质
      this.bones.forEach(bone => {
        bone.geometry.dispose();
        bone.material.dispose();
      });
      this.joints.forEach(joint => {
        joint.geometry.dispose();
        joint.material.dispose();
      });
      this.bones.clear();
      this.joints.clear();
      this.character = null;
    }
  }
  // 重置角色姿态
  resetPose() {
    this.joints.forEach(joint => {
      joint.position.set(0, 0, 0);
    });
    this.updateBones();
  }
}
// 姿态可视化器
class PoseVisualizer {
  constructor(canvas) {
    this.canvas = canvas;
    this.ctx = canvas.getContext('2d');
    this.keypointRadius = 4;
    this.skeletonColor = '#00ff00';
    this.keypointColor = '#ff0000';
  }
  drawPose(pose, videoWidth, videoHeight) {
    if (!pose || !pose.keypoints) return;
    // 清除画布
    this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
    // 设置画布尺寸匹配视频
    this.canvas.width = videoWidth;
    this.canvas.height = videoHeight;
    // 绘制骨骼
    this.drawSkeleton(pose.keypoints);
    // 绘制关键点
    this.drawKeypoints(pose.keypoints);
  }
  drawSkeleton(keypoints) {
    const connections = [
      // 身体主干
      ['left_shoulder', 'right_shoulder'],
      ['left_shoulder', 'left_hip'],
      ['right_shoulder', 'right_hip'],
      ['left_hip', 'right_hip'],
      // 左臂
      ['left_shoulder', 'left_elbow'],
      ['left_elbow', 'left_wrist'],
      // 右臂
      ['right_shoulder', 'right_elbow'],
      ['right_elbow', 'right_wrist'],
      // 左腿
      ['left_hip', 'left_knee'],
      ['left_knee', 'left_ankle'],
      // 右腿
      ['right_hip', 'right_knee'],
      ['right_knee', 'right_ankle'],
      // 面部（简化）
      ['nose', 'left_eye'],
      ['nose', 'right_eye'],
      ['left_eye', 'left_ear'],
      ['right_eye', 'right_ear']
    ];
    this.ctx.strokeStyle = this.skeletonColor;
    this.ctx.lineWidth = 2;
    connections.forEach(([start, end]) => {
      const startPoint = keypoints.find(k => k.name === start);
      const endPoint = keypoints.find(k => k.name === end);
      if (startPoint && endPoint && startPoint.score > 0.3 && endPoint.score > 0.3) {
        this.ctx.beginPath();
        this.ctx.moveTo(startPoint.x, startPoint.y);
        this.ctx.lineTo(endPoint.x, endPoint.y);
        this.ctx.stroke();
      }
    });
  }
  drawKeypoints(keypoints) {
    keypoints.forEach(keypoint => {
      if (keypoint.score > 0.3) {
        // 绘制关键点
        this.ctx.fillStyle = this.keypointColor;
        this.ctx.beginPath();
        this.ctx.arc(keypoint.x, keypoint.y, this.keypointRadius, 0, 2 * Math.PI);
        this.ctx.fill();
        // 绘制置信度圆环
        const ringRadius = this.keypointRadius * (1 + keypoint.score);
        this.ctx.strokeStyle = `rgba(255, 255, 255, ${keypoint.score})`;
        this.ctx.lineWidth = 1;
        this.ctx.beginPath();
        this.ctx.arc(keypoint.x, keypoint.y, ringRadius, 0, 2 * Math.PI);
        this.ctx.stroke();
      }
    });
  }
}
export default {
  name: 'PoseEstimation',
  setup() {
    // 响应式状态
    const videoElement = ref(null);
    const poseCanvas = ref(null);
    const characterCanvas = ref(null);
    const isCameraActive = ref(false);
    const isLoading = ref(false);
    const loadingProgress = ref(0);
    const selectedModel = ref('movenet');
    const modelComplexity = ref('lite');
    const minPoseConfidence = ref(0.3);
    const minKeypointConfidence = ref(0.3);
    const characterType = ref('stickman');
    const boneThickness = ref(0.1);
    const jointSize = ref(0.15);
    const smoothingFactor = ref(0.5);
    const showCharacter = ref(true);
    const enableRecording = ref(false);
    const isRecording = ref(false);
    const enablePlayback = ref(false);
    const isPlaying = ref(false);
    const enable3DReconstruction = ref(false);
    const enableSkeletonSmoothing = ref(true);
    const showKeypoints = ref(true);
    const showSkeleton = ref(true);
    const mirrorMode = ref('horizontal');
    const showKeypointPanel = ref(false);
    // 性能统计
    const detectionFPS = ref(0);
    const renderFPS = ref(0);
    const detectionTime = ref(0);
    const keypointCount = ref(0);
    const trackingQuality = ref('未知');
    // 当前关键点数据
    const currentKeypoints = ref([]);
    // 计算属性
    const trackingQualityClass = computed(() => {
      const quality = trackingQuality.value;
      if (quality === '优秀') return 'excellent';
      if (quality === '良好') return 'good';
      if (quality === '一般') return 'fair';
      return 'poor';
    });
    // 管理器实例
    let poseManager, characterManager, poseVisualizer;
    let renderer, scene, camera;
    let animationFrameId;
    let detectionFrameCount = 0;
    let renderFrameCount = 0;
    let lastFpsUpdate = 0;
    let videoStream = null;
    // 初始化
    const init = async () => {
      isLoading.value = true;
      try {
        await initPoseEstimation();
        await init3DRenderer();
        initPoseVisualizer();
        isLoading.value = false;
      } catch (error) {
        console.error('初始化失败:', error);
        isLoading.value = false;
      }
    };
    // 初始化姿态估计
    const initPoseEstimation = async () => {
      poseManager = new PoseEstimationManager();
      loadingProgress.value = 50;
      await poseManager.initialize(selectedModel.value, modelComplexity.value);
      loadingProgress.value = 100;
    };
    // 初始化3D渲染器
    const init3DRenderer = () => {
      renderer = new THREE.WebGLRenderer({
        canvas: characterCanvas.value,
        antialias: true
      });
      renderer.setSize(characterCanvas.value.clientWidth, characterCanvas.value.clientHeight);
      renderer.setClearColor(0x222222);
      scene = new THREE.Scene();
      camera = new THREE.PerspectiveCamera(75,
        characterCanvas.value.clientWidth / characterCanvas.value.clientHeight,
        0.1, 1000
      );
      camera.position.set(0, 0, 5);
      camera.lookAt(0, 0, 0);
      characterManager = new CharacterManager(renderer, scene, camera);
      characterManager.createStickFigure();
    };
    // 初始化姿态可视化
    const initPoseVisualizer = () => {
      poseVisualizer = new PoseVisualizer(poseCanvas.value);
    };
    // 切换摄像头
    const toggleCamera = async () => {
      if (isCameraActive.value) {
        stopCamera();
      } else {
        await startCamera();
      }
    };
    // 启动摄像头
    const startCamera = async () => {
      try {
        const constraints = {
          video: {
            width: { ideal: 640 },
            height: { ideal: 480 },
            facingMode: 'user'
          }
        };
        videoStream = await navigator.mediaDevices.getUserMedia(constraints);
        videoElement.value.srcObject = videoStream;
        isCameraActive.value = true;
        // 开始姿态估计循环
        startPoseEstimationLoop();
      } catch (error) {
        console.error('摄像头访问失败:', error);
        alert('无法访问摄像头，请检查权限设置');
      }
    };
    // 停止摄像头
    const stopCamera = () => {
      if (videoStream) {
        videoStream.getTracks().forEach(track => track.stop());
        videoStream = null;
      }
      videoElement.value.srcObject = null;
      isCameraActive.value = false;
      if (animationFrameId) {
        cancelAnimationFrame(animationFrameId);
      }
    };
    // 视频准备就绪
    const onVideoReady = () => {
      // 设置画布尺寸匹配视频
      if (poseCanvas.value) {
        poseCanvas.value.width = videoElement.value.videoWidth;
        poseCanvas.value.height = videoElement.value.videoHeight;
      }
    };
    // 开始姿态估计循环
    const startPoseEstimationLoop = () => {
      const estimatePose = async () => {
        if (!isCameraActive.value) return;
        try {
          // 估计姿态
          const result = await poseManager.estimatePoses(
            videoElement.value,
            minPoseConfidence.value
          );
          if (result.poses.length > 0) {
            const pose = result.poses[0];
            // 更新关键点数据
            currentKeypoints.value = pose.keypoints.map(kp => ({
              name: kp.name,
              score: kp.score,
              position: { x: kp.x, y: kp.y }
            }));
            // 可视化姿态
            if (showSkeleton.value || showKeypoints.value) {
              poseVisualizer.drawPose(pose,
                videoElement.value.videoWidth,
                videoElement.value.videoHeight
              );
            }
            // 更新3D角色
            if (showCharacter.value) {
              characterManager.updatePose(pose.keypoints, smoothingFactor.value);
            }
            // 更新性能统计
            detectionTime.value = result.detectionTime.toFixed(1);
            keypointCount.value = result.keypointCount;
            detectionFrameCount++;
          }
          // 更新跟踪质量
          updateTrackingQuality(result);
        } catch (error) {
          console.error('姿态估计错误:', error);
        }
        // 继续下一帧
        animationFrameId = requestAnimationFrame(estimatePose);
      };
      estimatePose();
    };
    // 更新跟踪质量
    const updateTrackingQuality = (result) => {
      if (result.poses.length === 0) {
        trackingQuality.value = '无检测';
      } else {
        const pose = result.poses[0];
        const visibleKeypoints = pose.keypoints.filter(kp => kp.score > minKeypointConfidence.value).length;
        const totalKeypoints = pose.keypoints.length;
        const visibilityRatio = visibleKeypoints / totalKeypoints;
        if (visibilityRatio > 0.8) trackingQuality.value = '优秀';
        else if (visibilityRatio > 0.6) trackingQuality.value = '良好';
        else if (visibilityRatio > 0.4) trackingQuality.value = '一般';
        else trackingQuality.value = '较差';
      }
    };
    // 切换镜像模式
    const toggleMirror = () => {
      mirrorMode.value = mirrorMode.value === 'horizontal' ? 'none' : 'horizontal';
      // 实际实现需要更新可视化
    };
    // 截图
    const takeSnapshot = () => {
      const canvas = document.createElement('canvas');
      const ctx = canvas.getContext('2d');
      canvas.width = videoElement.value.videoWidth;
      canvas.height = videoElement.value.videoHeight;
      // 绘制视频帧
      ctx.drawImage(videoElement.value, 0, 0);
      // 绘制姿态覆盖
      if (poseCanvas.value) {
        ctx.drawImage(poseCanvas.value, 0, 0);
      }
      // 创建下载链接
      const link = document.createElement('a');
      link.download = `pose-snapshot-${Date.now()}.png`;
      link.href = canvas.toDataURL();
      link.click();
    };
    // 切换角色显示
    const toggleCharacter = () => {
      showCharacter.value = !showCharacter.value;
      if (showCharacter.value && characterManager.character) {
        characterManager.character.visible = true;
      } else if (characterManager.character) {
        characterManager.character.visible = false;
      }
    };
    // 重置姿势
    const resetPose = () => {
      characterManager.resetPose();
    };
    // 校准姿态
    const calibratePose = () => {
      // 实现校准逻辑
      alert('校准功能开发中...');
    };
    // 加载模型
    const loadModel = async () => {
      if (poseManager) {
        poseManager.dispose();
      }
      await initPoseEstimation();
    };
    // 加载角色
    const loadCharacter = () => {
      if (characterManager) {
        switch (characterType.value) {
          case 'stickman':
            characterManager.createStickFigure();
            break;
          case 'humanoid':
            // 加载人形角色
            break;
          case 'robot':
            // 加载机器人角色
            break;
        }
      }
    };
    // 开始录制
    const startRecording = () => {
      isRecording.value = true;
      // 实现录制逻辑
    };
    // 停止录制
    const stopRecording = () => {
      isRecording.value = false;
    };
    // 保存录制
    const saveRecording = () => {
      // 实现保存逻辑
      alert('保存功能开发中...');
    };
    // 加载录制
    const loadRecording = () => {
      // 实现加载逻辑
      alert('加载功能开发中...');
    };
    // 播放录制
    const playRecording = () => {
      isPlaying.value = true;
      // 实现播放逻辑
    };
    // 暂停录制
    const pauseRecording = () => {
      isPlaying.value = false;
    };
    // 性能监控循环
    const startPerformanceMonitor = () => {
      const updateStats = () => {
        const now = performance.now();
        if (now - lastFpsUpdate >= 1000) {
          detectionFPS.value = Math.round((detectionFrameCount * 1000) / (now - lastFpsUpdate));
          renderFPS.value = Math.round((renderFrameCount * 1000) / (now - lastFpsUpdate));
          detectionFrameCount = 0;
          renderFrameCount = 0;
          lastFpsUpdate = now;
        }
        requestAnimationFrame(updateStats);
      };
      updateStats();
    };
    // 3D渲染循环
    const startRenderLoop = () => {
      const render = () => {
        if (showCharacter.value) {
          renderer.render(scene, camera);
          renderFrameCount++;
        }
        requestAnimationFrame(render);
      };
      render();
    };
    onMounted(() => {
      init();
      startPerformanceMonitor();
      startRenderLoop();
      window.addEventListener('resize', handleResize);
    });
    onUnmounted(() => {
      stopCamera();
      if (poseManager) {
        poseManager.dispose();
      }
      if (animationFrameId) {
        cancelAnimationFrame(animationFrameId);
      }
      window.removeEventListener('resize', handleResize);
    });
    const handleResize = () => {
      if (renderer && camera && characterCanvas.value) {
        camera.aspect = characterCanvas.value.clientWidth / characterCanvas.value.clientHeight;
        camera.updateProjectionMatrix();
        renderer.setSize(characterCanvas.value.clientWidth, characterCanvas.value.clientHeight);
      }
    };
    return {
      // 模板引用
      videoElement,
      poseCanvas,
      characterCanvas,
      // 状态数据
      isCameraActive,
      isLoading,
      loadingProgress,
      selectedModel,
      modelComplexity,
      minPoseConfidence,
      minKeypointConfidence,
      characterType,
      boneThickness,
      jointSize,
      smoothingFactor,
      showCharacter,
      enableRecording,
      isRecording,
      enablePlayback,
      isPlaying,
      enable3DReconstruction,
      enableSkeletonSmoothing,
      showKeypoints,
      showSkeleton,
      mirrorMode,
      showKeypointPanel,
      detectionFPS,
      renderFPS,
      detectionTime,
      keypointCount,
      trackingQuality,
      currentKeypoints,
      // 计算属性
      trackingQualityClass,
      // 方法
      toggleCamera,
      onVideoReady,
      toggleMirror,
      takeSnapshot,
      toggleCharacter,
      resetPose,
      calibratePose,
      loadModel,
      loadCharacter,
      startRecording,
      stopRecording,
      saveRecording,
      loadRecording,
      playRecording,
      pauseRecording
    };
  }
};
</script>

高级特性

3D姿态重建算法

// 3D姿态重建器
class Pose3DReconstructor {
constructor() {
this.cameraParams = {
focalLength: 1000,
principalPoint: { x: 0, y: 0 }
};
this.skeletonModel = this.createSkeletonModel();
}
// 从2D关键点重建3D姿态
reconstruct3DPose(keypoints2D) {
const keypoints3D = {};
// 使用骨骼长度约束和运动学先验
const rootPosition = this.estimateRootPosition(keypoints2D);
for (const [name, keypoint2D] of Object.entries(keypoints2D)) {
if (keypoint2D.score > 0.3) {
// 反投影到3D空间
const position3D = this.backprojectTo3D(keypoint2D, rootPosition);
keypoints3D[name] = {
position: position3D,
score: keypoint2D.score
};
}
}
// 应用运动学约束
this.applyKinematicConstraints(keypoints3D);
return keypoints3D;
}
estimateRootPosition(keypoints2D) {
// 使用臀部关键点作为根节点
const leftHip = keypoints2D.left_hip;
const rightHip = keypoints2D.right_hip;
if (leftHip && rightHip) {
const hipCenter2D = {
x: (leftHip.x + rightHip.x) / 2,
y: (leftHip.y + rightHip.y) / 2
};
// 假设初始深度为0
return this.backprojectTo3D(hipCenter2D, { x: 0, y: 0, z: 0 });
}
return { x: 0, y: 0, z: 0 };
}
backprojectTo3D(keypoint2D, referencePoint) {
// 简化反投影：使用针孔相机模型
const x = (keypoint2D.x - this.cameraParams.principalPoint.x) / this.cameraParams.focalLength;
const y = (keypoint2D.y - this.cameraParams.principalPoint.y) / this.cameraParams.focalLength;
// 使用参考点深度估计
const z = referencePoint.z || 0;
return { x, y, z };
}
applyKinematicConstraints(keypoints3D) {
// 应用骨骼长度约束
const boneLengths = this.estimateBoneLengths(keypoints3D);
this.enforceBoneLengths(keypoints3D, boneLengths);
// 应用关节角度限制
this.enforceJointLimits(keypoints3D);
}
estimateBoneLengths(keypoints3D) {
// 基于人体比例估计骨骼长度
const lengths = {};
const connections = [
['left_shoulder', 'left_elbow', 'upper_arm'],
['left_elbow', 'left_wrist', 'lower_arm'],
['left_hip', 'left_knee', 'upper_leg'],
['left_knee', 'left_ankle', 'lower_leg']
];
connections.forEach(([start, end, name]) => {
const startPoint = keypoints3D[start];
const endPoint = keypoints3D[end];
if (startPoint && endPoint) {
const length = this.calculateDistance(startPoint.position, endPoint.position);
lengths[name] = length;
}
});
return lengths;
}
calculateDistance(p1, p2) {
return Math.sqrt(
Math.pow(p2.x - p1.x, 2) +
Math.pow(p2.y - p1.y, 2) +
Math.pow(p2.z - p1.z, 2)
);
}
enforceBoneLengths(keypoints3D, targetLengths) {
// 迭代调整骨骼长度
for (const [boneName, targetLength] of Object.entries(targetLengths)) {
const [start, end] = this.getBoneJoints(boneName);
const startPoint = keypoints3D[start];
const endPoint = keypoints3D[end];
if (startPoint && endPoint) {
const currentLength = this.calculateDistance(startPoint.position, endPoint.position);
const scale = targetLength / currentLength;
if (Math.abs(scale - 1) > 0.1) {
// 调整骨骼长度
const direction = {
x: endPoint.position.x - startPoint.position.x,
y: endPoint.position.y - startPoint.position.y,
z: endPoint.position.z - startPoint.position.z
};
endPoint.position.x = startPoint.position.x + direction.x * scale;
endPoint.position.y = startPoint.position.y + direction.y * scale;
endPoint.position.z = startPoint.position.z + direction.z * scale;
}
}
}
}
}

运动重定向系统

// 运动重定向器
class MotionRetargeter {
constructor(sourceSkeleton, targetSkeleton) {
this.sourceSkeleton = sourceSkeleton;
this.targetSkeleton = targetSkeleton;
this.boneMapping = this.createBoneMapping();
}
createBoneMapping() {
// 定义源骨架和目标骨架之间的骨骼映射
return {
'spine': 'spine',
'left_upper_arm': 'left_upper_arm',
'left_lower_arm': 'left_lower_arm',
'right_upper_arm': 'right_upper_arm',
'right_lower_arm': 'right_lower_arm',
'left_upper_leg': 'left_upper_leg',
'left_lower_leg': 'left_lower_leg',
'right_upper_leg': 'right_upper_leg',
'right_lower_leg': 'right_lower_leg'
};
}
retargetMotion(sourcePose) {
const targetPose = {};
for (const [sourceBone, targetBone] of Object.entries(this.boneMapping)) {
const sourceRotation = sourcePose[sourceBone];
if (sourceRotation) {
// 应用旋转到目标骨骼
targetPose[targetBone] = this.adjustRotationForSkeleton(
sourceRotation,
sourceBone,
targetBone
);
}
}
return targetPose;
}
adjustRotationForSkeleton(rotation, sourceBone, targetBone) {
// 根据骨架差异调整旋转
const adjustment = this.calculateBoneAdjustment(sourceBone, targetBone);
// 应用调整
const adjustedRotation = new THREE.Quaternion();
adjustedRotation.multiplyQuaternions(rotation, adjustment);
return adjustedRotation;
}
calculateBoneAdjustment(sourceBone, targetBone) {
// 计算源骨骼和目标骨骼之间的方向差异
// 简化实现
return new THREE.Quaternion();
}
}

本节展示了如何将TensorFlow.js的机器学习能力与Three.js的3D渲染能力结合，实现实时的姿态估计和角色动画驱动。这种技术为虚拟试衣、运动分析、游戏交互等应用提供了强大的基础。