如何将 Three.js 与 MediaPipe 结合使用来制作 AR Web 应用程序

Question

我和我的朋友正在尝试为大学研讨会制作一个 AR Web 应用程序。我们想要跟踪手并将 3D 对象戴在手腕上。我们有两个问题：

• 如何使用 Threejs 上的网络摄像头作为场景？
• 我们如何获取手腕地标并在上面放一些东西？

谢谢您的建议

我们以网络摄像头流视频作为输入创建了手部检测，然后我们为 Threejs 创建了一个场景和一个摄像头，但它们是分开工作的。

更新：我们有这段代码，可以运行，但不能运行 Three.js：

import {
  HandLandmarker,
  FilesetResolver
} from "https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]";
import * as THREE from 'https://cdn.skypack.dev/[email protected]';

const video = document.getElementById("webcam");
const canvasElement = document.getElementById("output_canvas");
const canvasCtx = canvasElement.getContext("2d");
const cubeContainer = document.getElementById("cube-container");

let results = {};

let handLandmarker;
let webcamRunning = false;

const enableWebcamButton = document.getElementById("webcamButton");
enableWebcamButton.addEventListener("click", toggleWebcam);

const initializeHandLandmarker = async () => {
  const vision = await FilesetResolver.forVisionTasks(
    "https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm"
  );
  handLandmarker = await HandLandmarker.createFromOptions(vision, {
    baseOptions: {
      modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`,
      delegate: "GPU"
    },
    runningMode: "LIVE",
    numHands: 1
  });
};
initializeHandLandmarker();

async function toggleWebcam() {
  if (!handLandmarker) {
    console.log("Please wait for HandLandmarker to load.");
    return;
  }

  webcamRunning = !webcamRunning;

  if (webcamRunning) {
    enableWebcamButton.innerText = "DISABLE PREDICTIONS";
    await startWebcam();
    initializeThreeJs();
    predictWebcam();
  } else {
    enableWebcamButton.innerText = "ENABLE PREDICTIONS";
    stopWebcam();
  }
}

async function startWebcam() {
  const stream = await navigator.mediaDevices.getUserMedia({ video: true });
  video.srcObject = stream;
  await video.play();

  const { videoWidth, videoHeight } = video;
  canvasElement.width = videoWidth;
  canvasElement.height = videoHeight;

  //Imposta dimensioni del div cube-container in base alle dimensioni del video
  cubeContainer.style.width = video.videoWidth + 'px';
  cubeContainer.style.height = video.videoHeight + 'px';
}

function stopWebcam() {
  const stream = video.srcObject;
  const tracks = stream.getTracks();
  tracks.forEach(track => track.stop());
  video.srcObject = null;
}

async function predictWebcam() {
  if (!webcamRunning) return;

  if (!handLandmarker) {
    console.log("Please wait for HandLandmarker to load.");
    return;
  }

  const startTimeMs = performance.now();
  results = await handLandmarker.detectForVideo(video, startTimeMs);

  canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
  if (results.landmarks) {
    for (const landmarks of results.landmarks) {
      drawConnectors(canvasCtx, landmarks, HAND_CONNECTIONS, {
        color: "#00FF00",
        lineWidth: 5
      });
      drawLandmarks(
        canvasCtx,
        landmarks,
        { color: "#FF0000", lineWidth: 2 },
        video.offsetWidth,
        video.offsetHeight
      );

      updateCubePosition(landmarks);
    }
  }

  requestAnimationFrame(predictWebcam);
}

//________Three.js_________

function initializeThreeJs() {
  //Crea scena
  const scene = new THREE.Scene();

  //Crea camera prospettica
  const camera = new THREE.PerspectiveCamera(75, video.videoWidth / video.videoHeight, 0.1, 1000);
  camera.position.z = 2;

  //Crea un renderer Three.js
  const renderer = new THREE.WebGLRenderer({alpha: true});
  renderer.setSize(video.videoWidth, video.videoHeight);
  renderer.domElement.style.position = 'absolute';
  renderer.domElement.style.top = '0';
  renderer.domElement.style.left = '0';
  cubeContainer.appendChild(renderer.domElement);

  //Crea il cubo
  const cubeMaterial = new THREE.MeshBasicMaterial({ color: 0x00ff00 });
  const cubeSize = 0.1;
  const cubeGeometry = new THREE.BoxGeometry(cubeSize, cubeSize, cubeSize);
  const cube = new THREE.Mesh(cubeMaterial, cubeGeometry);
  scene.add(cube);

  // Log posizone sfera
  console.log('Cube position X:', cube.position.x);
  console.log('Cube position Y:', cube.position.y);

  // Funzione per posizionare il cubo sul polso
  function positionCubeOnWrist(wristPosition) {
    // Trasforma le coordinate 2D del polso in coordinate 3D della scena
    const normalizedX = (wristPosition[0] / video.videoWidth) * 2 - 1;
    const normalizedY = -((wristPosition[1] / video.videoHeight) * 2 - 1);

    // Aggiorna la posizione del cubo
    cube.position.x = normalizedX;
    cube.position.y = normalizedY;
  }

  // Aggiorna la posizione del cubo ad ogni frame
  function updateCubePosition(landmarks) {
    if (landmarks && Array.isArray(landmarks) && landmarks.length > 0) {
      const wristPosition = [
        landmarks[0].landmark[0].x * video.videoWidth,
        landmarks[0].landmark[0].y * video.videoHeight
        ];
        positionCubeOnWrist(wristPosition);

    }
  }

  // Funzione per il rendering della scena
  function render() {
    updateCubePosition();
    renderer.render(scene, camera);
    requestAnimationFrame(render);
  }

  render();

}

控制台错误如下：

three.js:2744 Uncaught (in promise) TypeError: Cannot read properties of undefined (reading 'center')
    at Sphere.copy (three.js:2744:29)
    at Frustum.intersectsObject (three.js:7224:15)
    at projectObject (three.js:15269:47)
    at projectObject (three.js:15292:7)
    at WebGLRenderer.render (three.js:15181:5)
    at render (script.js:163:14)
    at initializeThreeJs (script.js:167:3)
    at HTMLButtonElement.toggleWebcam (script.js:46:5)

Answer 1

首先确保您拥有干净的

x,y,z

坐标，没有来自 MediaPipe 的任何其他剩余信息。例如：

points = [
   {x:10, y:10, z:10},
   {x:16, y:16, z:20},
   {x:22, y:20, z:23}
]

现在您可以随意使用此

x,y,z

信息。要将对象放置在这些点上：

let geometry = new THREE.BoxGeometry( 1, 1, 1 )
let material = new THREE.MeshNormalMaterial()
let cube = new THREE.Mesh( geometry, material )
scene.add( cube )

cube.position.set(points[0].x, points[0].y, points[0].z)

Answer 2

我和一群朋友也在尝试将 AR 集成到一个与您的大学项目类似的网络应用程序中，该项目基于将对象放置在用户的手腕上。 如果可能的话我们可以联系吗，我们在实施和推进项目方面遇到了很多问题？