如何获取 3D 对象 RealityKit 表面上的点并用线连接这些点

Question

我目前正在开发一个项目，您可以将 usdz 文件加载到 Realitiykit 中。我对这个 API 还比较陌生。我想要的是预览 usdz 对象，并能够选择该对象的一些表面点并通过线连接这些点。为了实现这一目标，我遇到了一些问题：

1. 首先，如何在平面白色视图中加载对象（未添加到现实世界相机 AR 中）
2. 其次，如何通过手势识别来选择 3D 物体表面上的一些点。
3. 如何用线连接它们。

这是我当前的代码：

struct ContentView: View {
    var body: some View {
        ARViewContainer()
            .edgesIgnoringSafeArea(.all)
    }
}

struct ARViewContainer: UIViewRepresentable {
    func makeUIView(context: Context) -> ARView {
        let arView = ARView(frame: .zero)
        context.coordinator.arView = arView
        
        // Load the USDZ model
        let modelEntity = try! ModelEntity.loadModel(named: "yourModel.usdz")
        
        // Create an anchor to hold the model
        let anchorEntity = AnchorEntity(world: [0, 0, 0])
        anchorEntity.addChild(modelEntity)
        
        // Add the anchor to the ARView
        arView.scene.addAnchor(anchorEntity)
        
        // Configure the background color to white (simulates a plane white view)
        arView.environment.background = .color(.white)
        
        // Setup gesture recognizer
        context.coordinator.setupGestureRecognition()
        
        return arView
    }
    
    func updateUIView(_ uiView: ARView, context: Context) {}
    
    func makeCoordinator() -> Coordinator {
        return Coordinator()
    }
}

class Coordinator: NSObject {
    var arView: ARView?
    var selectedPoints: [SIMD3<Float>] = []
    
    override init() {
        super.init()
    }
    
    func setupGestureRecognition() {
        guard let arView = arView else { return }
        let tapGesture = UITapGestureRecognizer(target: self, action: #selector(handleTap(_:)))
        arView.addGestureRecognizer(tapGesture)
    }
    
    @objc private func handleTap(_ gesture: UITapGestureRecognizer) {
        guard let arView = arView else { return }
        let location = gesture.location(in: arView)
        
        // Perform a hit test to find where the user tapped on the model
        let results = arView.hitTest(location)
        
        if let firstResult = results.first {
            let position = firstResult.worldTransform.translation
            selectedPoints.append(position)
            addSphere(at: position)
            
            if selectedPoints.count > 1 {
                drawLine(from: selectedPoints[selectedPoints.count - 2], to: position)
            }
        }
    }
    
    private func addSphere(at position: SIMD3<Float>) {
        guard let arView = arView else { return }
        
        let sphere = MeshResource.generateSphere(radius: 0.01)
        let material = SimpleMaterial(color: .red, isMetallic: false)
        let sphereEntity = ModelEntity(mesh: sphere, materials: [material])
        sphereEntity.position = position
        
        let anchorEntity = AnchorEntity(world: position)
        anchorEntity.addChild(sphereEntity)
        arView.scene.addAnchor(anchorEntity)
    }
    
    private func drawLine(from start: SIMD3<Float>, to end: SIMD3<Float>) {
        guard let arView = arView else { return }
        
        let vertices: [SIMD3<Float>] = [start, end]
        let indices: [UInt32] = [0, 1]
        let lineMesh = MeshResource.generate(from: vertices, indices: indices)
        
        let material = SimpleMaterial(color: .blue, isMetallic: false)
        let lineEntity = ModelEntity(mesh: lineMesh, materials: [material])
        
        let anchorEntity = AnchorEntity(world: [0, 0, 0])
        anchorEntity.addChild(lineEntity)
        arView.scene.addAnchor(anchorEntity)
    }
}

extension matrix_float4x4 {
    /// Extracts the translation vector (position) from the 4x4 matrix
    var translation: SIMD3<Float> {
        return SIMD3<Float>(columns.3.x, columns.3.y, columns.3.z)
    }
}

我感谢您的时间！

Answer 1

1.白色背景的 NonAR 模式

要在没有 AR 功能的情况下运行 RealityKit 场景，您可以使用 .nonAR 相机模式初始化

ARView

。这允许您使用 RealityKit 作为虚拟场景：

let arView = ARView(
    frame: .zero, 
    cameraMode: .nonAR,
    automaticallyConfigureSession: false)

// Set background color
arView.environment.background = .color(.white)

---

2.碰撞形状和手势识别

为了能够选择网格表面上的点，您需要将

CollisionComponent

添加到导入的模型中。此功能的准确性取决于网格几何形状的复杂性以及将该几何形状重新创建为碰撞形状的能力。添加准确的碰撞形状对于原始形状来说很简单，但对于复杂的网格来说可能具有挑战性。

您可以使用

ShapeResource

生成原始形状或凸形状。从 iOS 18 开始，您还可以尝试使用 generateStaticMesh(from:) 生成每个面的静态碰撞形状。

这是如何从导入的网格生成凸形状并将其添加到具有新碰撞组件的模型实体的示例：

if let modelComponent = modelEntity.model {
    let convex = ShapeResource.generateConvex(from: modelComponent.mesh)
    modelEntity.collision = .init(shapes: [convex])
}

// Visualize the collision shapes: 
arView.debugOptions.insert(.showPhysics) 

您现有的手势和点击处理功能处于正确的轨道上。要获取光线投射命中位置并在代码中使用它，您可以将

results

返回为 [CollisionCastHit] 并提取第一个元素的

position

属性：

let results: [CollisionCastHit] = arView.hitTest(location)
            
// Grab the first CollisionCastHit element, if any
if let firstResult = results.first {
    let position = firstResult.position
    addSphere(at: position)
    {...}
}

不同碰撞形状和成功命中位置的演示（红色球体）：

---

第 3 部分：在点之间绘制线

在 3D 场景中绘制线条的方法有多种，具体取决于您的具体要求和所需的视觉效果。下面是一种涉及构建矩形网格的方法，该网格根据点之间的距离进行缩放并与线的方向对齐：

private func drawLine(from start: SIMD3<Float>, to end: SIMD3<Float>) {
    guard let arView = arView else { return }
 
    let midpoint = (start + end) / 2
    let direction = normalize(end - start)
    let distance = length(end - start)
    let lineWidth: Float = 0.012

    let lineMesh = MeshResource.generateBox(
        width: lineWidth, 
        height: lineWidth, 
        depth: distance, 
        cornerRadius: lineWidth / 2)

    let material = SimpleMaterial(color: .systemBlue, isMetallic: false)

    let lineEntity = ModelEntity(mesh: lineMesh, materials: [material])
 
    lineEntity.orientation = simd_quatf(
        from: SIMD3<Float>(0, 0, 1), 
        to: direction)
    lineEntity.position = midpoint
 
    let anchorEntity = AnchorEntity(world: [0, 0, 0])
    anchorEntity.addChild(lineEntity)
    arView.scene.addAnchor(anchorEntity)
 }

将所有内容放在一起，您现在应该能够在 RealityKit 中运行虚拟场景，点击以针对场景中的碰撞形状执行光线投射命中，并在点击位置之间绘制线条：