3D RGB-D 全景转 3D 网格

Question

我正在将全景图像投影回 3D，但是我在投影方面遇到了困难。

import argparse
import os
import cv2
import numpy as np
import open3d

class PointCloudReader():

    def __init__(self, resolution="full", random_level=0, generate_color=False, generate_normal=False):
        self.random_level = random_level
        self.resolution = resolution
        self.generate_color = generate_color
        self.point_cloud = self.generate_point_cloud(self.random_level, color=self.generate_color)

    def generate_point_cloud(self, random_level=0, color=False, normal=False):
        coords = []
        colors = []

        # Load and resize depth image
        depth_image_path = 'DPT/output_monodepth/basel_stapfelberg_panorama.png'
        depth_img = cv2.imread(depth_image_path, cv2.IMREAD_ANYDEPTH)
        depth_img = cv2.resize(depth_img, (depth_img.shape[1] // 2, depth_img.shape[0] // 2))

        # Load and resize RGB image
        equirectangular_image = 'DPT/input/basel_stapfelberg_panorama.png'
        rgb_img = cv2.imread(equirectangular_image)
        rgb_img = cv2.resize(rgb_img, (depth_img.shape[1], depth_img.shape[0]))

        # Define parameters for conversion
        focal_length = depth_img.shape[1] / 2
        sensor_width = 36
        sensor_height = 24
        y_ticks = np.deg2rad(np.arange(0, 360, 360 / depth_img.shape[1]))
        x_ticks = np.deg2rad(np.arange(-90, 90, 180 / depth_img.shape[0]))

        # Compute spherical coordinates
        theta, phi = np.meshgrid(y_ticks, x_ticks)
        depth = depth_img + np.random.random(depth_img.shape) * random_level
        x_sphere = depth * np.cos(phi) * np.sin(theta)
        y_sphere = depth * np.sin(phi)
        z_sphere = depth * np.cos(phi) * np.cos(theta)

        # Convert spherical coordinates to camera coordinates
        x_cam = x_sphere.flatten()
        y_cam = -y_sphere.flatten()
        z_cam = z_sphere.flatten()

        coords = np.stack((x_cam, y_cam, z_cam), axis=-1)

        if color:
            colors = rgb_img.reshape(-1, 3) / 255.0

        points = {'coords': coords}

        if color:
            points['colors'] = colors

        return points

    def visualize(self):
        pcd = open3d.geometry.PointCloud()
        pcd.points = open3d.utility.Vector3dVector(self.point_cloud['coords'])
        if self.generate_color:
            pcd.colors = open3d.utility.Vector3dVector(self.point_cloud['colors'])
        open3d.visualization.draw_geometries([pcd])

def main(args):
    reader = PointCloudReader(random_level=10, generate_color=True, generate_normal=False)
    reader.visualize()

if __name__ == "__main__":
    main(None)

如何从全景图中获得有效的 3D 表示？正如我们在重投影中看到的，全景的扭曲影响了我的 3D 场景。

Answer 1

我们可以使用 3DGS 来重建它吗？