深度感知的图像校正不正确

Question

我在尝试编写代码来校正两个图像时付出了很大的努力。我希望计算深度，然后提取点云。我无法计算深度，因为我无法校正图像并计算视差图。

这些是我校准相机和校正立体图像的功能

def CamIntr(image_paths, chessboard_params, square_size):
    # Calibration chessboard parameters
    #chessboard_params = [6, 9]  # Number of corners in y, x
    chessboard_params.append(square_size)  # Square size in m
    num_corners = chessboard_params[0] * chessboard_params[1]

    # Termination criteria for corner refinement
    criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)

    # Prepare object points (world coordinates)
    objp = np.zeros((chessboard_params[0] * chessboard_params[1], 3), np.float32)
    objp[:, :2] = np.mgrid[0:chessboard_params[0], 0:chessboard_params[1]].T.reshape(-1, 2)*chessboard_params[2]

    # Arrays to store object points and image points from all images
    objpoints = []  # 3D points in real-world space
    imgpoints = []  # 2D points in image plane

    for path in image_paths:
        # Read the image
        image = cv.imread(path)
        gray = cv.cvtColor(image, cv.COLOR_BGR2GRAY)  # Convert to grayscale

        # Find chessboard corners
        ret, corners = cv.findChessboardCorners(gray, (chessboard_params[0], chessboard_params[1]), None)

        
        #print(f"Chessboard found in {path}")
        objpoints.append(objp)
        corners2 = cv.cornerSubPix(gray, corners, (11, 11), (-1, -1), criteria)
        imgpoints.append(corners2)
        # Draw and display the corners
        #cv.drawChessboardCorners(image, (chessboard_params[1], chessboard_params[0]), corners2, ret)
        #cv.imshow('img', image)
        #cv.imwrite("Chessboard_draw.jpg", image) 
        #cv.waitKey(500)
      
    
    ret, mtx, dist, rvecs, tvecs = cv.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
    
    
    return ret, mtx, dist, rvecs, tvecs, objpoints, imgpoints 



def RectifyStereoImages(img_left, img_right, chessboardL, chessboardR):
    # Load stereo calibration data (intrinsics, extrinsics, etc.)
    
    gray_left = cv.imread(img_left,0)
    gray_right = cv.imread(img_right,0)
    
    ##CALIBRATION
    
    _, mtx_left, dist_left, rvecs_left, tvecs_left, objpoints_left, imgpoints_left = CamIntr(chessboardL, [6,9], 0.025)


    _, mtx_right, dist_right, rvecs_right, tvecs_right, objpoints_right, imgpoints_right = CamIntr(chessboardR, [6,9], 0.025)

    

   ##STEREO VISION CALIBRATION
   
    _, _, _, _, _, R, T, E, F = cv.stereoCalibrate(objpoints_left, imgpoints_left, imgpoints_right,
                                                    mtx_left, dist_left, mtx_right, dist_right,
                                                    gray_left.shape[::-1], criteria=(cv.CALIB_FIX_INTRINSIC, 30, 0.001))
    
    flags = 0
    flags |= cv.CALIB_FIX_INTRINSIC
    # Here we fix the intrinsic camara matrixes so that only Rot, Trns, Emat and Fmat are calculated.
    # Hence intrinsic parameters are the same 
    
    criteria_stereo= (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)
    
    # This step is performed to transformation between the two cameras and calculate Essential and Fundamenatl matrix
    retStereo, newCameraMatrixL, distL, newCameraMatrixR, distR, rot, trans, essentialMatrix, fundamentalMatrix = cv.stereoCalibrate(objpoints_left, imgpoints_left, imgpoints_right, mtx_left, dist_left, mtx_right, dist_right, gray_left.shape[::-1], criteria_stereo, flags)
    
    ##STEREO RECTIFICATION

    rectifyScale= 1
    rectL, rectR, projMatrixL, projMatrixR, Q, roi_L, roi_R= cv.stereoRectify(newCameraMatrixL, distL, newCameraMatrixR, distR, gray_left.shape[::-1], rot, trans, rectifyScale,(0,0), alpha = 0.5)
    
    stereoMapL = cv.initUndistortRectifyMap(mtx_left, distL, rectL, projMatrixL, gray_left.shape[::-1], cv.CV_16SC2)
    stereoMapR = cv.initUndistortRectifyMap(mtx_right, distR, rectR, projMatrixR, gray_right.shape[::-1], cv.CV_16SC2)
    
    map_left_x = stereoMapL[0]
    map_left_y = stereoMapL[1]
    map_right_x = stereoMapR[0]
    map_right_y = stereoMapR[1]
    
    # Rectify the images
    img_left_rectified = cv.remap(gray_left, map_left_x, map_left_y, cv.INTER_LANCZOS4, cv.BORDER_CONSTANT, 0)
    img_right_rectified = cv.remap(gray_right, map_right_x, map_right_y, cv.INTER_LANCZOS4, cv.BORDER_CONSTANT, 0)
    
    
    return img_left_rectified, img_right_rectified

这就是我所说的。

# Get all files in the directory
all_files = os.listdir(r"C:\Users\emmay\Desktop\Folder")
# Filter files starting with the prefix "Chessboard"
chessboard1 = [file for file in all_files if file.startswith("Chessboard1")]
chessboard2 = [file for file in all_files if file.startswith("Chessboard2")]
squaresize = 0.025

img1 = "Chessboard1_1.png"
img2 = "Chessboard2_1.png"


imgL = cv.imread(img1,0)
imgR = cv.imread(img2,0)

fig, ax = plt.subplots()
im = ax.imshow(imgL)
plt.show()
ax.set_title("Left original image")

fig, ax = plt.subplots()
im = ax.imshow(imgR)
plt.show()
ax.set_title("Right original image")

L_rect, R_rect = RectifyStereoImages(img1, img2, chessboard1, chessboard2)
#images after rectification
fig, ax = plt.subplots()
im = ax.imshow(L_rect)
plt.show()
ax.set_title("Left rectified image")

fig, ax = plt.subplots()
im = ax.imshow(R_rect)
plt.show()
ax.set_title("Right rectified image")

我有来自每个相机的 10 张棋盘图像，同时拍摄。我是否单独校准每个相机，即使它们是同一台相机？我将图像分开，以防万一，但如果将它们全部集中在一起以获得相机的内在特性，我可以通过图像校正获得更好的结果。当我分离图像时，即使它们是同一台相机，我也会得到不同的相机内在参数和畸变变量。为了方便起见，我正在尝试纠正我的一张棋盘图像。

这些是我正在尝试纠正的图像：

这是我的相机矩阵和第一个相机的畸变系数

array([[553.76951265,   0.        , 359.52945636],
       [  0.        , 558.7809926 , 322.92191203],
       [  0.        ,   0.        ,   1.        ]])

array([[-5.74144340e-01,  2.21330577e+00,  2.87945868e-03,
         9.47036694e-04, -3.28003833e+00]])

用于第二个相机

([[643.44291723,   0.        , 297.31281198],
       [  0.        , 639.47052736, 216.32413232],
       [  0.        ,   0.        ,   1.        ]])

array([[ 0.16309873, -1.99119008, -0.02374205, -0.01189547, 10.39937883]])

然后当我尝试查看校正后的图像时，我看到的是：

如果我尝试找到包含所有图片的相机内在函数，并且不要像这样通过左或右相机将它们分开（强制内在函数相同）：

_, mtx_left, dist_left, rvecs_left, tvecs_left, objpoints_left, imgpoints_left = CamIntr(chessboardL + chessboardR, [6,9], 0.025)


    _, mtx_right, dist_right, rvecs_right, tvecs_right, objpoints_right, imgpoints_right = CamIntr(chessboardR, chessboardL, [6,9], 0.025)

我得到了这张稍微好一点的图片，但它们现在彼此之间差异太大，无法计算视差图

有人知道发生了什么事或者可以帮助我吗？

Answer 1

您的代码似乎是正确的，应该能够实现您想要的。问题源于方法论。一些要点：

• 一般情况下，您总是会进行单独的本征校准，因为即使是同一类型的两个相机，在镜头对准、畸变等方面也会略有差异。但是，您不一定必须在立体校准（cv）之前进行本征校准（cv.calibrateCamera） .stereoCalibrate）。如果不是所有的点都能在所有相机中看到，OpenCV 文档建议使用它，以获得最佳的鲁棒性和最大程度地利用数据。
• 您不能指望手持式校准目标能获得良好的结果。相机同步必须完美无缺才能发挥作用，即经过验证的硬件触发相机。
• 校准目标的大小理想情况下应覆盖整个图像。编码目标（即 ChArUco）可用于确保在板仅部分可见时进行检测。
• 观察应覆盖整个传感器。您还需要观察透视以正确估计 f 和 cx、cy。
• 每个相机的 5 个图像可能不足以以足够的精度估计所有参数。
• 我建议调用 OpenCV 校准函数的扩展变体，它为您提供估计参数的标准误差。在良好的校准中，与参数值相比，这些误差应该很小。
• 如果只能拍摄少量图像，建议固定尽可能多的参数。您计算的失真参数值显然是错误的（太大）。对于您的镜头，仅使用 k1 可能就足够了。在这些情况下，主点坐标也应固定为图像中心。 这里有更多技巧和最佳实践（我是这些技巧和最佳实践的作者）：https://calib.io/blogs/knowledge-base/calibration-best-practices 可以在这里找到用于生成 ChArUco pdf 图案的便捷工具：https://calib.io/pages/camera-calibration-pattern-generator