在 OpenCV 中裁剪全景图像

Question

我正在尝试找到一种简单的算法来裁剪（删除黑色区域）使用 openCV Stitcher 模块创建的全景图像。

我的想法是计算图像中最内部的黑点，这将定义裁剪区域，如下图所示：

预期裁剪结果：

我尝试了接下来的两种方法，但它们没有按预期裁剪图像：

第一种方法：

void testCropA(cv::Mat& image)
{
    cv::Mat gray;
    cvtColor(image, gray, CV_BGR2GRAY);

    Size size = gray.size();
    int type = gray.type();
    int left = 0, top = 0, right = size.width, bottom = size.height;

    cv::Mat row_zeros = Mat::zeros(1, right, type);
    cv::Mat col_zeros = Mat::zeros(bottom, 1, type);

    while (countNonZero(gray.row(top) != row_zeros) == 0) { top++; }

    while (countNonZero(gray.col(left) != col_zeros) == 0) { left++; }

    while (countNonZero(gray.row(bottom-1) != row_zeros) == 0) { bottom--; }

    while (countNonZero(gray.col(right-1) != col_zeros) == 0) { right--;  }

    cv::Rect cropRect(left, top, right - left, bottom - top);
    image = image(cropRect);
}

第二种方法：

void testCropB(cv::Mat& image)
{
    cv::Mat gray;
    cvtColor(image, gray, CV_BGR2GRAY);

    int minCol = gray.cols;
    int minRow = gray.rows;
    int maxCol = 0;
    int maxRow = 0;

    for (int i = 0; i < gray.rows - 3; i++)
    {
        for (int j = 0; j < gray.cols; j++)
        {
            if (gray.at<char>(i, j) != 0)
            {
                if (i < minRow) {minRow = i;}
                if (j < minCol) {minCol = j;}
                if (i > maxRow) {maxRow = i;}
                if (j > maxCol) {maxCol = j;}
            }
        }
    }

    cv::Rect cropRect = Rect(minCol, minRow, maxCol - minCol, maxRow - minRow);
    image = image(cropRect);
}

Answer 1

这是我目前的解决方案。希望对其他人有帮助：

bool checkInteriorExterior(const cv::Mat &mask, const cv::Rect &croppingMask,
                                 int &top, int &bottom, int &left, int &right)
{
    // Return true if the rectangle is fine as it is
    bool result = true;

    cv::Mat sub = mask(croppingMask);
    int x = 0;
    int y = 0;

    // Count how many exterior pixels are, and choose that side for
    // reduction where mose exterior pixels occurred (that's the heuristic)

    int top_row = 0;
    int bottom_row = 0;
    int left_column = 0;
    int right_column = 0;

    for (y = 0, x = 0; x < sub.cols; ++x)
    {
        // If there is an exterior part in the interior we have
        // to move the top side of the rect a bit to the bottom
        if (sub.at<char>(y, x) == 0)
        {
            result = false;
            ++top_row;
        }
    }

    for (y = (sub.rows - 1), x = 0; x < sub.cols; ++x)
    {
        // If there is an exterior part in the interior we have
        // to move the bottom side of the rect a bit to the top
        if (sub.at<char>(y, x) == 0)
        {
            result = false;
            ++bottom_row;
        }
    }

    for (y = 0, x = 0; y < sub.rows; ++y)
    {
        // If there is an exterior part in the interior
        if (sub.at<char>(y, x) == 0)
        {
            result = false;
            ++left_column;
        }
    }

    for (x = (sub.cols - 1), y = 0; y < sub.rows; ++y)
    {
        // If there is an exterior part in the interior
        if (sub.at<char>(y, x) == 0)
        {
            result = false;
            ++right_column;
        }
    }

    // The idea is to set `top = 1` if it's better to reduce
    // the rect at the top than anywhere else.
    if (top_row > bottom_row)
    {
        if (top_row > left_column)
        {
            if (top_row > right_column)
            {
                top = 1;
            }
        }
    }
    else if (bottom_row > left_column)
    {
        if (bottom_row > right_column)
        {
            bottom = 1;
        }
    }

    if (left_column >= right_column)
    {
        if (left_column >= bottom_row)
        {
            if (left_column >= top_row)
            {
                left = 1;
            }
        }
    }
    else if (right_column >= top_row)
    {
        if (right_column >= bottom_row)
        {
            right = 1;
        }
    }

    return result;
}

bool compareX(cv::Point a, cv::Point b)
{
    return a.x < b.x;
}

bool compareY(cv::Point a, cv::Point b)
{
    return a.y < b.y;
}

void crop(cv::Mat &source)
{
    cv::Mat gray;
    source.convertTo(source, CV_8U);
    cvtColor(source, gray, cv::COLOR_RGB2GRAY);

    // Extract all the black background (and some interior parts maybe)

    cv::Mat mask = gray > 0;

    // now extract the outer contour
    std::vector<std::vector<cv::Point> > contours;
    std::vector<cv::Vec4i> hierarchy;

    cv::findContours(mask, contours, hierarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE, cv::Point(0, 0));
    cv::Mat contourImage = cv::Mat::zeros(source.size(), CV_8UC3);;

    // Find contour with max elements

    int maxSize = 0;
    int id = 0;

    for (int i = 0; i < contours.size(); ++i)
    {
        if (contours.at((unsigned long)i).size() > maxSize)
        {
            maxSize = (int)contours.at((unsigned long)i).size();
            id = i;
        }
    }

    // Draw filled contour to obtain a mask with interior parts

    cv::Mat contourMask = cv::Mat::zeros(source.size(), CV_8UC1);
    drawContours(contourMask, contours, id, cv::Scalar(255), -1, 8, hierarchy, 0, cv::Point());

    // Sort contour in x/y directions to easily find min/max and next

    std::vector<cv::Point> cSortedX = contours.at((unsigned long)id);
    std::sort(cSortedX.begin(), cSortedX.end(), compareX);
    std::vector<cv::Point> cSortedY = contours.at((unsigned long)id);
    std::sort(cSortedY.begin(), cSortedY.end(), compareY);

    int minXId = 0;
    int maxXId = (int)(cSortedX.size() - 1);
    int minYId = 0;
    int maxYId = (int)(cSortedY.size() - 1);

    cv::Rect croppingMask;

    while ((minXId < maxXId) && (minYId < maxYId))
    {
        cv::Point min(cSortedX[minXId].x, cSortedY[minYId].y);
        cv::Point max(cSortedX[maxXId].x, cSortedY[maxYId].y);
        croppingMask = cv::Rect(min.x, min.y, max.x - min.x, max.y - min.y);

        // Out-codes: if one of them is set, the rectangle size has to be reduced at that border

        int ocTop = 0;
        int ocBottom = 0;
        int ocLeft = 0;
        int ocRight = 0;

        bool finished = checkInteriorExterior(contourMask, croppingMask, ocTop, ocBottom, ocLeft, ocRight);

        if (finished == true)
        {
            break;
        }

        // Reduce rectangle at border if necessary

        if (ocLeft)
        { ++minXId; }
        if (ocRight)
        { --maxXId; }
        if (ocTop)
        { ++minYId; }
        if (ocBottom)
        { --maxYId; }
    }

    // Crop image with created mask

    source = source(croppingMask);
}

Answer 2

这里是裁剪全景图像的黑色或白色背景区域的Python代码：

import cv2 as cv
import numpy as np
import imutils
import glob
from numba import jit


def crop_stitched_image (stitched, canvas_color = 'black'):

    # Print cropping
    print("[INFO] cropping...")
    
    # Initilize the variables
    w = stitched.shape[1]
    h = stitched.shape[0]
    
    # Convert the stitched image to grayscale and threshold it
    # such that all pixels greater than zero are set to 255
    # (foreground) while all others remain 0 (background)
    gray = cv.cvtColor(stitched, cv.COLOR_BGR2GRAY)
    if canvas_color == 'black':
        thresh = cv.threshold(gray, 5, 255, cv.THRESH_BINARY)[1]
    else:
        thresh = cv.threshold(gray, 254, 255, cv.THRESH_BINARY_INV)[1]   
    
    # Find all external contours in the threshold image then find
    # the *largest* contour which will be the contour/outline of
    # the stitched image
    cnts = cv.findContours(thresh.copy(), cv.RETR_EXTERNAL, cv.CHAIN_APPROX_NONE)
    cnts = imutils.grab_contours(cnts)
    c = max(cnts, key=cv.contourArea)
    
    # Mask or stencil
    stencil_inner = np.zeros(stitched.shape, dtype=np.uint8)
    cv.fillPoly(stencil_inner, pts =[c], color=(255,255,255)) 
    stencil_outer = ~stencil_inner
    
    # Canvas outer indices
    canvas_outer_indices = np.where(stencil_outer == [255])

    # Normalize the image to -1 and others
    stitched = np.asarray(stitched, dtype=np.float32)    
    stitched[canvas_outer_indices] = -255
    stitched = stitched / 255.0


    @jit(nopython=True)
    def bulkRun():
        maxarea = 0
        height = np.zeros((w)).astype(np.int32) 
        left = np.zeros((w)).astype(np.int32) 
        right= np.zeros((w)).astype(np.int32) 
                
        ll = 0
        rr = 0 
        hh = 0 
        nl = 0
    
        for line in range(h):
            for k in range(w):
                p = stitched[line][k]
                m = max(max(p[0], p[1]), p[2])
                height[k] =  0 if m < 0 else height[k] + 1  #find Color::NO
                
    
            for k in range(w):
                left[k] = k;            
                while ((left[k] > 0) and (height[k] <= height[left[k] - 1])):
                    left[k] = left[left[k] - 1]
                    
            for k in range(w - 1, -1, -1):
                right[k] = k
                while ((right[k] < w - 1) and (height[k] <= height[right[k] + 1])):
                    right[k] = right[right[k] + 1]
                    
            for k in range(w):
                val = (right[k] - left[k] + 1) * height[k]
                if(maxarea < val):
                    maxarea = val
                    ll = left[k] 
                    rr = right[k]
                    hh = height[k] 
                    nl = line
        
        return ll, rr, hh, nl
    
    ll, rr, hh, nl = bulkRun()
    
    cropH = hh + 1
    cropW = rr - ll + 1
    offsetx = ll
    offsety = nl - hh + 1
    
    stitched *= 255

    return stitched[offsety : offsety + cropH, offsetx : offsetx + cropW].astype(np.uint8)

以下是代码用法：

def main():

    # Read stitched image
    image = cv.imread('input_image.jpg')
        
    # write the output stitched and cropped image to disk
    # stitched_cropped = crop_stitched_image (image, 'white')
    stitched_cropped = crop_stitched_image (image, 'black')
    cv.imwrite('cropped.jpg', stitched_cropped)
        
    print("Done")

if __name__ == '__main__':
    main()
    cv.destroyAllWindows()

我还分别在GitHub和MATLAB File Exchange中上传了Python和MATLAB版本。

Answer 3

我从未使用过缝合器类，但我认为您可能会获得每对图像的估计单应性矩阵，如果您可以轻松获得它，那么您可以将其与第一个原始图像的角点相乘，对于角点也是如此最后一个原始图像，您将获得它们的缝合坐标，然后获得每个图像的左右 x 坐标的最小值以及上下 y 坐标的最小值。您可以获得每个拼接图像的坐标，在某些裁剪情况下您需要做什么。