Python OpenCV – Extrapolating the largest rectangle off of a set of contour points

I’m trying to make an OpenCV detect a bed in the image. I am running the usual Grayscale, Blur, Canny, and I’ve tried Convex Hull. However, since there’s quite a number of “noise” which gives extra contours and messes up the object detection. Because of this, I am unable to detect the bed properly.

Here is the input image as well as the Canny Edge Detection result:

As you can see, it’s almost there. I have the outline of the bed already, albeit, that the upper right corner has a gap – which is preventing me from detecting a closed rectangle.

Here’s the code I’m running:

import cv2
import numpy as np

def contoursConvexHull(contours):
    print("contours length = ", len(contours))
    print("contours length of first item = ", len(contours[1]))
    pts = []
    for i in range(0, len(contours)):
        for j in range(0, len(contours[i])):
            pts.append(contours[i][j])

    pts = np.array(pts)

    result = cv2.convexHull(pts)

    print(len(result))
    return result

def auto_canny(image, sigma = 0.35):
    # compute the mediam of the single channel pixel intensities
    v = np.median(image)

    # apply automatic Canny edge detection using the computed median
    lower = int(max(0, (1.0 - sigma) * v))
    upper = int(min(255, (1.0 + sigma) *v))
    edged = cv2.Canny(image, lower, upper)

    # return edged image
    return edged


# Get our image in color mode (1)
src = cv2.imread("bed_cv.jpg", 1)

# Convert the color from BGR to Gray
srcGray = cv2.cvtColor(src, cv2.COLOR_BGR2GRAY)

# Use Gaussian Blur 
srcBlur = cv2.GaussianBlur(srcGray, (3, 3), 0)

# ret is the returned value, otsu is an image
##ret, otsu = cv2.threshold(srcBlur, 0, 255,
##                          cv2.THRESH_BINARY+cv2.THRESH_OTSU)

# Use canny
##srcCanny = cv2.Canny(srcBlur, ret, ret*2, 3)
srcCanny1 = auto_canny(srcBlur, 0.70)

# im is the output image
# contours is the contour list
# I forgot what hierarchy was
im, contours, hierarchy = cv2.findContours(srcCanny1,
                                           cv2.RETR_TREE,
                                           cv2.CHAIN_APPROX_SIMPLE)

##cv2.drawContours(src, contours, -1, (0, 255, 0), 3)

ConvexHullPoints = contoursConvexHull(contours)
##cv2.polylines(src, [ConvexHullPoints], True, (0, 0, 255), 3)

cv2.imshow("Source", src)
cv2.imshow("Canny1", srcCanny1)

cv2.waitKey(0)

Since the contour of the bed isn’t closed, I can’t fit a rectangle nor detect the contour with the largest area.

The solution I can think of is to extrapolate the largest possible rectangle using the contour points in the hopes of bridging that small gap, but I’m not too sure how to proceed since the rectangle is incomplete.

Answer

Since you haven’t provided any other examples, I provide an algorithm working with this case. But bare in mind that you will have to find ways of adapting it to however the light and background changes on other samples.

Since there is a lot of noise and a relatively high dynamic range, I suggest not to use Canny and instead use Adaptive Thresholding and Find Contours on that (it doesn’t need edges as an input), that helps with choosing different threshold values for different parts of the image.

My result:

Code:

import cv2
import numpy as np

def clahe(img, clip_limit=2.0, grid_size=(8,8)):
    clahe = cv2.createCLAHE(clipLimit=clip_limit, tileGridSize=grid_size)
    return clahe.apply(img)

src = cv2.imread("bed.png")

# HSV thresholding to get rid of as much background as possible
hsv = cv2.cvtColor(src.copy(), cv2.COLOR_BGR2HSV)
lower_blue = np.array([0, 0, 120])
upper_blue = np.array([180, 38, 255])
mask = cv2.inRange(hsv, lower_blue, upper_blue)
result = cv2.bitwise_and(src, src, mask=mask)
b, g, r = cv2.split(result)
g = clahe(g, 5, (3, 3))

# Adaptive Thresholding to isolate the bed
img_blur = cv2.blur(g, (9, 9))
img_th = cv2.adaptiveThreshold(img_blur, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
                               cv2.THRESH_BINARY, 51, 2)

im, contours, hierarchy = cv2.findContours(img_th,
                                           cv2.RETR_CCOMP,
                                           cv2.CHAIN_APPROX_SIMPLE)

# Filter the rectangle by choosing only the big ones
# and choose the brightest rectangle as the bed
max_brightness = 0
canvas = src.copy()
for cnt in contours:
    rect = cv2.boundingRect(cnt)
    x, y, w, h = rect
    if w*h > 40000:
        mask = np.zeros(src.shape, np.uint8)
        mask[y:y+h, x:x+w] = src[y:y+h, x:x+w]
        brightness = np.sum(mask)
        if brightness > max_brightness:
            brightest_rectangle = rect
            max_brightness = brightness
        cv2.imshow("mask", mask)
        cv2.waitKey(0)

x, y, w, h = brightest_rectangle
cv2.rectangle(canvas, (x, y), (x+w, y+h), (0, 255, 0), 1)
cv2.imshow("canvas", canvas)
cv2.imwrite("result.jpg", canvas)
cv2.waitKey(0)