How to crop white patches in image and make passport size photo using OpenCV

I have images that need to be cropped to perfect passport size photos. I have thousands of images that need to be cropped and straightened automatically like this. If the image is too blur and not able to crop I need it to be copied to the rejected folder. I tried to do using haar cascade but this approach is giving me only face. But I need a face with a photo-cropped background. Can anyone tell me how I can code this in OpenCV or any?

            gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
            faceCascade = cv2.CascadeClassifier(
                cv2.data.haarcascades + "haarcascade_frontalface_default.xml")
            faces = faceCascade.detectMultiScale(
                gray,
                scaleFactor=1.3,
                minNeighbors=3,
                minSize=(30, 30)
            )
            if(len(faces) == 1):
                for (x, y, w, h) in faces:
                    if(x-w < 100 and y-h < 100):
                        ystart = int(y-y*int(y1)/100)
                        xstart = int(x-x*int(x1)/100)
                        yend = int(h+h*int(y1)/100)
                        xend = int(w+w*int(y2)/100)
                        roi_color = img[ystart:y + yend, xstart:x + xend]
                        cv2.imwrite(path, roi_color)

                    else:
                        rejectedCount += 1
                        cv2.imwrite(path, img)

Before

After

Answer

If all photos have that thin white-black border around them, you can just

0. threshold the pictures
1. get all contours and
2. select those contours that
   • have the correct gradient
   • are large enough
   • that reduce to 4 corners when passed through approxPolyDP
3. get an oriented bounding box
4. construct affine transformation
5. apply affine transformation

If those photos aren’t scans but taken with a camera from an angle (not top-down), you’ll need to use a perspective transformation calculated from the corner points themselves.

If the photos aren’t flat but warped, that’s an entirely different problem.

import numpy as np
import cv2 as cv

im = cv.imread("Zh8QV.jpg")
gray = cv.cvtColor(im, cv.COLOR_BGR2GRAY)

gray = 255 - gray # invert so findContours' implicit black border doesn't bother us

height, width = gray.shape
minarea = (height * width) * 0.20

# (th_level, thresholded) = cv.threshold(gray, thresh=128, maxval=255, type=cv.THRESH_OTSU)

# threshold relative to estimated brightness of "white"
th_level = 255 - (255 - np.median(gray)) * 0.98
(th_level, thresholded) = cv.threshold(gray, thresh=th_level, maxval=255, type=cv.THRESH_BINARY)

(contours, hierarchy) = cv.findContours(thresholded, mode=cv.RETR_LIST, method=cv.CHAIN_APPROX_SIMPLE)

# black-to-white contours have negative area...
#areas = sorted([cv.contourArea(c, oriented=True) for c in contours])

large_areas = [ c for c in contours if cv.contourArea(c, oriented=True) <= -minarea ]

quads = [
    c for c in large_areas
    if len(cv.approxPolyDP(c, epsilon=0.02 * cv.arcLength(c, True), closed=True)) == 4
]

# if there is no quad, or multiple, that's an error (for this example)
assert len(quads) == 1, quads
[quad] = quads

bbox = cv.minAreaRect(quad)
(bcenter, bsize, bangle) = bbox
bcenter = np.array(bcenter)
bsize = np.array(bsize)

# keep orientation upright, fix up bbox size
(rot90, bangle) = divmod(bangle + 45, 90)
bangle -= 45
if rot90 % 2 != 0:
    bsize = bsize[::-1]

# construct affine transformation
M1 = np.eye(3)
M1[0:2,2] = -bcenter

R = np.eye(3)
R[0:2] = cv.getRotationMatrix2D(center=(0,0), angle=bangle, scale=1.0)

M2 = np.eye(3)
M2[0:2,2] = +bsize * 0.5

M = M2 @ R @ M1

bwidth, bheight = np.ceil(bsize)
dsize = (int(bwidth), int(bheight))

output = cv.warpAffine(im, M[0:2], dsize=dsize, flags=cv.INTER_CUBIC)

cv.imshow("output", output)
cv.waitKey(-1)
cv.destroyWindow("output")