OpenCV transform image shape transformation into a given contour

Does anyone know whether it’s possible to transform image A into image B contour if their shapes are random, using OpenCV or any other python libraries that work with images?

Here is what I have so far with 2 images:

I’ve been able to find draw contours of the bulb and insert a fox in it using bitwise_and method, but what it does is it crops the second image, whereas I need it to transform its shape into the bulb contour.

import cv2


src1 = cv2.imread('fox.png')
src2 = cv2.imread('bulb-contour-filled.png')

src2 = cv2.resize(src2, src1.shape[1::-1])

dst = cv2.bitwise_and(src1, src2)

cv2.imwrite('img_fin.jpg', dst)

Bulb original image:

Fox original image:

Bulb contour:

Answer

The Concept

For this we are going to need to slice the into triangles, and warp each triangle individually. The starting points of the image should be along the outline of the original image, and the ending points should be along the outline of the destination shape.

Although below I have hard-coded the 2 sets of points, you’ll just need to figure out the optimal processing to retrieve the contours of the 2 images (each needs have the same number of points and in the same order). Also, I have programmed an interactive OpenCV program that will allow us to easily retrieve the coordinates.

The Code

import cv2
import numpy as np

def triangles(points):
    points = np.where(points, points, 1)
    subdiv = cv2.Subdiv2D((*points.min(0), *points.max(0)))
    for pt in points:
        subdiv.insert(tuple(map(int, pt)))
    for pts in subdiv.getTriangleList().reshape(-1, 3, 2):
        yield [np.where(np.all(points == pt, 1))[0][0] for pt in pts]

def crop(img, pts):
    x, y, w, h = cv2.boundingRect(pts)
    img_cropped = img[y: y + h, x: x + w]
    pts[:, 0] -= x
    pts[:, 1] -= y
    return img_cropped, pts

def warp(img1, img2, pts1, pts2):
    img2 = img2.copy()
    for indices in triangles(pts1):
        img1_cropped, triangle1 = crop(img1, pts1[indices])
        img2_cropped, triangle2 = crop(img2, pts2[indices])
        transform = cv2.getAffineTransform(np.float32(triangle1), np.float32(triangle2))
        img2_warped = cv2.warpAffine(img1_cropped, transform, img2_cropped.shape[:2][::-1], None, cv2.INTER_LINEAR, cv2.BORDER_REFLECT_101)
        mask = np.zeros_like(img2_cropped)
        cv2.fillConvexPoly(mask, np.int32(triangle2), (1, 1, 1), 16, 0)
        img2_cropped *= 1 - mask
        img2_cropped += img2_warped * mask
    return img2

def resize(img, size):
    h, w = img.shape[:2]
    return cv2.resize(img, (int(w * size), int(h * size)))

img1 = resize(cv2.imread("dog.png"), 0.8)
img2 = resize(cv2.imread("bulb.png"), 0.8)

pts1 = np.array([[322, 508], [390, 475], [413, 425], [440, 367], [453, 305], [458, 289], [446, 202], [434, 139], [392, 104], [324, 94], [246, 97], [194, 101], [111, 127], [98, 185], [88, 240], [95, 306], [90, 363], [123, 431], [160, 487], [223, 508]])
pts2 = np.array([[459, 793], [513, 715], [541, 580], [552, 470], [583, 398], [633, 323], [643, 233], [616, 144], [557, 71], [470, 28], [354, 27], [264, 72], [206, 138], [179, 225], [178, 302], [236, 401], [266, 480], [278, 564], [297, 707], [357, 792]])

cv2.imshow("result", warp(img1, img2, pts1, pts2))
cv2.waitKey(0)
cv2.destroyAllWindows()

The Output

The Explanation

1. Import the necessary libraries:

import cv2
import numpy as np

2. Define a function, triangles, that will take in an array of coordinates, points, and yield lists of 3 indices of the array for triangles that will cover the area of the original array of coordinates:

def triangles(points):
    points = np.where(points, points, 1)
    subdiv = cv2.Subdiv2D((*points.min(0), *points.max(0)))
    for pt in points:
        subdiv.insert(tuple(map(int, pt)))
    for pts in subdiv.getTriangleList().reshape(-1, 3, 2):
        yield [np.where(np.all(points == pt, 1))[0][0] for pt in pts]

3. Define a function, crop, that will take in an image array, img, and an array of three coordinates, pts. It will return a rectangular segment of the image just large enough to fit the triangle formed by the three point, and return the array of three coordinates transferred to the top-left corner of image:

def crop(img, pts):
    x, y, w, h = cv2.boundingRect(pts)
    img_cropped = img[y: y + h, x: x + w]
    pts[:, 0] -= x
    pts[:, 1] -= y
    return img_cropped, pts

4. Define a function, warp, that will take in 2 image arrays, img1 and img2, and 2 arrays of coordinates, pts1 and pts2. It will utilize the triangles function defined before iterate through the triangles from the first array of coordinates, the crop function defined before to crop both images at coordinates corresponding to the triangle indices and use the cv2.warpAffine() method to warp the image at the current triangle of the iterations:

def warp(img1, img2, pts1, pts2):
    img2 = img2.copy()
    for indices in triangles(pts1):
        img1_cropped, triangle1 = crop(img1, pts1[indices])
        img2_cropped, triangle2 = crop(img2, pts2[indices])
        transform = cv2.getAffineTransform(np.float32(triangle1), np.float32(triangle2))
        img2_warped = cv2.warpAffine(img1_cropped, transform, img2_cropped.shape[:2][::-1], None, cv2.INTER_LINEAR, cv2.BORDER_REFLECT_101)
        mask = np.zeros_like(img2_cropped)
        cv2.fillConvexPoly(mask, np.int32(triangle2), (1, 1, 1), 16, 0)
        img2_cropped *= 1 - mask
        img2_cropped += img2_warped * mask
    return img2

5. Read in your images. Note that I have resized the images to better fit my screen. If you remove the resizing part, you’ll need to use the program below to re-adjust the points and get the corrected sets of points:

def resize(img, size):
    h, w = img.shape[:2]
    return cv2.resize(img, (int(w * size), int(h * size)))

img1 = resize(cv2.imread("dog.png"), 0.8)
img2 = resize(cv2.imread("bulb.png"), 0.8)

6. Finally, define the 2 sets of points; the first set outlining the first image, and the second one outlining the second. Use the warp function defined before to warp img1 to have its keypoints overlap with the kewpoints of img2 and show the resulting image:

pts1 = np.array([[0, 0], [286, 0], [286, 198], [174, 198], [158, 116], [0, 97]])
pts2 = np.array([[80, 37], [409, 42], [416, 390], [331, 384], [291, 119], [111, 311]])

cv2.imshow("result", warp(img1, img2, pts1, pts2))
cv2.waitKey(0)
cv2.destroyAllWindows()

Tools

Use the program below to manually drag the point onto each image, and see the warp effect in real-time. Of course, rather than manually doing this, you can detect the contours of the two images (make sure that they have the same number of points and are in the same order):

import cv2
import numpy as np

def triangles(points):
    points = np.where(points, points, 1)
    subdiv = cv2.Subdiv2D((*points.min(0), *points.max(0)))
    for pt in points:
        subdiv.insert(tuple(map(int, pt)))
    for pts in subdiv.getTriangleList().reshape(-1, 3, 2):
        yield [np.where(np.all(points == pt, 1))[0][0] for pt in pts]

def crop(img, pts):
    x, y, w, h = cv2.boundingRect(pts)
    img_cropped = img[y: y + h, x: x + w]
    pts[:, 0] -= x
    pts[:, 1] -= y
    return img_cropped, pts

def warp(img1, img2, pts1, pts2):
    img2 = img2.copy()
    for indices in triangles(pts1):
        img1_cropped, triangle1 = crop(img1, pts1[indices])
        img2_cropped, triangle2 = crop(img2, pts2[indices])
        transform = cv2.getAffineTransform(np.float32(triangle1), np.float32(triangle2))
        img2_warped = cv2.warpAffine(img1_cropped, transform, img2_cropped.shape[:2][::-1], None, cv2.INTER_LINEAR, cv2.BORDER_REFLECT_101)
        mask = np.zeros_like(img2_cropped)
        cv2.fillConvexPoly(mask, np.int32(triangle2), (1, 1, 1), 16, 0)
        img2_cropped *= 1 - mask
        img2_cropped += img2_warped * mask
    return img2

def draw_circle(event, x, y, flags, param):
    pts = param
    if event == cv2.EVENT_LBUTTONDOWN:
        for pt in pts:
            dist = (pt[0] - x) ** 2 + (pt[1] - y) ** 2
            if dist < 225:
                active_pt[:] = pt
    elif event == cv2.EVENT_LBUTTONUP:
        active_pt[:] = 0
    elif event == cv2.EVENT_MOUSEMOVE:
        if np.any(active_pt):
            for pt in pts:
                if np.all(active_pt == pt):
                    pt[:] = active_pt[:] = x, y

def draw_circles(img, pts):
    img = img.copy()
    for i, (x, y) in enumerate(pts):
        cv2.circle(img, (x, y), 15, (0, 0, 255), -1)
        cv2.putText(img, str(i), (x - 10, y + 10), cv2.FONT_HERSHEY_COMPLEX, 0.8, (0, 0, 0), 2)
    return img

def resize(img, size):
    h, w = img.shape[:2]
    return cv2.resize(img, (int(w * size), int(h * size)))

img1 = resize(cv2.imread("dog.png"), 0.8)
img2 = resize(cv2.imread("bulb.png"), 0.8)

pts_count = 20

pts1 = np.arange(pts_count * 2).reshape((pts_count, 2))
pts2 = np.arange(pts_count * 2).reshape((pts_count, 2))

active_pt = np.array([0, 0])

cv2.namedWindow("image 1")
cv2.setMouseCallback('image 1', draw_circle, pts1)
cv2.namedWindow("image 2")
cv2.setMouseCallback('image 2', draw_circle, pts2)

pause = False
while True:
    cv2.imshow('image 1', draw_circles(img1, pts1))
    cv2.imshow('image 2', draw_circles(img2, pts2))
    if not pause:
        try:
            cv2.imshow("result", warp(img1, img2, pts1, pts2))
        except:
            pass
    key = cv2.waitKey(20)
    if key & 0xFF == ord("q"):
        break
    if key & 0xFF == ord("p"):
        pause = not pause
    
cv2.waitKey(0)
cv2.destroyAllWindows()

Here is a rough demonstration of how it works (sped up x4):

If the real-time warping is too slow on your computer, simply press the p key to pause the warp updating, and press it again to resume it.