multiprocessing with moviepy

Recently I made a script that take a 5 minutes video clip and cuts for 5 video, 1 min each video, it works well, but its taking too long for pc like my, and my pc with very good part performance:

Intel(R) Core(TM) i7-10700 CPU @ 2.90GHz, 2904 Mhz, 8 Core(s), 16 Logical Processor(s)

Installed Physical Memory (RAM) 16.0 GB

So I search on the moviepy’s docs “threads”, I found something in the “write_videofile” function that i can set my threads to speed up, I tried it, but its didnt worked, I mean its worked but its only it changed maybe to more 2 or 3 it/s.

Also I found example code with multithreading but its seems like the code doesnt work because moviepy.multithreading doesnt exists in the moviepy library, Please help me speed up the rendering, Thank you

here is the code that i found:

from moviepy.multithreading import multithread_write_videofile

def concat_clips():
    files = [
        "myclip1.mp4",
        "myclip2.mp4",
        "myclip3.mp4",
        "myclip4.mp4",
    ]
    multithread_write_videofile("output.mp4", get_final_clip, {"files": files})


def get_final_clip(files):
    clips = [VideoFileClip(file) for file in files]
    final = concatenate_videoclips(clips, method="compose")
    return final

this is my code:

from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip
from moviepy.editor import *
from numpy import array, true_divide
import cv2
import time


# ffmpeg_extract_subclip("full.mp4", start_seconds, end_seconds, targetname="cut.mp4")




def duration_clip(filename):
    clip = VideoFileClip(filename)
    duration = clip.duration
    return duration


current_time = time.strftime("%Y_%m_%d_%H_%M_%S")


def main():
    global duration
    start = 0
    cut_name_num = 1
    end_seconds = start + 60
    video_duration = duration_clip("video.mp4")
    


    txt = input("Enter Your text please: ") [::-1]
    txt_part = 1

    while start < int(video_duration):
        final_text = f"{str(txt_part)} {txt}"



        try:
            try:
                os.makedirs(f"result_{str(current_time)}/result_edit")
            except FileExistsError:
                pass            
            ffmpeg_extract_subclip("video.mp4", start, end_seconds, targetname=f"result_{str(current_time)}/cut_{str(cut_name_num)}.mp4")

            clip = VideoFileClip(f"result_{str(current_time)}/cut_{str(cut_name_num)}.mp4")

            clip = clip.subclip(0, 60)

            clip = clip.volumex(2)

            txt_clip = TextClip(final_text, font="font/VarelaRound-Regular.ttf", fontsize = 50, color = 'white')

            txt_clip = txt_clip.set_pos(("center","top")).set_duration(60) 

            video = CompositeVideoClip([clip, txt_clip])
            
            clip.write_videofile(f"result_{str(current_time)}/result_edit/cut_{str(cut_name_num)}.mp4")

        except:
            try:
                os.makedirs(f"result_{str(current_time)}/result_edit")
            except FileExistsError:
                pass
            
            ffmpeg_extract_subclip("video.mp4", start, video_duration, targetname=f"result_{str(current_time)}/cut_{str(cut_name_num)}.mp4")

            clip_duration = duration_clip(f"result_{str(current_time)}/cut_{str(cut_name_num)}.mp4")

            clip = VideoFileClip(f"result_{str(current_time)}/cut_{str(cut_name_num)}.mp4")

            clip = clip.subclip(0, clip_duration)

            clip = clip.volumex(2)

            txt_clip = TextClip(final_text, font="font/VarelaRound-Regular.ttf", fontsize = 50, color = 'white')

            txt_clip = txt_clip.set_pos(("center","top")).set_duration(60) 

            video = CompositeVideoClip([clip, txt_clip])

            clip.write_videofile(f"result_{str(current_time)}/result_edit/cut_{str(cut_name_num)}.mp4")

        start += 60
        cut_name_num += 1
        end_seconds = start + 60
        txt_part += 1


if __name__ == "__main__":
    main()

Answer

Using processes I reduced time only by 15-20 seconds because ffmpeg even in single process was using almost full CPU power and my computer didn’t have power to run other processes faster.

---

First I reduced code to make it shorter.

Code in try and except had similar elements so I moved them ouside try/except.

Next I used

        if end > video_duration:
            end = video_duration

and I didn’t need try/except at all.

Using os.makedirs(..., exist_ok=True) I don’t need to run it in try/except

Meanwhile I reduced time by 20 seconds using

        clip = VideoFileClip(filename).subclip(start, end)

instead of

        temp_filename  = f"{base_folder}/cut_{number}.mp4"
        
        fmpeg_extract_subclip(filename, start, end, targetname=temp_filename)
        
        clip = VideoFileClip(temp_filename)

This way I don’t write subclip on disk and I don’t have to read it again from disk.

---

from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip
from moviepy.editor import *
import time

def main():

    text = input("Enter Your text please: ") [::-1]
    #text = 'Hello World'
    
    base_folder = time.strftime("result_%Y_%m_%d_%H_%M_%S")    
    os.makedirs(f"{base_folder}/result_edit", exist_ok=True)
    
    filename = "video.mp4"
    #filename = "BigBuckBunny.mp4"
    
    video_duration = VideoFileClip(filename).duration

    number = 0 # instead of `cut_name_num` and `txt_part` because both had the same value

    time_start = time.time()

    for start in range(0, int(video_duration), 60):

        end = start + 60
        
        if end > video_duration:
            end = video_duration

        number += 1

        clip_duration = end - start
        print(f'[DEBUG] number: {number:2} | start: {start:6.2f} | end: {end:6.2f} | duration: {clip_duration:.2f}')
       
        final_text = f"{number} {text}"
    
        temp_filename  = f"{base_folder}/cut_{number}.mp4"
        final_filename = f"{base_folder}/result_edit/cut_{number}.mp4"
        
        #ffmpeg_extract_subclip(filename, start, end, targetname=temp_filename)
        
        #clip = VideoFileClip(temp_filename)
        clip = VideoFileClip(filename).subclip(start, end)
        clip = clip.volumex(2)
    
        txt_clip = TextClip(final_text, font="font/VarelaRound-Regular.ttf", fontsize=50, color='white')
        txt_clip = txt_clip.set_pos(("center","top")).set_duration(60)
        
        video = CompositeVideoClip([clip, txt_clip])
    
        video.write_videofile(final_filename)

    # - after loop -
    
    # because I use `number += 1` before loop so now `number` has number of subclips
    print('number of subclips:', number)

    time_end = time.time()
    
    diff = time_end - time_start
    print(f'time: {diff:.2f}s ({diff//60:02.0f}:{diff%60:02.2f})')
    
    
if __name__ == "__main__":
    main()

Next I moved code to function with arguments my_process(filename, text, start, end, number, base_folder)

from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip
from moviepy.editor import *
import time

def my_process(filename, text, start, end, number, base_folder):

    clip_duration = end - start
    print(f'[DEBUG] number: {number:2} | start: {start:6.2f} | end: {end:6.2f} | duration: {clip_duration:.2f}')
    
    final_text = f"{number} {text}"

    temp_filename  = f"{base_folder}/cut_{number}.mp4"
    final_filename = f"{base_folder}/result_edit/cut_{number}.mp4"
    
    #print('[DEBUG] ffmpeg_extract_subclip')
    #ffmpeg_extract_subclip(filename, start, end, targetname=temp_filename)
    
    #print('[DEBUG] VideoClip')
    #clip = VideoFileClip(temp_filename)
    clip = VideoFileClip(filename).subclip(start, end)
    clip = clip.volumex(2)

    #print('[DEBUG] TextClip')
    txt_clip = TextClip(final_text, font="font/VarelaRound-Regular.ttf", fontsize=50, color='white')
    txt_clip = txt_clip.set_pos(("center","top")).set_duration(60)
    
    #print('[DEBUG] CompositeVideoClip')
    video = CompositeVideoClip([clip, txt_clip])

    #print('[DEBUG] CompositeVideoClip write')
    video.write_videofile(final_filename)
    #print('[DEBUG] CompositeVideoClip end')
    

def main():

    text = input("Enter Your text please: ") [::-1]
    #text = 'Hello World'
    
    base_folder = time.strftime("result_%Y_%m_%d_%H_%M_%S")    
    os.makedirs(f"{base_folder}/result_edit", exist_ok=True)
    
    filename = "video.mp4"
    #filename = "BigBuckBunny.mp4"
    
    video_duration = VideoFileClip(filename).duration

    number = 0 # instead of `cut_name_num` and `txt_part` because both had the same value

    time_start = time.time()
    
    for start in range(0, int(video_duration), 60):

        end = start + 60
        
        if end > video_duration:
            end = video_duration
        
        number += 1
        
        my_process(filename, text, start, end, number, base_folder)
        
    # - after loop -
    
    # because I use `number += 1` before loop so now `number` has number of subclips
    print('number of subclips:', number)

    time_end = time.time()
    
    diff = time_end - time_start
    print(f'time: {diff:.2f}s ({diff//60:02.0f}:{diff%60:02.2f})')


if __name__ == "__main__":
    main()

And now I can run function in separated processes using standard module multiprocessing

(or standard modules threading, concurrent.futures or external modules Joblib, Ray, etc.).

It starts single process

# it has to use named arguments`target=`, `args=`

p = multiprocessing.Process(target=my_process, args=(filename, text, start, end, number, base_folder))
p.start()  # start it

but if I use it in loop then I will start many processes at the same time.

---

from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip
from moviepy.editor import *
import time
import multiprocessing

def my_process(filename, text, start, end, number, base_folder):

    clip_duration = end - start
    print(f'[DEBUG] number: {number:2} | start: {start:6.2f} | end: {end:6.2f} | duration: {clip_duration:.2f}')
    
    final_text = f"{number} {text}"

    temp_filename  = f"{base_folder}/cut_{number}.mp4"
    final_filename = f"{base_folder}/result_edit/cut_{number}.mp4"
    
    #print('[DEBUG] ffmpeg_extract_subclip')
    #ffmpeg_extract_subclip(filename, start, end, targetname=temp_filename)
    
    #print('[DEBUG] VideoClip')
    #clip = VideoFileClip(temp_filename)
    clip = VideoFileClip(filename).subclip(start, end)
    clip = clip.volumex(2)

    #print('[DEBUG] TextClip')
    txt_clip = TextClip(final_text, font="font/VarelaRound-Regular.ttf", fontsize=50, color='white')
    txt_clip = txt_clip.set_pos(("center","top")).set_duration(60)
    
    #print('[DEBUG] CompositeVideoClip')
    video = CompositeVideoClip([clip, txt_clip])

    #print('[DEBUG] CompositeVideoClip write')
    video.write_videofile(final_filename)
    #print('[DEBUG] CompositeVideoClip end')
    

def main():

    text = input("Enter Your text please: ") [::-1]
    #text = 'Hello World'
    
    base_folder = time.strftime("result_%Y_%m_%d_%H_%M_%S")    
    os.makedirs(f"{base_folder}/result_edit", exist_ok=True)
    
    filename = "video.mp4"
    #filename = "BigBuckBunny.mp4"
    
    video_duration = VideoFileClip(filename).duration

    number = 0 # instead of `cut_name_num` and `txt_part` because both had the same value

    time_start = time.time()
    
    all_processes = []
    for start in range(0, int(video_duration), 60):

        end = start + 60
        
        if end > video_duration:
            end = video_duration
        
        number += 1
        
        print("add process:", number)
        p = multiprocessing.Process(target=my_process, args=(filename, text, start, end, number, base_folder)) # it has to use `target=`, `args=`
        p.start()  # start it
        all_processes.append(p)  # keep it to use `join()`
            
    # - after loop -
    
    for p in all_processes:
        p.join()  # wait for the end of process
        
    # because I use `number += 1` before loop so now `number` has number of subclips
    print('number of subclips:', number)

    time_end = time.time()
    
    diff = time_end - time_start
    print(f'time: {diff:.2f}s ({diff//60:02.0f}:{diff%60:02.2f})')


if __name__ == "__main__":
    main()

---

Previous version for 11 subclips starts 11 processes. Using Pool(4) you can put all processes in pool and it will run 4 processes at the same time. When one process will finish task then it will start next process with new arguments.

This time I use loop to create list with arguments for all processes

args_for_all_processes = []

for start in range(0, int(video_duration), 60):

    end = start + 60
    
    if end > video_duration:
        end = video_duration
    
    number += 1
    print("add process:", number)
    
    args_for_all_processes.append( (filename, text, start, end, number, base_folder) )

and I use this list with Pool and it will do the rest.

# I have 4 CPU so I use Pool(4) - but without value it should automatically use `os.cpu_count()`
with multiprocessing.Pool(4) as pool:      
    results = pool.starmap(my_process, args_for_all_processes)
    #print(results)

Pool may start processes in different order but if they use return to send some result then Pool will give results in correct order.

from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip
from moviepy.editor import *
import time
import multiprocessing

def my_process(filename, text, start, end, number, base_folder):

    clip_duration = end - start
    print(f'[DEBUG] number: {number:2} | start: {start:6.2f} | end: {end:6.2f} | duration: {clip_duration:.2f}')
    
    final_text = f"{number} {text}"

    temp_filename  = f"{base_folder}/cut_{number}.mp4"
    final_filename = f"{base_folder}/result_edit/cut_{number}.mp4"
    
    #print('[DEBUG] ffmpeg_extract_subclip')
    #ffmpeg_extract_subclip(filename, start, end, targetname=temp_filename)
    
    #print('[DEBUG] VideoClip')
    #clip = VideoFileClip(temp_filename)
    clip = VideoFileClip(filename).subclip(start, end)
    clip = clip.volumex(2)

    #print('[DEBUG] TextClip')
    txt_clip = TextClip(final_text, font="font/VarelaRound-Regular.ttf", fontsize=50, color='white')
    txt_clip = txt_clip.set_pos(("center","top")).set_duration(60)
    
    #print('[DEBUG] CompositeVideoClip')
    video = CompositeVideoClip([clip, txt_clip])

    #print('[DEBUG] CompositeVideoClip write')
    video.write_videofile(final_filename)
    #print('[DEBUG] CompositeVideoClip end')
    
    # return "OK"  # you can use `return` to send result/information to main process.
    
def main():

    text = input("Enter Your text please: ") [::-1]
    #text = 'Hello World'
    
    base_folder = time.strftime("result_%Y_%m_%d_%H_%M_%S")    
    os.makedirs(f"{base_folder}/result_edit", exist_ok=True)
    
    filename = "video.mp4"
    #filename = "BigBuckBunny.mp4"
    
    video_duration = VideoFileClip(filename).duration

    number = 0 # instead of `cut_name_num` and `txt_part` because both had the same value

    time_start = time.time()
    
    # first create list with arguments for all processes
    
    args_for_all_processes = []
    
    for start in range(0, int(video_duration), 60):

        end = start + 60
        
        if end > video_duration:
            end = video_duration
        
        number += 1
        print("add process:", number)
        
        args_for_all_processes.append( (filename, text, start, end, number, base_folder) )

    # - after loop -            
        
    # next put all processes to pool
        
    with multiprocessing.Pool(4) as pool:  # I have 4 CPU so I use Pool(4) - but it should use `os.cpu_count()` in `Pool()
    
        results = pool.starmap(my_process, args_for_all_processes)
        #print(results)
            
    # - after loop -
    
    # because I use `number += 1` before loop so now `number` has number of subclips
    print('number of subclips:', number)

    time_end = time.time()
    
    diff = time_end - time_start
    print(f'time: {diff:.2f}s ({diff//60:02.0f}:{diff%60:02.2f})')


if __name__ == "__main__":
    main()