Dynamically created QImage frames to ffmpeg stdin using QThread

I am trying to create video files with ffmpeg using frames dynamically created on a separate thread.

While I can create those frames and store them on disk/memory, I’d like to avoid that passage since the amount/size of the frames can be high and many “jobs” could be created with different format or options. But, also importantly, I’d like to better understand the logic behind this, as I admit I’ve not a very deep knowledge on how thread/processing actually works.

Right now I’m trying to create the QProcess in the QThread object, and then run the image creation thread as soon as the process is started, but it doesn’t seem to work: no file is created, and I don’t even get any output from standard error (but I know I should, since I can get it if I don’t use the thread).

Unfortunately, due to my little knowledge on how QProcess deals with threads and piping (and, obviously, all possible ffmpeg options), I really don’t understand how can achieve this.

Besides obviously getting the output file created, the expected result is to be able to launch the encoding (and possibly queue more encodings in the meantime) while keeping the UI responding and get notifications of the current processing state.

import re
from PyQt5 import QtCore, QtGui, QtWidgets

logRegExp = r'(?:(n:s+)(?P<frame>d+)s).*(?:(pts_time:s*)(?P<time>d+.d*))'

class Encoder(QtCore.QThread):
    completed = QtCore.pyqtSignal()
    frameDone = QtCore.pyqtSignal(object)
    def __init__(self, width=1280, height=720, frameCount=100):
        super().__init__()
        self.width = width
        self.height = height
        self.frameCount = frameCount

    def start(self):
        self.currentLog = ''
        self.currentData = bytes()
        self.process = QtCore.QProcess()
        self.process.setReadChannel(self.process.StandardError)
        self.process.finished.connect(self.completed)
        self.process.readyReadStandardError.connect(self.stderr)
        self.process.started.connect(super().start)
        self.process.start('ffmpeg', [
            '-y', 
            '-f', 'png_pipe', 
            '-i', '-', 
            '-c:v', 'libx264', 
            '-b:v', '800k', 
            '-an', 
            '-vf', 'showinfo',
            '/tmp/test.h264', 
        ])

    def stderr(self):
        self.currentLog += str(self.process.readAllStandardError(), 'utf-8')
        *lines, self.currentLog = self.currentLog.split('n')
        for line in lines:
            print('STDERR: {}'.format(line))
            match = re.search(logRegExp, line)
            if match:
                data = match.groupdict()
                self.frameDone.emit(int(data['frame']))

    def run(self):
        font = QtGui.QFont()
        font.setPointSize(80)
        rect = QtCore.QRect(0, 0, self.width, self.height)
        for frame in range(1, self.frameCount + 1):
            img = QtGui.QImage(QtCore.QSize(self.width, self.height), QtGui.QImage.Format_ARGB32)
            img.fill(QtCore.Qt.white)
            qp = QtGui.QPainter(img)
            qp.setFont(font)
            qp.setPen(QtCore.Qt.black)
            qp.drawText(rect, QtCore.Qt.AlignCenter, 'Frame {}'.format(frame))
            qp.end()
            img.save(self.process, 'PNG')
        print('frame creation complete')


class Test(QtWidgets.QWidget):
    def __init__(self):
        super().__init__()
        layout = QtWidgets.QVBoxLayout(self)
        self.startButton = QtWidgets.QPushButton('Start')
        layout.addWidget(self.startButton)

        self.frameLabel = QtWidgets.QLabel()
        layout.addWidget(self.frameLabel)

        self.process = Encoder()
        self.process.completed.connect(lambda: self.startButton.setEnabled(True))
        self.process.frameDone.connect(self.frameLabel.setNum)
        self.startButton.clicked.connect(self.create)

    def create(self):
        self.startButton.setEnabled(False)
        self.process.start()


import sys
app = QtWidgets.QApplication(sys.argv)
test = Test()
test.show()
sys.exit(app.exec_())

If I add the following lines at the end of run(), then the file is actually created and I get the stderr output, but I can see that it’s processed after the completion of the for cycle, which obviously is not the expected result:

    self.process.closeWriteChannel()
    self.process.waitForFinished()
    self.process.terminate()

Bonus: I’m on Linux, I don’t know if it works differently on Windows (and I suppose it would work similarly on MacOS), but in any case I’d like to know if there are differences and how to possibly deal with them.

Answer

It turns out that I was partially right and wrong.

1. ffmpeg has multiple levels and amounts of internal buffering, depending on input/output formats, filters and codecs: I just didn’t create enough frames to see that happening;
2. interaction with the QProcess should happen in the thread in which it was created;
3. for that reason, data cannot be directly written to the write channel from a different thread, so signals must be used instead;
4. the write channel must be closed (from its same thread) when all data has been written in order to ensure completion of the encoding;

Considering the above, I only use the thread to create the images, then emit a signal with the saved QByteArray of each image; finally, after image creation is completed I wait for the actual completion (based on the showinfo filter output) so that the thread is actually considered finished at that point. Some optimization could be used to queue further image creation in case of multiple jobs, but considering that it probably won’t improve performance that much, I prefer the current approach.

Here is the revised code, I tested with different formats and it seems to work as expected.

import re
from PyQt5 import QtCore, QtGui, QtWidgets

logRegExp = r'(?:(n:s+)(?P<frame>d+)s).*(?:(pts_time:s*)(?P<time>d+.d*))'

class Encoder(QtCore.QThread):
    completed = QtCore.pyqtSignal()
    frameDone = QtCore.pyqtSignal(object)
    imageReady = QtCore.pyqtSignal(object)
    def __init__(self):
        super().__init__()
        self.imageReady.connect(self.writeImage)
        self.queue = []

        self.process = QtCore.QProcess()
        self.process.setReadChannel(self.process.StandardError)
        self.process.finished.connect(self.processQueue)
        self.process.readyReadStandardError.connect(self.stderr)
        self.process.started.connect(self.start)

    def addJob(self, width=1280, height=720, frameCount=500, format='h264', *opts):
        self.queue.append((width, height, frameCount, format, opts))
        if not self.process.state():
            self.processQueue()

    def writeImage(self, image):
        self.process.write(image)
        self.imageCount += 1
        if self.imageCount == self.frameCount:
            self.process.closeWriteChannel()

    def processQueue(self):
        if not self.queue:
            return
        self.currentLog = ''
        self.lastFrameWritten = -1
        self.imageCount = 0
        self.width, self.height, self.frameCount, format, opts = self.queue.pop(0)
        args = [
            '-y', 
            '-f', 'png_pipe', 
            '-i', '-',
        ]
        if opts:
            args += [str(o) for o in opts]
        args += [
            '-an', 
            '-vf', 'showinfo',
            '/tmp/test.{}'.format(format), 
        ]
        self.process.start('ffmpeg', args)

    def stderr(self):
        self.currentLog += str(self.process.readAllStandardError(), 'utf-8')
        *lines, self.currentLog = self.currentLog.split('n')
        for line in lines:
            match = re.search(logRegExp, line)
            if match:
                data = match.groupdict()
                self.lastFrameWritten = int(data['frame'])
                self.frameDone.emit(self.lastFrameWritten + 1)
            else:
                print(line)

    def run(self):
        font = QtGui.QFont()
        font.setPointSize(80)
        rect = QtCore.QRect(0, 0, self.width, self.height)
        for frame in range(1, self.frameCount + 1):
            img = QtGui.QImage(QtCore.QSize(self.width, self.height), 
                QtGui.QImage.Format_ARGB32)
            img.fill(QtCore.Qt.white)
            qp = QtGui.QPainter(img)
            qp.setFont(font)
            qp.setPen(QtCore.Qt.black)
            qp.drawText(rect, QtCore.Qt.AlignCenter, 'Frame {}'.format(frame))
            qp.end()
            ba = QtCore.QByteArray()
            buffer = QtCore.QBuffer(ba)
            img.save(buffer, 'PNG')
            self.imageReady.emit(ba)
        while self.lastFrameWritten < self.frameCount - 1:
            self.sleep(.5)
        self.completed.emit()


class Test(QtWidgets.QWidget):
    def __init__(self):
        super().__init__()
        layout = QtWidgets.QVBoxLayout(self)
        self.startButton = QtWidgets.QPushButton('Start')
        layout.addWidget(self.startButton)

        self.frameLabel = QtWidgets.QLabel()
        layout.addWidget(self.frameLabel)

        self.encoder = Encoder()
        self.encoder.completed.connect(lambda: self.startButton.setEnabled(True))
        self.encoder.frameDone.connect(self.frameLabel.setNum)
        self.startButton.clicked.connect(self.create)

    def create(self):
        self.startButton.setEnabled(False)
        self.encoder.addJob()


if __name__ == '__main__':
    import sys
    app = QtWidgets.QApplication(sys.argv)
    test = Test()
    test.show()
    sys.exit(app.exec_())