PyQt5/Pyqtgraph Get Numpy Array for What is Currently on the Scene

I have a pg.GraphicsLayoutWidget with some images and some ROIs displayed . I would like to get obtain a numpy array for what is currently on the scene, just like the export command in the context menu of the viewbox .

---

Let an arbitrary RGB image (i.e. a numpy array) image be given. For example , I am using https://drive.google.com/drive/folders/1ejY0CjfEwS6SGS2qe_uRX2JvlruMKvPX?usp=sharing . The following code runs:

from PyQt5.QtWidgets import*
import pyqtgraph as pg
import numpy as np

class MainWindow(QWidget):
    def __init__(self):
        super().__init__()
        self.layout = QVBoxLayout()
        
        self.button = QPushButton("Save")
        self.button.clicked.connect(self.get_numpy_array)
        self.layout.addWidget(self.button)
        
        self.graphics = Graphics(image, pg.ImageItem(image))
        self.layout.addWidget(self.graphics)
        
        self.setLayout(self.layout)
    
    def get_numpy_array(self):
        return
    
class Graphics(pg.GraphicsLayoutWidget):
    def __init__(self,image, image_item):
        super().__init__()
        layout = self.addLayout()
        self.image = image
        self.shape = image.shape
        self.viewbox = layout.addViewBox(lockAspect=True)
        self.image_item = image_item
        self.viewbox.addItem(self.image_item)
        self.viewbox.setLimits(minXRange = 0, 
                               minYRange = 0,
                               maxXRange = self.shape[0],
                               maxYRange = self.shape[1])
        x,h = 300,50
        polyline = pg.PolyLineROI(
[[x, x], [x+h, x],  [x+h,x+h],[x, x+h]], pen=pg.mkPen("b", width=5), closed=True, rotatable = False)
        self.viewbox.addItem(polyline)
        
        
if __name__ == '__main__':
    import sys
    app = QApplication(sys.argv)
    main = MainWindow()
    main.show()
    
    app.exec()

We see something like:

You will notice that an ROI has been added. It has nothing to do with my problem. I added it to only to emphasize I need what is currently on the scene. That is, the part in yellow dotted lines when right clicking on pg.GraphicsLayoutWidget.

In steps:

1. Copy paste the above code and write a command to read the image.
2. Scroll the mouse or drag the image or drag the ROI or toggle the ROI. Do whatever you like inside the pg.GraphicsLayoutWidget (it does not matter what exactly you do). We may end up in a situation like:

3. Move the mouse inside pg.GraphicsLayoutWidget and right click. We get the context menu:

4. Click Export... will give (the region inside yellow dotted lines is what I called on the scene):

5. For the export format, choose “Image File (PNG,TIF,JPG,…)”. Now click export will save an image like this:

6. Observe that this saved image is exactly the same as the part in step4 inside the yellow dotted lines.

My goal is to programmatically read an RGB numpy array upon user button click such that, if saved, is exactly this image. That is, whenever user clicks button, the program reads what is currently on the scene.

---

I know this is probably just a matter of how to copy exportDialog.py in the GraphicsScene folder . The lines of interest are :

def exportItemChanged(self, item, prev):
        if item is None:
            return
        if item.gitem is self.scene:
            newBounds = self.scene.views()[0].viewRect()
        else:
            newBounds = item.gitem.sceneBoundingRect()
        self.selectBox.setRect(newBounds)
        self.selectBox.show()
        self.updateFormatList()

But I get stuck: I don’t understand how to get the scene from my viewbox or how to get the array from the scene.

Answer

In my previous answer I was using ROI to get image data. However as you refine your question, it’s clear that you need 1:1 copy of graphics view with possibly other objects (not just copy of Image data in a view). You can use pyqtgraph’s ImageExporter object with toBytes parameter. Then you have to transform QImage into RGB numpy array.

Here is working example:

import numpy as np
import pyqtgraph as pg
from PIL import Image
from PyQt5.QtGui import QImage
from PyQt5.QtWidgets import *
from numpy import asarray
from pyqtgraph.exporters import ImageExporter


class MainWindow(QWidget):
    def __init__(self):
        super().__init__()
        self.layout = QVBoxLayout()

        self.button = QPushButton("Save")
        self.button.clicked.connect(self.get_numpy_array)
        self.layout.addWidget(self.button)
        image = asarray(Image.open("image.jpg"))
        self.graphics = Graphics(image, pg.ImageItem(image))
        self.layout.addWidget(self.graphics)

        self.setLayout(self.layout)

    def get_numpy_array(self):
        # Export current viewvbox into bytes
        exporter = ImageExporter(self.graphics.viewbox)
        data = exporter.export(toBytes=True)

        # Convert QIMage into RGB image
        input_img = data.convertToFormat(QImage.Format_RGB888)
        width = input_img.width()
        height = input_img.height()
        # Get pointer to data
        ptr = input_img.bits()
        ptr.setsize(input_img.byteCount())
        # Create numpy array from data
        arr = np.array(ptr).reshape(height, width, 3)
        # This part transforms array back to image
        # img = Image.fromarray(arr, 'RGB')
        # img.save("./slice.png")
        return arr


class Graphics(pg.GraphicsLayoutWidget):
    def __init__(self, image, image_item):
        super().__init__()
        layout = self.addLayout()
        self.image = image
        self.shape = image.shape
        self.viewbox = layout.addViewBox(lockAspect=True)
        self.image_item = image_item
        self.viewbox.addItem(self.image_item)
        self.viewbox.setLimits(minXRange=0,
                               minYRange=0,
                               maxXRange=self.shape[0],
                               maxYRange=self.shape[1])

        x, h = 300, 50
        polyline = pg.PolyLineROI(
            [[x, x], [x + h, x], [x + h, x + h], [x, x + h]], pen=pg.mkPen("b", width=5), closed=True, rotatable=False)
        self.viewbox.addItem(polyline)


if __name__ == '__main__':
    import sys

    app = QApplication(sys.argv)
    main = MainWindow()
    main.show()

    app.exec()