Retrieving unpack requires a buffer of 8 bytes error socket python

I have the following code I am sending the result of intermediate predictions results from the client to server.

Client

server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host_name = socket.gethostname()
host_ip = 'localhost'
print('HOST IP:', host_ip)
port = 9999
socket_address = (host_ip, port)
server_socket.bind(socket_address)
server_socket.listen(5)
while True:
    client_socket, addr = server_socket.accept()
    print('GOT CONNECTION FROM:', addr)
    if client_socket:
        vid = cv2.VideoCapture("D:/testing.mp4")
        while (vid.isOpened()):
            img, frame = vid.read()
            image = img_to_array(frame)
            image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2]))
            image = preprocess_input(image)
            preds = model.predict(image)
            a = pickle.dumps(preds)
            message = struct.pack("Q", len(a)) + a
        client_socket.sendall(message)
client_socket.close()

server

    client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    host_ip = 'localhost'
    port = 9999
    client_socket.connect((host_ip, port))
    data = b""
    payload_size = struct.calcsize("Q")
    while True:
        while len(data) < payload_size:
            packet = client_socket.recv(4 * 1024)
            if not packet: break
            data += packet
        packed_msg_size = data[:payload_size]
        data = data[payload_size:]
        msg_size = struct.unpack("Q", packed_msg_size)[0]
        while len(data) < msg_size:
            data += client_socket.recv(4 * 1024)
        frame_data = data[:msg_size]
        data = data[msg_size:]
        frame = pickle.loads(frame_data)

While running the above code I am facing the below error

msg_size = struct.unpack(“Q”, packed_msg_size)[0] struct.error: unpack requires a buffer of 8 bytes

Thank you

Answer

You haven’t sorted out normal end of connection. As mentioned in the comments, when the connection is closed by the client, you hit if not packet: break but that only breaks you out of the inner loop and you keep on trying to process a non-existent message. There is another bug in there where you don’t clear out data per message.

When nesting loops, especially in communications where end of processing is discovered several steps down, I find it best to break the work into multiple functions with more narrowly defined roles. Here, you want to receive exactly one message at a time, build that into a python object and then use it. 3 functions for three things.

def do_the_things():
    client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    host_ip = 'localhost'
    port = 9999
    client_socket.connect((host_ip, port))
    while True:
        frame = recv_frame(client_socket)
        if frame is None:
            break

payload_size = struct.calcsize("Q")
    
def recv_obj(client_socket):
    data = recvall(client_socket, payload_size)
    if len(data) < payload_size:
        if not data:
            # client closed after last message, assume all is well
            return None
        else:
            raise IOError("Truncated message")
    packed_msg_size = data[:payload_size]
    data = data[payload_size:]
    msg_size = struct.unpack("Q", packed_msg_size)[0]
    remaining_data = recvall(client_socket, msg_size)
    if len(remaining_data) < msg_size:
        raise IOError("Truncated message")
    obj = pickle.loads(data + remaining_data)
    return obj

def recvall(sock, count):
    """Keep receiving up to `count` bytes, returning partial result
    if connection closed normally by remote."""
    data_bufs = []
    while count:
        data = sock.recv(count)
        if not data:
            break
        count -= len(data)
        data_bufs.append(data)
    return b"".join(data_bufs)