In the Python application, I have a bytearray with some data. My python code is called by some external native library (DLL/so/dylib) (using ctypes and its callback functions). One of the callback functions includes a pointer to the unmanaged buffer, allocated in that external library.
I need to copy a part of the bytearray into this unmanaged buffer or copy contents of the unmanaged buffer into the bytearray at specific location.
To put it simply, I have
def copy_from_python_callback(c_void_p_parameter : c_void_p, offset : int, size : int): managed_buf = bytearray(some_size) # suppose we have data in this buffer already unmanaged_buf = c_void_p_parameter # what do I need to do here? # src_buf = pointer_to_specific_byte_in_managed_buf memmove(unmanaged_buf, src_buf, size) def copy_to_python_callback(c_void_p_parameter : c_void_p, offset : int, size : int): managed_buf = bytearray(some_size) #some_size is assumed to be larger than offset + size unmanaged_buf = c_void_p_parameter # what do I need to do here? # dst_buf = pointer_to_specific_byte_in_managed_buf memmove(dst_buf, unmanaged_buf, size)
In other languages, the answer would be simple – I’d either call the dedicated method (e.g. in .NET Framework’s Marshal class), or get the pointer to the specific byte in the bytearray (in native languages like C++ or Pascal) and be done. Unfortunately, I don’t see how to do either of these operations in Python without an intermediate bytes() or similar buffer.
I have some methods which use the intermediate bytes() instance, but copying the data just because there’s no way to get a pointer seems to be weird to me.
I am looking for version-independent solution, if possible, but can live with python3-only one as well. Thank you in advance.
Advertisement
Answer
Listing [Python.Docs]: ctypes – A foreign function library for Python.
What you’re looking for is possible. Arrays (CTypes) come in handy.
Below it’s a “small” example. For demo purposes, the buffers only contain “human friendly” chars.
dll00.c:
#include <stdio.h> #include <stdint.h> #include <stdlib.h> #if defined(_WIN32) # define DLL00_EXPORT_API __declspec(dllexport) #else # define DLL00_EXPORT_API #endif #define C_TAG "From C - " typedef int (*ReadFunc)(void *ptr, uint32_t offset, uint32_t size); typedef int (*WriteFunc)(void *ptr, uint32_t offset, uint32_t size); #if defined(__cplusplus) extern "C" { #endif DLL00_EXPORT_API int testBufferCallbacks(uint32_t bufSize, ReadFunc read, uint32_t readOffset, uint32_t readSize, WriteFunc write, uint32_t writeOffset, uint32_t writeSize); #if defined(__cplusplus) } #endif void prinBuffer(const uint8_t *buf, uint32_t size) { printf("%sBuffer (size %d) 0x%016llXn Contents: ", C_TAG, size, (uint64_t)buf); for (uint32_t i = 0; i < size; i++) { printf("%c", ((uint8_t*)buf)[i]); } printf("n"); } int testBufferCallbacks(uint32_t bufSize, ReadFunc read, uint32_t readOffset, uint32_t readSize, WriteFunc write, uint32_t writeOffset, uint32_t writeSize) { const uint8_t charOffset = 0x41; void *buf = malloc(bufSize); uint8_t *cBuf = (uint8_t*)buf; for (uint32_t i = 0; i < bufSize; i++) { cBuf[i] = (uint8_t)((i + charOffset) % 0x100); } prinBuffer(cBuf, bufSize); if ((read != NULL) && (readSize > 0)) { printf("n%sCalling read(0x%016llX, %u, %u)...n", C_TAG, (uint64_t)buf, readOffset, readSize); read(buf, readOffset, readSize); } if ((write != NULL) && (writeSize > 0)) { printf("n%sCalling write(0x%016llX, %u, %u)...n", C_TAG, (uint64_t)buf, writeOffset, writeSize); write(buf, writeOffset, writeSize); prinBuffer(cBuf, bufSize); } if ((read != NULL) && (readSize > 0)) { printf("n%sCalling read(0x%016llX, %u, %u)...n", C_TAG, (uint64_t)buf, readOffset, readSize); read(buf, readOffset, readSize); } prinBuffer(cBuf, bufSize); free(buf); printf("n%sDone.n", C_TAG); return 0; }
code00.py:
#!/usr/bin/env python3 import sys import ctypes as ct DLL_NAME = "./dll00.dll" ReadFunc = ct.CFUNCTYPE(ct.c_int, ct.c_void_p, ct.c_uint32, ct.c_uint32) WriteFunc = ct.CFUNCTYPE(ct.c_int, ct.c_void_p, ct.c_uint32, ct.c_uint32) def create_bytearray(size, offset_char=0x61): contents = "".join(chr(i) for i in range(offset_char, offset_char + size)) return bytearray(contents.encode()) def read_c_buf(buf : ct.c_void_p, offset : ct.c_uint32, size : ct.c_uint32): print("C buf: 0x{0:016X}".format(buf)) ba = create_bytearray(0x1A) print("Python initial buffer: {0:}".format(ba)) UCharArr = ct.c_uint8 * size uchar_arr = UCharArr.from_buffer(ba, offset) # Shared memory ct.memmove(uchar_arr, buf, size) print("Python final buffer: {0:}n".format(ba)) return 0 def write_c_buf(buf : ct.c_void_p, offset : ct.c_uint32, size : ct.c_uint32): print("C buf: 0x{0:016X}".format(buf)) ba = create_bytearray(size + offset, offset_char=0x30 - offset) print("Python buffer: {0:}n".format(ba)) UCharArr = ct.c_uint8 * size uchar_arr = UCharArr.from_buffer(ba, offset) # Shared memory ct.memmove(buf, uchar_arr, size) return 0 def main(*argv): dll00 = ct.CDLL(DLL_NAME) testBufferCallbacks = dll00.testBufferCallbacks testBufferCallbacks.argtypes = (ct.c_uint32, ReadFunc, ct.c_uint32, ct.c_uint32, WriteFunc, ct.c_uint32, ct.c_uint32) testBufferCallbacks.restype = ct.c_int read_callback = ReadFunc(read_c_buf) buf_size = 0x1A read_offset = 10 read_size = 16 write_callback = WriteFunc(write_c_buf) write_offset = 5 write_size = 10 res = testBufferCallbacks(buf_size, read_callback, read_offset, read_size, write_callback, write_offset, write_size) print("n{0:s} returned: {1:d}".format(testBufferCallbacks.__name__, res)) if __name__ == "__main__": print("Python {:s} {:03d}bit on {:s}n".format(" ".join(elem.strip() for elem in sys.version.split("n")), 64 if sys.maxsize > 0x100000000 else 32, sys.platform)) rc = main(*sys.argv[1:]) print("nDone.") sys.exit(rc)
Output:
[cfati@CFATI-5510-0:e:WorkDevStackOverflowq059255471]> sopr.bat ### Set shorter prompt to better fit when pasted in StackOverflow (or other) pages ### [prompt]> "c:Installx86MicrosoftVisual Studio Community2017VCAuxiliaryBuildvcvarsall.bat" x64 ********************************************************************** ** Visual Studio 2017 Developer Command Prompt v15.9.17 ** Copyright (c) 2017 Microsoft Corporation ********************************************************************** [vcvarsall.bat] Environment initialized for: 'x64' [prompt]> dir /b code00.py code01.py dll00.c [prompt]> cl /nologo /MD /DDLL dll00.c /link /NOLOGO /DLL /OUT:dll00.dll dll00.c Creating library dll00.lib and object dll00.exp [prompt]> dir /b code00.py code01.py dll00.c dll00.dll dll00.exp dll00.lib dll00.obj [prompt]> "e:WorkDevVEnvspy_064_03.07.03_test0Scriptspython.exe" code00.py Python 3.7.3 (v3.7.3:ef4ec6ed12, Mar 25 2019, 22:22:05) [MSC v.1916 64 bit (AMD64)] 064bit on win32 From C - Buffer (size 26) 0x0000016BFE546EB0 Contents: ABCDEFGHIJKLMNOPQRSTUVWXYZ From C - Calling read(0x0000016BFE546EB0, 10, 16)... C buf: 0x0000016BFE546EB0 Python initial buffer: bytearray(b'abcdefghijklmnopqrstuvwxyz') Python final buffer: bytearray(b'abcdefghijABCDEFGHIJKLMNOP') From C - Calling write(0x0000016BFE546EB0, 5, 10)... C buf: 0x0000016BFE546EB0 Python buffer: bytearray(b'+,-./0123456789') From C - Buffer (size 26) 0x0000016BFE546EB0 Contents: 0123456789KLMNOPQRSTUVWXYZ From C - Calling read(0x0000016BFE546EB0, 10, 16)... C buf: 0x0000016BFE546EB0 Python initial buffer: bytearray(b'abcdefghijklmnopqrstuvwxyz') Python final buffer: bytearray(b'abcdefghij0123456789KLMNOP') From C - Buffer (size 26) 0x0000016BFE546EB0 Contents: 0123456789KLMNOPQRSTUVWXYZ From C - Done. testBufferCallbacks returned: 0 Done.
Needless to say that going outside (C) buffer’s bounds, would yield Undefined Behavior (and my code doesn’t perform that kind of checks).