From 6f155a94db27e041d7e4451ab0b5d1cb7ae7fc82 Mon Sep 17 00:00:00 2001 From: JeanExtreme002 Date: Sun, 14 Jun 2026 22:58:27 -0300 Subject: [PATCH] feat: add read_process_memory_into for zero-copy buffer reuse (#71) read_process_memory allocates a fresh object per call, which creates constant GC churn in long-running loops that poll the same-sized region over and over. Add read_process_memory_into(address, buffer) to read straight into a caller-owned, reusable buffer with no intermediate allocation, keeping memory use constant. Accepts any writable, contiguous buffer-protocol object (bytearray, ctypes array, writable memoryview, numpy array - sized in bytes) and returns the number of bytes read. Buffer prep/validation lives in a shared util.convert.as_writable_c_buffer helper; each backend reuses its existing low-level read primitive (ReadProcessMemory / process_vm_readv / mach_vm_read_overwrite), including the same partial-read OSError guard. --- PyMemoryEditor/linux/functions.py | 13 +++ PyMemoryEditor/linux/process.py | 5 ++ PyMemoryEditor/macos/functions.py | 13 +++ PyMemoryEditor/macos/process.py | 5 ++ PyMemoryEditor/process/abstract.py | 43 ++++++++++ PyMemoryEditor/util/__init__.py | 1 + PyMemoryEditor/util/convert.py | 48 +++++++++++ PyMemoryEditor/win32/functions.py | 35 ++++++++ PyMemoryEditor/win32/process.py | 6 ++ docs/api/openprocess.md | 17 ++++ docs/guide/read-write.md | 54 ++++++++++++ tests/memory/test_read_into.py | 132 +++++++++++++++++++++++++++++ 12 files changed, 372 insertions(+) create mode 100644 tests/memory/test_read_into.py diff --git a/PyMemoryEditor/linux/functions.py b/PyMemoryEditor/linux/functions.py index e310c0c..7921873 100644 --- a/PyMemoryEditor/linux/functions.py +++ b/PyMemoryEditor/linux/functions.py @@ -30,6 +30,7 @@ from ..process.thread_info import ThreadInfo from ..util import ( _validate_pytype, + as_writable_c_buffer, get_c_type_of, values_to_bytes, ) @@ -378,6 +379,18 @@ def read_process_memory(pid: int, address: int, pytype: Type[T], bufflength: int return data.value +def read_process_memory_into(pid: int, address: int, buffer) -> int: + """ + Read ``len(buffer)`` bytes from ``address`` directly into the writable + ``buffer``, with no intermediate allocation. Returns the number of bytes + read (always the buffer's byte length on success; a short read raises + ``_LinuxPartialIOError``). + """ + c_buffer = as_writable_c_buffer(buffer) + size = len(c_buffer) + return _process_vm_readv(pid, addressof(c_buffer), address, size) + + def search_addresses_by_value( pid: int, pytype: Type[T], diff --git a/PyMemoryEditor/linux/process.py b/PyMemoryEditor/linux/process.py index 4da0216..9a09afc 100644 --- a/PyMemoryEditor/linux/process.py +++ b/PyMemoryEditor/linux/process.py @@ -21,6 +21,7 @@ get_modules, get_threads, read_process_memory, + read_process_memory_into, search_addresses_by_pattern, search_addresses_by_value, search_values_by_addresses, @@ -117,6 +118,10 @@ def read_process_memory( self.pid, address, pytype, resolve_bufflength(pytype, bufflength) ) + def read_process_memory_into(self, address: int, buffer: object) -> int: + self.__require_open() + return read_process_memory_into(self.pid, address, buffer) + def search_by_addresses( self, pytype: Type[T], diff --git a/PyMemoryEditor/macos/functions.py b/PyMemoryEditor/macos/functions.py index 80f88f6..f6f9bf9 100644 --- a/PyMemoryEditor/macos/functions.py +++ b/PyMemoryEditor/macos/functions.py @@ -28,6 +28,7 @@ from ..process.thread_info import ThreadInfo from ..util import ( _validate_pytype, + as_writable_c_buffer, get_c_type_of, values_to_bytes, ) @@ -545,6 +546,18 @@ def read_process_memory( return data.value +def read_process_memory_into(task: int, address: int, buffer) -> int: + """ + Read ``len(buffer)`` bytes from ``address`` directly into the writable + ``buffer``, with no intermediate allocation. Returns the number of bytes + read (always the buffer's byte length on success; a short read raises + ``MachPartialReadError``). + """ + c_buffer = as_writable_c_buffer(buffer) + size = len(c_buffer) + return _mach_read(task, address, ctypes.addressof(c_buffer), size) + + def write_process_memory( task: int, address: int, diff --git a/PyMemoryEditor/macos/process.py b/PyMemoryEditor/macos/process.py index 78f226e..189c103 100644 --- a/PyMemoryEditor/macos/process.py +++ b/PyMemoryEditor/macos/process.py @@ -25,6 +25,7 @@ get_task_for_pid, get_threads, read_process_memory, + read_process_memory_into, release_task, search_addresses_by_pattern, search_addresses_by_value, @@ -292,6 +293,10 @@ def read_process_memory( self.__task, address, pytype, resolve_bufflength(pytype, bufflength) ) + def read_process_memory_into(self, address: int, buffer: object) -> int: + self.__require_open() + return read_process_memory_into(self.__task, address, buffer) + def write_process_memory( self, address: int, diff --git a/PyMemoryEditor/process/abstract.py b/PyMemoryEditor/process/abstract.py index 0846d84..35ff1b8 100644 --- a/PyMemoryEditor/process/abstract.py +++ b/PyMemoryEditor/process/abstract.py @@ -449,6 +449,49 @@ def read_process_memory( """ raise NotImplementedError() + @abstractmethod + def read_process_memory_into(self, address: int, buffer: object) -> int: + """ + Read ``len(buffer)`` bytes from ``address`` directly into a + pre-allocated, writable ``buffer`` and return the number of bytes read. + + This is the zero-copy counterpart of :meth:`read_process_memory`: where + that method allocates a fresh object on every call, this one fills a + buffer you own and reuse. In a tight polling / recording loop that reads + the same-sized region over and over, reusing one buffer keeps memory use + constant instead of producing a stream of short-lived objects for the + garbage collector to reclaim. + + :param address: target memory address (ex: 0x006A9EC0). + :param buffer: any writable, contiguous buffer-protocol object — a + ``bytearray``, a ``ctypes`` array, a writable ``memoryview``, a + ``numpy`` array, etc. Its byte length determines how many bytes are + read; an element-typed buffer (e.g. a ``numpy`` ``int32`` array) is + sized in **bytes**, not elements. The bytes are written in place; no + decoding is performed (this is the raw-``bytes`` read path). Decode + or reinterpret them yourself afterwards + (``int.from_bytes`` / ``struct.unpack`` / ``numpy`` views / ...). + :return: the number of bytes read — always equal to the buffer's byte + length on success (a short read raises instead, mirroring + :meth:`read_process_memory`). + + :raises TypeError: if ``buffer`` is not a writable buffer (e.g. an + immutable ``bytes`` object). + :raises ValueError: if ``buffer`` is empty or not contiguous. + :raises OSError: if the read fails, or returns fewer bytes than + requested (e.g. the range crosses an unreadable/freed page). + + Example + ------- + :: + + buffer = bytearray(16) + while recording: + process.read_process_memory_into(addr, buffer) + handle(buffer) # same buffer reused every iteration + """ + raise NotImplementedError() + @abstractmethod def write_process_memory( self, diff --git a/PyMemoryEditor/util/__init__.py b/PyMemoryEditor/util/__init__.py index cbc1b23..6e69307 100644 --- a/PyMemoryEditor/util/__init__.py +++ b/PyMemoryEditor/util/__init__.py @@ -4,6 +4,7 @@ UNSET, _check_int_fits, _validate_pytype, + as_writable_c_buffer, convert_from_byte_array, get_c_type_of, prepare_write, diff --git a/PyMemoryEditor/util/convert.py b/PyMemoryEditor/util/convert.py index 155fe93..cbc4677 100644 --- a/PyMemoryEditor/util/convert.py +++ b/PyMemoryEditor/util/convert.py @@ -209,6 +209,54 @@ def prepare_write( return pytype, length, value +def as_writable_c_buffer(buffer: Any) -> "ctypes.Array": + """ + Wrap a writable buffer-protocol object as a ``ctypes`` ``c_char`` array + that shares the same underlying storage, so a backend can read process + memory *directly into the caller's buffer* with no intermediate + allocation. Returning the ``ctypes`` array (rather than a bare address) + keeps a reference to the source alive for as long as the caller holds it, + which is what makes the address safe to pass to the OS read call. + + Accepts anything exposing a writable, contiguous buffer — ``bytearray``, + a ``ctypes`` array, a writable ``memoryview``, a ``numpy`` array, etc. The + array's byte length is taken from the buffer itself (``memoryview.nbytes``), + so element-typed buffers like a ``numpy`` ``int32`` array are sized in + bytes, not elements. + + :raises TypeError: if ``buffer`` does not support the buffer protocol or is + read-only (e.g. ``bytes`` — use ``bytearray`` instead). + :raises ValueError: if ``buffer`` is empty or not contiguous. + """ + try: + view = memoryview(buffer) + except TypeError: + raise TypeError( + "buffer must support the writable buffer protocol " + "(e.g. bytearray, a ctypes array, or a numpy array), got %s." + % type(buffer).__name__ + ) + + try: + if view.readonly: + raise TypeError( + "buffer must be writable; got a read-only buffer " + "(e.g. bytes). Use bytearray or another writable buffer." + ) + if not view.contiguous: + raise ValueError("buffer must be contiguous.") + nbytes = view.nbytes + finally: + # Release the inspection view promptly so it never lingers as an extra + # export on the source object (e.g. blocking a later bytearray resize). + view.release() + + if nbytes == 0: + raise ValueError("buffer must have a non-zero length.") + + return (ctypes.c_char * nbytes).from_buffer(buffer) + + def convert_from_byte_array( byte_array: ctypes.Array, pytype: Type[T], length: int ) -> T: diff --git a/PyMemoryEditor/win32/functions.py b/PyMemoryEditor/win32/functions.py index 6653043..7626cc7 100644 --- a/PyMemoryEditor/win32/functions.py +++ b/PyMemoryEditor/win32/functions.py @@ -27,6 +27,7 @@ from ..process.thread_info import ThreadInfo from ..util import ( _validate_pytype, + as_writable_c_buffer, get_c_type_of, values_to_bytes, ) @@ -501,6 +502,40 @@ def ReadProcessMemory( return data.value +def ReadProcessMemoryInto(process_handle: int, address: int, buffer) -> int: + """ + Read ``len(buffer)`` bytes from ``address`` directly into the writable + ``buffer``, with no intermediate allocation. Returns the number of bytes + read. + + Raises OSError if the read fails or returns fewer bytes than requested + (same partial-read guard as :func:`ReadProcessMemory`). + """ + c_buffer = as_writable_c_buffer(buffer) + size = len(c_buffer) + bytes_read = ctypes.c_size_t(0) + + ctypes.set_last_error(0) + success = kernel32.ReadProcessMemory( + process_handle, + ctypes.c_void_p(address), + ctypes.byref(c_buffer), + size, + ctypes.byref(bytes_read), + ) + + if not success: + _raise_last_error("ReadProcessMemory") + + if bytes_read.value != size: + raise OSError( + "ReadProcessMemory partial read at 0x%X: %d of %d bytes read." + % (address, bytes_read.value, size) + ) + + return bytes_read.value + + def _read_region(process_handle: int, address: int, size: int): """Read a memory region; returns the byte buffer or None on failure.""" region_data = (ctypes.c_byte * size)() diff --git a/PyMemoryEditor/win32/process.py b/PyMemoryEditor/win32/process.py index f86292a..ed62fb6 100644 --- a/PyMemoryEditor/win32/process.py +++ b/PyMemoryEditor/win32/process.py @@ -27,6 +27,7 @@ GetProcessHandle, GetThreads, ReadProcessMemory, + ReadProcessMemoryInto, SearchAddressesByPattern, SearchAddressesByValue, SearchValuesByAddresses, @@ -313,6 +314,11 @@ def read_process_memory( resolve_bufflength(pytype, bufflength), ) + def read_process_memory_into(self, address: int, buffer: object) -> int: + self.__require_open() + self.__require_read() + return ReadProcessMemoryInto(self.__process_handle, address, buffer) + def write_process_memory( self, address: int, diff --git a/docs/api/openprocess.md b/docs/api/openprocess.md index c6796f0..648d6eb 100644 --- a/docs/api/openprocess.md +++ b/docs/api/openprocess.md @@ -158,6 +158,23 @@ with OpenProcess( :returns: the original ``value`` you passed in — **not** the truncated/encoded form actually written (a capped ``str``/``bytes`` write returns the full original value). + +.. py:method:: read_process_memory_into(address, buffer) + + Read ``len(buffer)`` raw bytes from ``address`` directly into a + pre-allocated, writable ``buffer`` (no intermediate allocation) — the + zero-copy counterpart of :py:meth:`read_process_memory` for tight + read-the-same-region loops. See :doc:`../guide/read-write` for examples. + + :param int address: target memory address. + :param buffer: any writable, contiguous buffer-protocol object + (``bytearray``, ``ctypes`` array, writable ``memoryview``, ``numpy`` + array, …). Its byte length sets how many bytes are read; the bytes land + verbatim (no decoding). + :returns: the number of bytes read (the buffer's byte length on success). + :raises TypeError: if ``buffer`` is not a writable buffer (e.g. ``bytes``). + :raises ValueError: if ``buffer`` is empty or not contiguous. + :raises OSError: if the read fails or returns fewer bytes than requested. ``` ### Typed shortcuts diff --git a/docs/guide/read-write.md b/docs/guide/read-write.md index f3ca335..43f1723 100644 --- a/docs/guide/read-write.md +++ b/docs/guide/read-write.md @@ -195,6 +195,60 @@ Need the raw bytes with zero interpretation? Use `read_bytes(address, length)` and `write_bytes(address, data)`. ``` +## Reusing a buffer (zero-copy reads) + +Every `read_process_memory` call **allocates a fresh Python object** for the +result. That's fine for one-off reads, but in a tight loop that reads the same +region thousands of times — a recorder, a live overlay, a poller — those +throwaway objects pile up and keep the garbage collector busy. + +`read_process_memory_into(address, buffer)` reads straight into a buffer **you +own and reuse**, so a long-running loop runs with constant memory instead of a +steady stream of allocations: + +```python +buffer = bytearray(16) # allocate once + +with OpenProcess(name="game.exe") as process: + while recording: + process.read_process_memory_into(0x7FF40010, buffer) + handle(buffer) # same buffer, refilled in place every loop +``` + +It fills **`len(buffer)` bytes** (the buffer's size decides how much is read) +and returns the number of bytes read. The bytes land verbatim — no decoding — +so reinterpret them yourself with `int.from_bytes`, `struct.unpack`, a `numpy` +view, and so on. + +Any writable, contiguous buffer works — a `bytearray`, a `ctypes` array, a +writable `memoryview`, or a `numpy` array (sized in **bytes**, so a 4-element +`int32` array reads 16 bytes): + +```python +import numpy as np + +frame = np.zeros(4, dtype=np.int32) # 16 bytes +process.read_process_memory_into(0x7FF40010, frame) +# frame now holds the four int32 values, no per-read allocation +``` + +### Method signature + +```{eval-rst} +.. py:method:: read_process_memory_into(address, buffer) + :no-index: + + :param int address: target memory address. + :param buffer: a writable, contiguous buffer-protocol object + (``bytearray``, ``ctypes`` array, writable ``memoryview``, ``numpy`` + array, …). Its byte length sets how many bytes are read; the bytes are + written in place with no decoding. + :return: the number of bytes read (the buffer's byte length on success). + :raises TypeError: if ``buffer`` is not a writable buffer (e.g. ``bytes``). + :raises ValueError: if ``buffer`` is empty or not contiguous. + :raises OSError: if the read fails or returns fewer bytes than requested. +``` + ## Common errors - **`OSError`** — the address may have been freed between scan and write, or diff --git a/tests/memory/test_read_into.py b/tests/memory/test_read_into.py new file mode 100644 index 0000000..1d8dfe6 --- /dev/null +++ b/tests/memory/test_read_into.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- + +""" +Tests for ``read_process_memory_into`` — the zero-copy read that fills a +caller-owned, reusable buffer instead of allocating a fresh object per call +(GitHub issue #71). + +As with the typed-accessor tests, we plant ctypes values on the test's own +heap and read them back through the public API. +""" + +import ctypes +import os +import sys + +import pytest + +if sys.platform not in ("win32", "darwin") and not sys.platform.startswith("linux"): + pytest.skip("Platform not supported by PyMemoryEditor", allow_module_level=True) + + +from PyMemoryEditor import OpenProcess # noqa: E402 + + +@pytest.fixture +def process(): + handle = OpenProcess(pid=os.getpid()) + try: + yield handle + finally: + handle.close() + + +def test_reads_into_bytearray(process): + source = (ctypes.c_uint8 * 4)(0xDE, 0xAD, 0xBE, 0xEF) + buffer = bytearray(4) + + read = process.read_process_memory_into(ctypes.addressof(source), buffer) + + assert read == 4 + assert bytes(buffer) == b"\xde\xad\xbe\xef" + + +def test_returns_buffer_byte_length(process): + source = (ctypes.c_uint8 * 16)(*range(16)) + buffer = bytearray(16) + + assert process.read_process_memory_into(ctypes.addressof(source), buffer) == 16 + assert bytes(buffer) == bytes(source) + + +def test_buffer_can_be_reused_across_reads(process): + """The whole point of the API: one buffer, many reads, no new objects.""" + first = (ctypes.c_uint8 * 8)(1, 2, 3, 4, 5, 6, 7, 8) + second = (ctypes.c_uint8 * 8)(8, 7, 6, 5, 4, 3, 2, 1) + buffer = bytearray(8) + + process.read_process_memory_into(ctypes.addressof(first), buffer) + assert bytes(buffer) == bytes(first) + + process.read_process_memory_into(ctypes.addressof(second), buffer) + assert bytes(buffer) == bytes(second) + + +def test_buffer_length_decides_how_many_bytes_are_read(process): + """A 4-byte buffer reads exactly 4 bytes from an 8-byte source.""" + source = (ctypes.c_uint8 * 8)(0xAA, 0xBB, 0xCC, 0xDD, 0x11, 0x22, 0x33, 0x44) + buffer = bytearray(4) + + process.read_process_memory_into(ctypes.addressof(source), buffer) + + assert bytes(buffer) == b"\xaa\xbb\xcc\xdd" + + +def test_reads_into_ctypes_array(process): + source = (ctypes.c_uint8 * 4)(0x01, 0x02, 0x03, 0x04) + destination = (ctypes.c_uint8 * 4)() + + process.read_process_memory_into(ctypes.addressof(source), destination) + + assert bytes(destination) == b"\x01\x02\x03\x04" + + +def test_reads_into_writable_memoryview(process): + source = (ctypes.c_uint8 * 4)(0x10, 0x20, 0x30, 0x40) + backing = bytearray(4) + + process.read_process_memory_into(ctypes.addressof(source), memoryview(backing)) + + assert bytes(backing) == b"\x10\x20\x30\x40" + + +def test_element_typed_buffer_sized_in_bytes(process): + """A 2-element int32 buffer is 8 bytes, not 2.""" + source = (ctypes.c_int32 * 2)(0x01020304, 0x05060708) + destination = (ctypes.c_int32 * 2)() + + read = process.read_process_memory_into(ctypes.addressof(source), destination) + + assert read == 8 + assert destination[0] == 0x01020304 + assert destination[1] == 0x05060708 + + +def test_matches_read_process_memory(process): + """The bytes filled in must equal the plain read_process_memory result.""" + source = (ctypes.c_uint8 * 6)(0x09, 0x08, 0x07, 0x06, 0x05, 0x04) + addr = ctypes.addressof(source) + + expected = process.read_process_memory(addr, bytes, 6) + buffer = bytearray(6) + process.read_process_memory_into(addr, buffer) + + assert bytes(buffer) == expected + + +def test_read_only_buffer_is_rejected(process): + source = (ctypes.c_uint8 * 4)(0, 0, 0, 0) + with pytest.raises(TypeError): + process.read_process_memory_into(ctypes.addressof(source), b"immutable") + + +def test_empty_buffer_is_rejected(process): + source = (ctypes.c_uint8 * 4)(0, 0, 0, 0) + with pytest.raises(ValueError): + process.read_process_memory_into(ctypes.addressof(source), bytearray(0)) + + +def test_non_buffer_is_rejected(process): + source = (ctypes.c_uint8 * 4)(0, 0, 0, 0) + with pytest.raises(TypeError): + process.read_process_memory_into(ctypes.addressof(source), 12345)