Build and run
cd code/bufsum
python -m pip install -e . --no-build-isolation -v
python -m pytest -qSources on disk are under code/bufsum/ and are compilable as-is.
Each section below links to its original file.
Files
bufsum.c— 181 linessetup.py— 18 linespyproject.toml— 13 linestests/test_bufsum.py— 87 linesREADME.md— 41 lines
bufsum.c
/*
* bufsum -- zero-copy bulk data plus correct interpreter release.
*
* Demonstrates:
* - acquiring, validating and releasing a Py_buffer on every path
* - Py_BEGIN_ALLOW_THREADS around pure-C work, with the error recorded
* in a plain C variable and raised only after re-attaching
* - a writable buffer request (in-place mutation)
* - the y* argument-parsing shortcut
*
* Build: pip install -e . --no-build-isolation
* Lessons: 06, 10, 12
*/
#include <Python.h>
#include <math.h>
#include <string.h>
/* Shared validation: a 1-D contiguous buffer of C doubles. */
static int
get_double_view(PyObject *obj, Py_buffer *view, int writable)
{
int flags = PyBUF_C_CONTIGUOUS | PyBUF_FORMAT;
if (writable) {
flags |= PyBUF_WRITABLE;
}
if (PyObject_GetBuffer(obj, view, flags) < 0) {
return -1; /* BufferError already set */
}
if (view->ndim != 1) {
PyErr_Format(PyExc_ValueError, "expected a 1-D buffer, got %d dimensions",
view->ndim);
goto error;
}
if (view->format == NULL || strcmp(view->format, "d") != 0) {
PyErr_Format(PyExc_TypeError, "expected a buffer of C doubles ('d'), got '%s'",
view->format ? view->format : "B");
goto error;
}
if (view->itemsize != (Py_ssize_t)sizeof(double)) {
PyErr_SetString(PyExc_TypeError, "unexpected item size");
goto error;
}
return 0;
error:
PyBuffer_Release(view);
return -1;
}
/* ------------------------------------------------------------------ *
* sum_doubles(buf, /) -> float
* ------------------------------------------------------------------ */
static PyObject *
bufsum_sum_doubles(PyObject *module, PyObject *obj)
{
Py_buffer view;
if (get_double_view(obj, &view, 0) < 0) {
return NULL;
}
const double *p = (const double *)view.buf; /* pinned by the view */
Py_ssize_t n = view.shape[0];
double total = 0.0;
int not_finite = 0; /* error flag in plain C */
Py_BEGIN_ALLOW_THREADS
for (Py_ssize_t i = 0; i < n; i++) {
total += p[i];
}
if (!isfinite(total)) {
not_finite = 1;
}
Py_END_ALLOW_THREADS
PyBuffer_Release(&view);
if (not_finite) {
PyErr_SetString(PyExc_OverflowError, "sum is not finite");
return NULL;
}
return PyFloat_FromDouble(total);
}
/* ------------------------------------------------------------------ *
* scale(buf, k, /) -> None (in place, requires a writable buffer)
* ------------------------------------------------------------------ */
static PyObject *
bufsum_scale(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
{
if (nargs != 2) {
PyErr_SetString(PyExc_TypeError, "scale() takes exactly 2 arguments");
return NULL;
}
double k = PyFloat_AsDouble(args[1]);
if (k == -1.0 && PyErr_Occurred()) {
return NULL;
}
Py_buffer view;
if (get_double_view(args[0], &view, 1) < 0) {
return NULL;
}
double *p = (double *)view.buf;
Py_ssize_t n = view.shape[0];
Py_BEGIN_ALLOW_THREADS
for (Py_ssize_t i = 0; i < n; i++) {
p[i] *= k;
}
Py_END_ALLOW_THREADS
PyBuffer_Release(&view);
Py_RETURN_NONE;
}
/* ------------------------------------------------------------------ *
* checksum(data, /) -> int
*
* Uses the y* format unit, which acquires a Py_buffer for you. Note that
* it must still be released on every path, including the error path.
* ------------------------------------------------------------------ */
static PyObject *
bufsum_checksum(PyObject *module, PyObject *args)
{
Py_buffer data;
if (!PyArg_ParseTuple(args, "y*:checksum", &data)) {
return NULL;
}
const unsigned char *p = (const unsigned char *)data.buf;
Py_ssize_t n = data.len;
unsigned long long h = 1469598103934665603ULL; /* FNV-1a offset basis */
Py_BEGIN_ALLOW_THREADS
for (Py_ssize_t i = 0; i < n; i++) {
h ^= (unsigned long long)p[i];
h *= 1099511628211ULL;
}
Py_END_ALLOW_THREADS
PyBuffer_Release(&data);
return PyLong_FromUnsignedLongLong(h);
}
/* ------------------------------------------------------------------ */
static PyMethodDef bufsum_methods[] = {
{"sum_doubles", bufsum_sum_doubles, METH_O,
PyDoc_STR("sum_doubles(buf, /)\n--\n\nSum a 1-D contiguous buffer of doubles.")},
{"scale", (PyCFunction)(void (*)(void))bufsum_scale, METH_FASTCALL,
PyDoc_STR("scale(buf, k, /)\n--\n\nMultiply every element of a writable buffer by k.")},
{"checksum", bufsum_checksum, METH_VARARGS,
PyDoc_STR("checksum(data, /)\n--\n\nFNV-1a hash of any bytes-like object.")},
{NULL}
};
static PyModuleDef_Slot bufsum_slots[] = {
{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
#if PY_VERSION_HEX >= 0x030D0000
{Py_mod_gil, Py_MOD_GIL_NOT_USED},
#endif
{0, NULL}
};
static PyModuleDef bufsum_module = {
.m_base = PyModuleDef_HEAD_INIT,
.m_name = "bufsum",
.m_doc = PyDoc_STR("Buffer protocol and interpreter release."),
.m_size = 0,
.m_methods = bufsum_methods,
.m_slots = bufsum_slots,
};
PyMODINIT_FUNC
PyInit_bufsum(void)
{
return PyModuleDef_Init(&bufsum_module);
}
Original source: code/bufsum/bufsum.c
setup.py
import sys
from setuptools import Extension, setup
warn_flags = ["/W4"] if sys.platform == "win32" else ["-Wall", "-Wextra", "-Wno-unused-parameter"]
libs = [] if sys.platform == "win32" else ["m"]
setup(
ext_modules=[
Extension(
name="bufsum",
sources=["bufsum.c"],
libraries=libs,
extra_compile_args=warn_flags,
),
],
)
Original source: code/bufsum/setup.py
pyproject.toml
[build-system]
requires = ["setuptools>=77"]
build-backend = "setuptools.build_meta"
[project]
name = "bufsum"
version = "0.1.0"
description = "Buffer protocol and GIL release example"
requires-python = ">=3.13"
[tool.pytest.ini_options]
testpaths = ["tests"]
Original source: code/bufsum/pyproject.toml
tests/test_bufsum.py
import array
import threading
import pytest
import bufsum
def doubles(*values):
return array.array("d", values)
def test_sum():
assert bufsum.sum_doubles(doubles(1.0, 2.0, 3.5)) == 6.5
assert bufsum.sum_doubles(doubles()) == 0.0
def test_sum_rejects_wrong_element_type():
with pytest.raises(TypeError, match="C doubles"):
bufsum.sum_doubles(array.array("i", [1, 2, 3]))
def test_sum_rejects_non_buffer():
with pytest.raises(TypeError):
bufsum.sum_doubles([1.0, 2.0]) # a list is not a buffer exporter
def test_sum_rejects_non_finite():
with pytest.raises(OverflowError):
bufsum.sum_doubles(doubles(float("inf"), 1.0))
def test_scale_in_place():
a = doubles(1.0, 2.0, 3.0)
assert bufsum.scale(a, 2.0) is None
assert list(a) == [2.0, 4.0, 6.0]
def test_scale_requires_writable():
ro = memoryview(doubles(1.0, 2.0)).toreadonly()
with pytest.raises(BufferError):
bufsum.scale(ro, 2.0)
def test_checksum_accepts_any_bytes_like():
h = bufsum.checksum(b"hello")
assert h == bufsum.checksum(bytearray(b"hello"))
assert h == bufsum.checksum(memoryview(b"hello"))
assert h != bufsum.checksum(b"hellp")
def test_buffer_pin_is_real():
ba = bytearray(b"x" * 16)
mv = memoryview(ba)
with pytest.raises(BufferError):
ba.extend(b"more") # refused while a view exists
del mv
ba.extend(b"more")
def test_non_contiguous_is_rejected():
np = pytest.importorskip("numpy")
arr = np.arange(12, dtype=np.float64).reshape(3, 4)
with pytest.raises((ValueError, BufferError)):
bufsum.sum_doubles(arr.T)
assert bufsum.sum_doubles(np.ascontiguousarray(arr.T).ravel()) == arr.sum()
def test_releases_the_interpreter():
"""Two threads summing large buffers should overlap, not serialize.
This is a smoke test, not a benchmark: it only asserts that the work
completes, since timing assertions are flaky in CI.
"""
data = array.array("d", [1.0]) * 1_000_000
results = []
def work():
results.append(bufsum.sum_doubles(data))
ts = [threading.Thread(target=work) for _ in range(4)]
for t in ts:
t.start()
for t in ts:
t.join()
assert results == [1_000_000.0] * 4
Original source: code/bufsum/tests/test_bufsum.py
README.md
# bufsum
Zero-copy bulk data through the buffer protocol, plus correct release of the
interpreter around pure-C work.
```bash
python -m pip install -e . --no-build-isolation -v
python -m pytest -q
```
```python
import array, bufsum
a = array.array("d", [1.0, 2.0, 3.5])
print(bufsum.sum_doubles(a)) # 6.5
bufsum.scale(a, 2.0) # in place, writable buffer
print(list(a)) # [2.0, 4.0, 7.0]
print(bufsum.checksum(b"hello"))
```
## What to look for in `bufsum.c`
| Pattern | Where |
|---|---|
| Acquire, validate, release on every path | `get_double_view` |
| Error recorded in plain C, raised after re-attaching | `bufsum_sum_doubles` |
| Writable buffer request | `get_double_view(..., writable=1)` |
| The `y*` parsing shortcut, still needing a release | `bufsum_checksum` |
## Things to try
1. Move `PyErr_SetString` inside the `Py_BEGIN_ALLOW_THREADS` block and run
the tests under a debug build.
2. Replace the `Py_buffer` with `PyUnicode_AsUTF8` on a `str` argument and
use that pointer while detached. Reason about why it is unsound even if
it appears to work.
3. Drop `PyBUF_C_CONTIGUOUS` from the request flags and pass `arr.T` from
NumPy. What does the function compute now?
4. Time `sum_doubles` from four threads with and without the
`Py_BEGIN_ALLOW_THREADS` pair.
Original source: code/bufsum/README.md