gh-146452: Fix pickle segfault when pickling dict with concurrent mutation

Author: overlorde

Lib/test/test_free_threading/test_pickle.py

@@ -0,0 +1,49 @@
+import pickle
+import threading
+import unittest
+
+from test.support import threading_helper
+
+
+@threading_helper.requires_working_threading()
+class TestPickleFreeThreading(unittest.TestCase):
+
+    def test_pickle_dumps_with_concurrent_dict_mutation(self):
+        # gh-146452: Pickling a dict while another thread mutates it
+        # used to segfault. batch_dict_exact() iterated dict items via
+        # PyDict_Next() which returns borrowed references, and a
+        # concurrent pop/replace could free the value before Py_INCREF
+        # got to it.
+        shared = {str(i): list(range(20)) for i in range(50)}
+
+        def dumper():
+            for _ in range(1000):
+                try:
+                    pickle.dumps(shared)
+                except RuntimeError:
+                    # "dictionary changed size during iteration" is expected
+                    pass
+
+        def mutator():
+            for j in range(1000):
+                key = str(j % 50)
+                shared[key] = list(range(j % 20))
+                if j % 10 == 0:
+                    shared.pop(key, None)
+                    shared[key] = [j]
+
+        threads = []
+        for _ in range(10):
+            threads.append(threading.Thread(target=dumper))
+        threads.append(threading.Thread(target=mutator))
+
+        for t in threads:
+            t.start()
+        for t in threads:
+            t.join()
+
+        # If we get here without a segfault, the test passed.
+
+
+if __name__ == "__main__":
+    unittest.main()

Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst

@@ -0,0 +1,4 @@
+Fix segfault in :mod:`pickle` when pickling a dictionary concurrently
+mutated by another thread in the free-threaded build. The dict iteration in
+``batch_dict_exact`` now holds a critical section to prevent borrowed
+references from being invalidated mid-iteration.

Modules/_pickle.c

@@ -3454,7 +3454,7 @@ static int
 batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj)
 {
     PyObject *key = NULL, *value = NULL;
-    int i;
+    int i, has_next;
     Py_ssize_t dict_size, ppos = 0;
 
     const char mark_op = MARK;
@@ -3471,9 +3471,16 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj)
     Py_ssize_t total = 0;
     do {
         if (dict_size - total == 1) {
+            /* gh-146452: Prevent concurrent dict mutation from
+               invalidating the borrowed refs from PyDict_Next(). */
+            Py_BEGIN_CRITICAL_SECTION(obj);
             PyDict_Next(obj, &ppos, &key, &value);
-            Py_INCREF(key);
-            Py_INCREF(value);
+            Py_XINCREF(key);
+            Py_XINCREF(value);
+            Py_END_CRITICAL_SECTION();
+            if (key == NULL) {
+                return 0;
+            }
             if (save(state, self, key, 0) < 0) {
                 goto error;
             }
@@ -3491,9 +3498,17 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj)
         i = 0;
         if (_Pickler_Write(self, &mark_op, 1) < 0)
             return -1;
-        while (PyDict_Next(obj, &ppos, &key, &value)) {
-            Py_INCREF(key);
-            Py_INCREF(value);
+        for (;;) {
+            /* gh-146452: same as above. */
+            Py_BEGIN_CRITICAL_SECTION(obj);
+            has_next = PyDict_Next(obj, &ppos, &key, &value);
+            if (has_next) {
+                Py_INCREF(key);
+                Py_INCREF(value);
+            }
+            Py_END_CRITICAL_SECTION();
+            if (!has_next)
+                break;
             if (save(state, self, key, 0) < 0) {
                 goto error;
             }