list: make list thread-safe

colesbury · Apr 23, 2023 · df4c51f · df4c51f
1 parent d896dfc
commit df4c51f
Show file tree

Hide file tree

Showing 14 changed files with 1,022 additions and 676 deletions.
diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat
diff --git a/Include/cpython/listobject.h b/Include/cpython/listobject.h
@@ -19,8 +19,17 @@ typedef struct {
      * the list is not yet visible outside the function that builds it.
      */
     Py_ssize_t allocated;
+
+    _PyMutex mutex;
+
+    uint8_t maybe_shared;
 } PyListObject;
 
+typedef struct {
+    Py_ssize_t allocated;
+    PyObject *ob_item[1];
+} _PyListArray;
+
 PyAPI_FUNC(PyObject *) _PyList_Extend(PyListObject *, PyObject *);
 PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out);
 

diff --git a/Include/cpython/setobject.h b/Include/cpython/setobject.h
@@ -56,6 +56,7 @@ typedef struct {
 
     setentry smalltable[PySet_MINSIZE];
     PyObject *weakreflist;      /* List of weak references */
+    _PyMutex mutex;
 } PySetObject;
 
 #define _PySet_CAST(so) \

diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h
@@ -37,6 +37,9 @@ struct _Py_list_state {
 
 #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item)
 
+// append without acquiring lock
+PyAPI_FUNC(int) _PyList_AppendPrivate(PyObject *, PyObject *);
+
 extern int
 _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem);
 

diff --git a/Include/listobject.h b/Include/listobject.h
@@ -29,6 +29,7 @@ PyAPI_FUNC(PyObject *) PyList_New(Py_ssize_t size);
 PyAPI_FUNC(Py_ssize_t) PyList_Size(PyObject *);
 
 PyAPI_FUNC(PyObject *) PyList_GetItem(PyObject *, Py_ssize_t);
+PyAPI_FUNC(PyObject *) PyList_FetchItem(PyObject *, Py_ssize_t);
 PyAPI_FUNC(int) PyList_SetItem(PyObject *, Py_ssize_t, PyObject *);
 PyAPI_FUNC(int) PyList_Insert(PyObject *, Py_ssize_t, PyObject *);
 PyAPI_FUNC(int) PyList_Append(PyObject *, PyObject *);

diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py
@@ -1162,7 +1162,7 @@ def test_refcount_errors(self):
         p.stderr.close()
         # Verify that stderr has a useful error message:
         self.assertRegex(stderr,
-            br'gcmodule\.c:[0-9]+: .*Assertion "gc_get_refs\(gc\) > 0" failed.')
+            br'gcmodule\.c:[0-9]+: .*Assertion.* failed: refcount is too small')
         self.assertRegex(stderr,
             br'refcount is too small')
         # "address : 0x7fb5062efc18"

diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
@@ -1277,7 +1277,7 @@ def test_gc_head_size(self):
         # bool objects are not gc tracked
         self.assertEqual(sys.getsizeof(True), vsize('') + self.longdigit)
         # but lists are
-        self.assertEqual(sys.getsizeof([]), vsize('Pn') + gc_header_size)
+        self.assertEqual(sys.getsizeof([]), vsize('PnP') + gc_header_size)
 
     def test_errors(self):
         class BadSizeof:

diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml
@@ -2233,6 +2233,8 @@
     added = '3.12'
 [function.PyDict_FetchItemWithError]
     added = '3.12'
+[function.PyList_FetchItem]
+    added = '3.12'
 
 # Support for Stable ABI in debug builds