PR for tracking changes for immutability

.github/workflows/build.yml

@@ -5,10 +5,12 @@ on:
   push:
     branches:
     - 'main'
+    - '*-main'
     - '3.*'
   pull_request:
     branches:
     - 'main'
+    - '*-main'
     - '3.*'
 
 permissions:
@@ -168,10 +170,11 @@ jobs:
           - arm64
         free-threading:
           - false
-          - true
-        exclude:
-          # Skip Win32 on free-threaded builds
-          - { arch: Win32, free-threading: true }
+        # TODO(Immutable): Enable free-threading build when it is made to work.
+        #        - true
+        # exclude:
+        #   # Skip Win32 on free-threaded builds
+        #   - { arch: Win32, free-threading: true }
     uses: ./.github/workflows/reusable-windows.yml
     with:
       arch: ${{ matrix.arch }}
@@ -209,10 +212,11 @@ jobs:
         - macos-15-intel
         free-threading:
         - false
-        - true
-        exclude:
-        - os: macos-15-intel
-          free-threading: true
+        # TODO(Immutable): Enable free-threading build when it is made to work.
+        #        - true
+        # exclude:
+        # - os: macos-15-intel
+        #   free-threading: true
     uses: ./.github/workflows/reusable-macos.yml
     with:
       config_hash: ${{ needs.build-context.outputs.config-hash }}
@@ -234,14 +238,15 @@ jobs:
         - true
         free-threading:
         - false
-        - true
+        # TODO(Immutable): Enable free-threading build when it is made to work.
+        #        - true
         os:
         - ubuntu-24.04
         - ubuntu-24.04-arm
         exclude:
-        # Do not test BOLT with free-threading, to conserve resources
-        - bolt: true
-          free-threading: true
+        # # Do not test BOLT with free-threading, to conserve resources
+        # - bolt: true
+        #   free-threading: true
         # BOLT currently crashes during instrumentation on aarch64
         - os: ubuntu-24.04-arm
           bolt: true
@@ -614,13 +619,14 @@ jobs:
         - Thread
         free-threading:
         - false
-        - true
+        # TODO(Immutable): Enable free-threading build when it is made to work.
+        #        - true
         sanitizer:
         - TSan
-        include:
-        - check-name: Undefined behavior
-          sanitizer: UBSan
-          free-threading: false
+        # include:
+        # - check-name: Undefined behavior
+        #   sanitizer: UBSan
+        #   free-threading: false
     uses: ./.github/workflows/reusable-san.yml
     with:
       sanitizer: ${{ matrix.sanitizer }}

.github/workflows/jit.yml

@@ -129,33 +129,33 @@ jobs:
           make all --jobs 4
           ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
-  jit-with-disabled-gil:
-    name: Free-Threaded (Debug)
-    needs: interpreter
-    runs-on: ubuntu-24.04
-    timeout-minutes: 90
-    strategy:
-      fail-fast: false
-      matrix:
-        llvm:
-          - 19
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-      - name: Build with JIT enabled and GIL disabled
-        run: |
-          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
-          export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
-          ./configure --enable-experimental-jit --with-pydebug --disable-gil
-          make all --jobs 4
-      - name: Run tests
-        run: |
-          ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
-        continue-on-error: true
+  # jit-with-disabled-gil:
+  #   name: Free-Threaded (Debug)
+  #   needs: interpreter
+  #   runs-on: ubuntu-24.04
+  #   timeout-minutes: 90
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       llvm:
+  #         - 19
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         persist-credentials: false
+  #     - uses: actions/setup-python@v5
+  #       with:
+  #         python-version: '3.11'
+  #     - name: Build with JIT enabled and GIL disabled
+  #       run: |
+  #         sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
+  #         export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
+  #         ./configure --enable-experimental-jit --with-pydebug --disable-gil
+  #         make all --jobs 4
+  #     - name: Run tests
+  #       run: |
+  #         ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+  #       continue-on-error: true
 
   no-opt-jit:
     name: JIT without optimizations (Debug)

.github/workflows/tail-call.yml

@@ -122,11 +122,11 @@ jobs:
           make all --jobs 4
           ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
-      - name: Native Linux with free-threading (release)
-        if: matrix.target == 'free-threading'
-        run: |
-          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
-          export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
-          CC=clang-20 ./configure --with-tail-call-interp --disable-gil
-          make all --jobs 4
-          ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+      # - name: Native Linux with free-threading (release)
+      #   if: matrix.target == 'free-threading'
+      #   run: |
+      #     sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
+      #     export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
+      #     CC=clang-20 ./configure --with-tail-call-interp --disable-gil
+      #     make all --jobs 4
+      #     ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3

Doc/includes/typestruct.h

@@ -92,4 +92,7 @@ typedef struct _typeobject {
      * Otherwise, limited to MAX_VERSIONS_PER_CLASS (defined elsewhere).
      */
     uint16_t tp_versions_used;
+
+    /* call function for all referenced objects (includes non-cyclic refs) */
+    traverseproc tp_reachable;
 } PyTypeObject;

Include/Python.h

@@ -147,6 +147,7 @@ __pragma(warning(disable: 4201))
 #include "fileutils.h"
 #include "cpython/pyfpe.h"
 #include "cpython/tracemalloc.h"
+#include "immutability.h"
 
 #ifdef _MSC_VER
 __pragma(warning(pop))  // warning(disable: 4201)

Include/cpython/immutability.h

@@ -0,0 +1,13 @@
+#ifndef Py_CPYTHON_IMMUTABLE_H
+#  error "this header file must not be included directly"
+#endif
+
+PyAPI_DATA(PyTypeObject) _PyNotFreezable_Type;
+
+PyAPI_FUNC(int) _PyImmutability_Freeze(PyObject*);
+PyAPI_FUNC(int) _PyImmutability_FreezeMany(PyObject *const *, Py_ssize_t);
+PyAPI_FUNC(int) _PyImmutability_RegisterFreezable(PyTypeObject*);
+PyAPI_FUNC(int) _PyImmutability_RegisterShallowImmutable(PyTypeObject*);
+PyAPI_FUNC(int) _PyImmutability_CanViewAsImmutable(PyObject*);
+PyAPI_FUNC(int) _PyImmutability_SetFreezable(PyObject *, int);
+PyAPI_FUNC(int) _PyImmutability_GetFreezable(PyObject *);

Include/cpython/listobject.h

@@ -44,6 +44,7 @@ PyList_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) {
     PyListObject *list = _PyList_CAST(op);
     assert(0 <= index);
     assert(index < list->allocated);
+    // TODO(Immutable): Add assert to check if the list is immutable
     list->ob_item[index] = value;
 }
 #define PyList_SET_ITEM(op, index, value) \

Include/cpython/object.h

@@ -239,6 +239,12 @@ struct _typeobject {
      * Otherwise, limited to MAX_VERSIONS_PER_CLASS (defined elsewhere).
      */
     uint16_t tp_versions_used;
+
+    /* call function for all referenced objects (includes non-cyclic refs) */
+    traverseproc tp_reachable;
+
+    /* A callback called before a type is frozen. */
+    prefreezeproc tp_prefreeze;
 };
 
 #define _Py_ATTR_CACHE_UNUSED (30000)  // (see tp_versions_used)
@@ -291,6 +297,7 @@ typedef struct _heaptypeobject {
 PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *);
 PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *);
 PyAPI_FUNC(PyObject *) _PyType_LookupRef(PyTypeObject *, PyObject *);
+PyAPI_FUNC(int) _PyType_HasExtensionSlots(PyTypeObject *);
 PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *);
 
 PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int);

Include/cpython/weakrefobject.h

@@ -16,6 +16,11 @@ struct _PyWeakReference {
 
     /* A callable to invoke when wr_object dies, or NULL if none. */
     PyObject *wr_callback;
+    /* ID of the interpreter where the callback resides.
+     * Used for immutable objects to know which interpreter to call back into.
+     * This is -1 if the callback is NULL.
+     */
+    int64_t callback_ipid;
 
     /* A cache for wr_object's hash code.  As usual for hashes, this is -1
      * if the hash code isn't known yet.

Include/descrobject.h

@@ -84,6 +84,7 @@ struct PyMemberDef {
 #define Py_AUDIT_READ          2 // Added in 3.10, harmless no-op before that
 #define _Py_WRITE_RESTRICTED   4 // Deprecated, no-op. Do not reuse the value.
 #define Py_RELATIVE_OFFSET     8
+// TODO(Immutable): Could use this to mark members as needing a lock.
 
 PyAPI_FUNC(PyObject *) PyMember_GetOne(const char *, PyMemberDef *);
 PyAPI_FUNC(int) PyMember_SetOne(char *, PyMemberDef *, PyObject *);

Include/dictobject.h

@@ -23,7 +23,8 @@ PyAPI_FUNC(PyObject *) PyDict_GetItem(PyObject *mp, PyObject *key);
 PyAPI_FUNC(PyObject *) PyDict_GetItemWithError(PyObject *mp, PyObject *key);
 PyAPI_FUNC(int) PyDict_SetItem(PyObject *mp, PyObject *key, PyObject *item);
 PyAPI_FUNC(int) PyDict_DelItem(PyObject *mp, PyObject *key);
-PyAPI_FUNC(void) PyDict_Clear(PyObject *mp);
+// Note(Immutable): If dictionary is immutable, then clear can fail. Had to change signature here!
+PyAPI_FUNC(int) PyDict_Clear(PyObject *mp);
 PyAPI_FUNC(int) PyDict_Next(
     PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value);
 PyAPI_FUNC(PyObject *) PyDict_Keys(PyObject *mp);

Include/immutability.h

@@ -0,0 +1,19 @@
+#ifndef Py_IMMUTABILITY_H
+#define Py_IMMUTABILITY_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#ifndef Py_LIMITED_API
+#  define Py_CPYTHON_IMMUTABLE_H
+#  include "cpython/immutability.h"
+#  undef Py_CPYTHON_IMMUTABLE_H
+#endif
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_IMMUTABILITY_H */

Include/internal/pycore_cell.h

@@ -16,21 +16,30 @@ extern "C" {
 // Sets the cell contents to `value` and return previous contents. Steals a
 // reference to `value`.
 static inline PyObject *
-PyCell_SwapTakeRef(PyCellObject *cell, PyObject *value)
+PyCell_SwapTakeRef(PyCellObject *cell, PyObject *value, int* result)
 {
-    PyObject *old_value;
+    PyObject *old_value = NULL;
+    *result = 0;
     Py_BEGIN_CRITICAL_SECTION(cell);
-    old_value = cell->ob_ref;
-    FT_ATOMIC_STORE_PTR_RELEASE(cell->ob_ref, value);
+    if(Py_CHECKWRITE(cell)){
+        old_value = cell->ob_ref;
+        FT_ATOMIC_STORE_PTR_RELEASE(cell->ob_ref, value);
+    }
+    else {
+        *result = -1;
+        Py_XDECREF(value);
+    }
     Py_END_CRITICAL_SECTION();
     return old_value;
 }
 
-static inline void
+static inline int
 PyCell_SetTakeRef(PyCellObject *cell, PyObject *value)
 {
-    PyObject *old_value = PyCell_SwapTakeRef(cell, value);
+    int result = 0;
+    PyObject *old_value = PyCell_SwapTakeRef(cell, value, &result);
     Py_XDECREF(old_value);
+    return result;
 }
 
 // Gets the cell contents. Returns a new reference.

Include/internal/pycore_ceval.h

@@ -295,6 +295,7 @@ PyAPI_FUNC(int) _PyEval_ExceptionGroupMatch(_PyInterpreterFrame *, PyObject* exc
 PyAPI_FUNC(void) _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg);
 PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj);
 PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg);
+PyAPI_FUNC(void) _PyEval_FormatExcNotWriteable(PyThreadState *tstate, PyCodeObject *co, int oparg);
 PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs);
 PyAPI_FUNC(PyObject *) _PyEval_ImportFrom(PyThreadState *, PyObject *, PyObject *);
 PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *);

Include/internal/pycore_freelist_state.h

@@ -8,29 +8,33 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#define MAXFREELIST(x) x
+//#define MAXFREELIST(x) 0
+
+
 #  define PyTuple_MAXSAVESIZE 20     // Largest tuple to save on freelist
-#  define Py_tuple_MAXFREELIST 2000  // Maximum number of tuples of each size to save
-#  define Py_lists_MAXFREELIST 80
-#  define Py_list_iters_MAXFREELIST 10
-#  define Py_tuple_iters_MAXFREELIST 10
-#  define Py_dicts_MAXFREELIST 80
-#  define Py_dictkeys_MAXFREELIST 80
-#  define Py_floats_MAXFREELIST 100
-#  define Py_complexes_MAXFREELIST 100
-#  define Py_ints_MAXFREELIST 100
-#  define Py_slices_MAXFREELIST 1
-#  define Py_ranges_MAXFREELIST 6
-#  define Py_range_iters_MAXFREELIST 6
-#  define Py_contexts_MAXFREELIST 255
-#  define Py_async_gens_MAXFREELIST 80
-#  define Py_async_gen_asends_MAXFREELIST 80
-#  define Py_futureiters_MAXFREELIST 255
-#  define Py_object_stack_chunks_MAXFREELIST 4
-#  define Py_unicode_writers_MAXFREELIST 1
-#  define Py_bytes_writers_MAXFREELIST 1
-#  define Py_pycfunctionobject_MAXFREELIST 16
-#  define Py_pycmethodobject_MAXFREELIST 16
-#  define Py_pymethodobjects_MAXFREELIST 20
+#  define Py_tuple_MAXFREELIST MAXFREELIST(2000)  // Maximum number of tuples of each size to save
+#  define Py_lists_MAXFREELIST MAXFREELIST(80)
+#  define Py_list_iters_MAXFREELIST MAXFREELIST(10)
+#  define Py_tuple_iters_MAXFREELIST MAXFREELIST(10)
+#  define Py_dicts_MAXFREELIST MAXFREELIST(80)
+#  define Py_dictkeys_MAXFREELIST MAXFREELIST(80)
+#  define Py_floats_MAXFREELIST MAXFREELIST(100)
+#  define Py_complexes_MAXFREELIST MAXFREELIST(100)
+#  define Py_ints_MAXFREELIST MAXFREELIST(100)
+#  define Py_slices_MAXFREELIST MAXFREELIST(1)
+#  define Py_ranges_MAXFREELIST MAXFREELIST(6)
+#  define Py_range_iters_MAXFREELIST MAXFREELIST(6)
+#  define Py_contexts_MAXFREELIST MAXFREELIST(255)
+#  define Py_async_gens_MAXFREELIST MAXFREELIST(80)
+#  define Py_async_gen_asends_MAXFREELIST MAXFREELIST(80)
+#  define Py_futureiters_MAXFREELIST MAXFREELIST(255)
+#  define Py_object_stack_chunks_MAXFREELIST MAXFREELIST(4)
+#  define Py_unicode_writers_MAXFREELIST MAXFREELIST(1)
+#  define Py_bytes_writers_MAXFREELIST MAXFREELIST(1)
+#  define Py_pycfunctionobject_MAXFREELIST MAXFREELIST(16)
+#  define Py_pycmethodobject_MAXFREELIST MAXFREELIST(16)
+#  define Py_pymethodobjects_MAXFREELIST MAXFREELIST(20)
 
 // A generic freelist of either PyObjects or other data structures.
 struct _Py_freelist {

Include/internal/pycore_gc.h

@@ -205,6 +205,16 @@ static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) {
 #endif
 }
 
+static inline void _PyGC_CLEAR_COLLECTING(PyObject *op) {
+#ifdef Py_GIL_DISABLED
+    // TODO(immutable): Does NoGil have a collecting flag? If so, how do we
+    // clear it?
+#else
+    PyGC_Head *gc = _Py_AS_GC(op);
+    gc->_gc_prev &= ~_PyGC_PREV_MASK_COLLECTING;
+#endif
+}
+
 
 /* Tell the GC to track this object.
  *