Merge branch 'main' into pythongh-91896

.azure-pipelines/ci.yml

@@ -57,7 +57,7 @@ jobs:
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1q
+    openssl_version: 1.1.1t
 
   steps:
   - template: ./posix-steps.yml
@@ -83,7 +83,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1q
+    openssl_version: 1.1.1t
 
   steps:
   - template: ./posix-steps.yml

.azure-pipelines/pr.yml

@@ -57,7 +57,7 @@ jobs:
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1q
+    openssl_version: 1.1.1t
 
   steps:
   - template: ./posix-steps.yml
@@ -83,7 +83,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1q
+    openssl_version: 1.1.1t
 
   steps:
   - template: ./posix-steps.yml

.github/CODEOWNERS

@@ -151,6 +151,8 @@ Lib/ast.py                    @isidentical
 
 **/*sysconfig*                @FFY00
 
+**/*cjkcodecs*                @corona10
+
 # macOS
 /Mac/                         @python/macos-team
 **/*osx_support*              @python/macos-team

.github/workflows/build.yml

@@ -111,6 +111,9 @@ jobs:
         run: make smelly
       - name: Check limited ABI symbols
         run: make check-limited-abi
+      - name: Check for unsupported C global variables
+        if: github.event_name == 'pull_request'  # $GITHUB_EVENT_NAME
+        run: make check-c-globals
 
   build_win32:
     name: 'Windows (x86)'
@@ -154,15 +157,23 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
+      HOMEBREW_NO_ANALYTICS: 1
+      HOMEBREW_NO_AUTO_UPDATE: 1
+      HOMEBREW_NO_INSTALL_CLEANUP: 1
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v3
-    - name: Prepare homebrew environment variables
-      run: |
-        echo "LDFLAGS=-L$(brew --prefix tcl-tk)/lib" >> $GITHUB_ENV
-        echo "PKG_CONFIG_PATH=$(brew --prefix [email protected])/lib/pkgconfig:$(brew --prefix tcl-tk)/lib/pkgconfig" >> $GITHUB_ENV
+    - name: Install Homebrew dependencies
+      run: brew install pkg-config [email protected] xz gdbm tcl-tk
     - name: Configure CPython
-      run: ./configure --with-pydebug --prefix=/opt/python-dev
+      run: |
+        CFLAGS="-I$(brew --prefix gdbm)/include -I$(brew --prefix xz)/include" \
+        LDFLAGS="-L$(brew --prefix gdbm)/lib -I$(brew --prefix xz)/lib" \
+        PKG_CONFIG_PATH="$(brew --prefix tcl-tk)/lib/pkgconfig" \
+        ./configure \
+          --with-pydebug \
+          --prefix=/opt/python-dev \
+          --with-openssl="$(brew --prefix [email protected])"
     - name: Build CPython
       run: make -j4
     - name: Display build info
@@ -176,7 +187,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 1.1.1s
+      OPENSSL_VER: 1.1.1t
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v3
@@ -235,7 +246,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        openssl_ver: [1.1.1s, 3.0.7, 3.1.0-beta1]
+        openssl_ver: [1.1.1t, 3.0.8, 3.1.0-beta1]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
       MULTISSL_DIR: ${{ github.workspace }}/multissl
@@ -282,7 +293,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 1.1.1s
+      OPENSSL_VER: 1.1.1t
       PYTHONSTRICTEXTENSIONBUILD: 1
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:

.github/workflows/doc.yml

@@ -53,6 +53,30 @@ jobs:
     - name: 'Build HTML documentation'
       run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html
 
+    # Add pull request annotations for Sphinx nitpicks (missing references)
+    - name: 'Get list of changed files'
+      id: changed_files
+      uses: Ana06/[email protected]
+      with:
+        filter: "Doc/**"
+    - name: 'Build changed files in nit-picky mode'
+      continue-on-error: true
+      run: |
+        # Mark files the pull request modified
+        touch ${{ steps.changed_files.outputs.added_modified }}
+        # Build docs with the '-n' (nit-picky) option; convert warnings to annotations
+        make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n --keep-going" html 2>&1 |
+            python Doc/tools/warnings-to-gh-actions.py
+
+    # Ensure some files always pass Sphinx nit-picky mode (no missing references)
+    - name: 'Build known-good files in nit-picky mode'
+      run: |
+        # Mark files that must pass nit-picky
+        touch Doc/whatsnew/3.12.rst
+        touch Doc/library/sqlite3.rst
+        # Build docs with the '-n' (nit-picky) option, convert warnings to errors (-W)
+        make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n -W --keep-going" html 2>&1
+
   # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release
   doctest:
     name: 'Doctest'

.github/workflows/new-bugs-announce-notifier.yml

@@ -19,13 +19,13 @@ jobs:
       - name: Send notification
         uses: actions/github-script@v6
         env:
-          MAILGUN_API_KEY: ${{ secrets.PSF_MAILGUN_KEY }}
+          MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }}
         with:
           script: |
             const Mailgun = require("mailgun.js");
             const formData = require('form-data');
             const mailgun = new Mailgun(formData);
-            const DOMAIN = "mg.python.org";
+            const DOMAIN = "mailgun.python.org";
             const mg = mailgun.client({username: 'api', key: process.env.MAILGUN_API_KEY});
             github.rest.issues.get({
               issue_number: context.issue.number,
@@ -44,7 +44,7 @@ jobs:
               };
 
               const data = {
-                from: "CPython Issues <github@mg.python.org>",
+                from: "CPython Issues <github@mailgun.python.org>",
                 to: "[email protected]",
                 subject: `[Issue ${issue.data.number}] ${issue.data.title}`,
                 template: "new-github-issue",

Doc/bugs.rst

@@ -70,7 +70,7 @@ Click on the "New issue" button in the top bar to report a new issue.
 The submission form has two fields, "Title" and "Comment".
 
 For the "Title" field, enter a *very* short description of the problem;
-less than ten words is good.
+fewer than ten words is good.
 
 In the "Comment" field, describe the problem in detail, including what you
 expected to happen and what did happen.  Be sure to include whether any

Doc/c-api/buffer.rst

@@ -499,7 +499,7 @@ Buffer-related functions
    This function fails if *len* != *src->len*.
 
 
-.. c:function:: int PyObject_CopyData(Py_buffer *dest, Py_buffer *src)
+.. c:function:: int PyObject_CopyData(PyObject *dest, PyObject *src)
 
    Copy data from *src* to *dest* buffer. Can convert between C-style and
    or Fortran-style buffers.

Doc/c-api/code.rst

@@ -33,28 +33,47 @@ bound into a function.
 
    Return the number of free variables in *co*.
 
-.. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
+.. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
 
    Return a new code object.  If you need a dummy code object to create a frame,
-   use :c:func:`PyCode_NewEmpty` instead.  Calling :c:func:`PyCode_New` directly
-   will bind you to a precise Python version since the definition of the bytecode
-   changes often. The many arguments of this function are inter-dependent in complex
+   use :c:func:`PyCode_NewEmpty` instead.
+
+   Since the definition of the bytecode changes often, calling
+   :c:func:`PyCode_New` directly can bind you to a precise Python version.
+
+   The many arguments of this function are inter-dependent in complex
    ways, meaning that subtle changes to values are likely to result in incorrect
    execution or VM crashes. Use this function only with extreme care.
 
    .. versionchanged:: 3.11
       Added ``exceptiontable`` parameter.
 
-.. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
+   .. index:: single: PyCode_New
+
+   .. versionchanged:: 3.12
+
+      Renamed from ``PyCode_New`` as part of :ref:`unstable-c-api`.
+      The old name is deprecated, but will remain available until the
+      signature changes again.
+
+.. c:function:: PyCodeObject* PyUnstable_Code_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
 
    Similar to :c:func:`PyCode_New`, but with an extra "posonlyargcount" for positional-only arguments.
    The same caveats that apply to ``PyCode_New`` also apply to this function.
 
-   .. versionadded:: 3.8
+   .. index:: single: PyCode_NewWithPosOnlyArgs
+
+   .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs``
 
    .. versionchanged:: 3.11
       Added ``exceptiontable`` parameter.
 
+   .. versionchanged:: 3.12
+
+      Renamed to ``PyUnstable_Code_NewWithPosOnlyArgs``.
+      The old name is deprecated, but will remain available until the
+      signature changes again.
+
 .. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
 
    Return a new empty code object with the specified filename,
@@ -153,15 +172,93 @@ bound into a function.
    before the destruction of *co* takes place, so the prior state of *co*
    can be inspected.
 
+   If *event* is ``PY_CODE_EVENT_DESTROY``, taking a reference in the callback
+   to the about-to-be-destroyed code object will resurrect it and prevent it
+   from being freed at this time. When the resurrected object is destroyed
+   later, any watcher callbacks active at that time will be called again.
+
    Users of this API should not rely on internal runtime implementation
    details. Such details may include, but are not limited to, the exact
    order and timing of creation and destruction of code objects. While
    changes in these details may result in differences observable by watchers
    (including whether a callback is invoked or not), it does not change
    the semantics of the Python code being executed.
 
-   If the callback returns with an exception set, it must return ``-1``; this
-   exception will be printed as an unraisable exception using
-   :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``.
+   If the callback sets an exception, it must return ``-1``; this exception will
+   be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`.
+   Otherwise it should return ``0``.
+
+   There may already be a pending exception set on entry to the callback. In
+   this case, the callback should return ``0`` with the same exception still
+   set. This means the callback may not call any other API that can set an
+   exception unless it saves and clears the exception state first, and restores
+   it before returning.
 
    .. versionadded:: 3.12
+
+
+Extra information
+-----------------
+
+To support low-level extensions to frame evaluation, such as external
+just-in-time compilers, it is possible to attach arbitrary extra data to
+code objects.
+
+These functions are part of the unstable C API tier:
+this functionality is a CPython implementation detail, and the API
+may change without deprecation warnings.
+
+.. c:function:: Py_ssize_t PyUnstable_Eval_RequestCodeExtraIndex(freefunc free)
+
+   Return a new an opaque index value used to adding data to code objects.
+
+   You generally call this function once (per interpreter) and use the result
+   with ``PyCode_GetExtra`` and ``PyCode_SetExtra`` to manipulate
+   data on individual code objects.
+
+   If *free* is not ``NULL``: when a code object is deallocated,
+   *free* will be called on non-``NULL`` data stored under the new index.
+   Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`.
+
+   .. index:: single: _PyEval_RequestCodeExtraIndex
+
+   .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex``
+
+   .. versionchanged:: 3.12
+
+     Renamed to ``PyUnstable_Eval_RequestCodeExtraIndex``.
+     The old private name is deprecated, but will be available until the API
+     changes.
+
+.. c:function:: int PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra)
+
+   Set *extra* to the extra data stored under the given index.
+   Return 0 on success. Set an exception and return -1 on failure.
+
+   If no data was set under the index, set *extra* to ``NULL`` and return
+   0 without setting an exception.
+
+   .. index:: single: _PyCode_GetExtra
+
+   .. versionadded:: 3.6 as ``_PyCode_GetExtra``
+
+   .. versionchanged:: 3.12
+
+     Renamed to ``PyUnstable_Code_GetExtra``.
+     The old private name is deprecated, but will be available until the API
+     changes.
+
+.. c:function:: int PyUnstable_Code_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
+
+   Set the extra data stored under the given index to *extra*.
+   Return 0 on success. Set an exception and return -1 on failure.
+
+   .. index:: single: _PyCode_SetExtra
+
+   .. versionadded:: 3.6 as ``_PyCode_SetExtra``
+
+   .. versionchanged:: 3.12
+
+     Renamed to ``PyUnstable_Code_SetExtra``.
+     The old private name is deprecated, but will be available until the API
+     changes.

Doc/c-api/dict.rst

@@ -80,7 +80,7 @@ Dictionary Objects
 
 .. c:function:: int PyDict_DelItem(PyObject *p, PyObject *key)
 
-   Remove the entry in dictionary *p* with key *key*. *key* must be hashable;
+   Remove the entry in dictionary *p* with key *key*. *key* must be :term:`hashable`;
    if it isn't, :exc:`TypeError` is raised.
    If *key* is not in the dictionary, :exc:`KeyError` is raised.
    Return ``0`` on success or ``-1`` on failure.
@@ -298,13 +298,26 @@ Dictionary Objects
    dictionary.
 
    The callback may inspect but must not modify *dict*; doing so could have
-   unpredictable effects, including infinite recursion.
+   unpredictable effects, including infinite recursion. Do not trigger Python
+   code execution in the callback, as it could modify the dict as a side effect.
+
+   If *event* is ``PyDict_EVENT_DEALLOCATED``, taking a new reference in the
+   callback to the about-to-be-destroyed dictionary will resurrect it and
+   prevent it from being freed at this time. When the resurrected object is
+   destroyed later, any watcher callbacks active at that time will be called
+   again.
 
    Callbacks occur before the notified modification to *dict* takes place, so
    the prior state of *dict* can be inspected.
 
-   If the callback returns with an exception set, it must return ``-1``; this
-   exception will be printed as an unraisable exception using
-   :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``.
+   If the callback sets an exception, it must return ``-1``; this exception will
+   be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`.
+   Otherwise it should return ``0``.
+
+   There may already be a pending exception set on entry to the callback. In
+   this case, the callback should return ``0`` with the same exception still
+   set. This means the callback may not call any other API that can set an
+   exception unless it saves and clears the exception state first, and restores
+   it before returning.
 
    .. versionadded:: 3.12