Add an option to create arch specific, python version independent pkgs (

#5456) * Add an option to create arch specific, python version independent pkgs * Fix link_json * Fix multi-output recipes * Add a test * Add docs * update test * fix test * fix test again * fix windows * fix docs * add news * use walrus operator Co-authored-by: jaimergp <[email protected]> * doc improvements Co-authored-by: Jannis Leidel <[email protected]> Co-authored-by: jaimergp <[email protected]> * Update news Co-authored-by: jaimergp <[email protected]> * clarify noarch: python * clarify more * noarch_python --------- Co-authored-by: jaimergp <[email protected]> Co-authored-by: Jannis Leidel <[email protected]>
conda · Feb 25, 2025 · 6e6bb13 · 6e6bb13
1 parent 0325185
commit 6e6bb13
Show file tree

Hide file tree

Showing 9 changed files with 162 additions and 8 deletions.
diff --git a/conda_build/build.py b/conda_build/build.py
@@ -1247,8 +1247,11 @@ def write_info_files_file(m, files):
 def write_link_json(m):
     package_metadata = OrderedDict()
     noarch_type = m.get_value("build/noarch")
-    if noarch_type:
-        noarch_type_str = str(noarch_type)
+    if noarch_type or m.python_version_independent:
+        if noarch_type:
+            noarch_type_str = str(noarch_type)
+        elif m.python_version_independent:
+            noarch_type_str = "python"
         noarch_dict = OrderedDict(type=noarch_type_str)
         if noarch_type_str.lower() == "python":
             entry_points = m.get_value("build/entry_points")
@@ -1441,13 +1444,14 @@ def create_info_files(m, replacements, files, prefix):
 
 
 def get_short_path(m, target_file):
+    if m.python_version_independent:
+        if (site_packages_idx := target_file.find("site-packages")) >= 0:
+            return target_file[site_packages_idx:]
     if m.noarch == "python":
         entry_point_script_names = get_entry_point_script_names(
             m.get_value("build/entry_points")
         )
-        if target_file.find("site-packages") >= 0:
-            return target_file[target_file.find("site-packages") :]
-        elif target_file.startswith("bin") and (
+        if target_file.startswith("bin") and (
             target_file not in entry_point_script_names
         ):
             return target_file.replace("bin", "python-scripts")
@@ -1665,6 +1669,9 @@ def post_process_files(m: MetaData, initial_prefix_files):
         noarch_python.populate_files(
             m, pkg_files, host_prefix, entry_point_script_names
         )
+    elif m.python_version_independent:
+        # For non noarch: python ones, we don't need to handle entry points in a special way.
+        noarch_python.populate_files(m, pkg_files, host_prefix, [])
 
     current_prefix_files = utils.prefix_files(prefix=host_prefix)
     new_files = current_prefix_files - initial_prefix_files
@@ -3036,7 +3043,7 @@ def _set_env_variables_for_build(m, env):
     # locally, and if we don't, it's a problem.
     env["PIP_NO_INDEX"] = True
 
-    if m.noarch == "python":
+    if m.python_version_independent:
         env["PYTHONDONTWRITEBYTECODE"] = True
 
     # The stuff in replacements is not parsable in a shell script (or we need to escape it)

diff --git a/conda_build/metadata.py b/conda_build/metadata.py
@@ -608,6 +608,7 @@ def parse(data, config, path=None):
         "script": list,
         "noarch": str,
         "noarch_python": bool,
+        "python_version_independent": bool,
         "has_prefix_files": None,
         "binary_has_prefix_files": None,
         "ignore_prefix_files": None,
@@ -1855,6 +1856,12 @@ def info_index(self):
             build_noarch = self.get_value("build/noarch")
             if build_noarch:
                 d["noarch"] = build_noarch
+        elif self.python_version_independent:
+            # This is required by CEP 20 (https://github.com/conda/ceps/blob/main/cep-0020.md)
+            # to make current mamba/micromamba compile the pure python files
+            # and for micromamba to move the files in site-packages to the correct dir.
+            # (i.e. we need A2 action to apply actions B1-B4 as mentioned in CEP 20)
+            d["noarch"] = "python"
         if self.is_app():
             d.update(self.app_meta())
         return d
@@ -2337,6 +2344,18 @@ def copy(self: Self) -> MetaData:
         )
         return new
 
+    @property
+    def python_version_independent(self) -> bool:
+        return (
+            self.get_value("build/python_version_independent")
+            or self.get_value("build/noarch") == "python"
+            or self.noarch_python
+        )
+
+    @python_version_independent.setter
+    def python_version_independent(self, value: bool) -> None:
+        self.meta.setdefault("build", {})["python_version_independent"] = bool(value)
+
     @property
     def noarch(self):
         return self.get_value("build/noarch")
@@ -2494,6 +2513,11 @@ def get_output_metadata(self, output):
             output_metadata.final = False
             output_metadata.noarch = output.get("noarch", False)
             output_metadata.noarch_python = output.get("noarch_python", False)
+            output_metadata.python_version_independent = (
+                output.get("python_version_independent")
+                or output_metadata.noarch == "python"
+                or output_metadata.noarch_python
+            )
             # primarily for tests - make sure that we keep the platform consistent (setting noarch
             #      would reset it)
             if (

diff --git a/conda_build/render.py b/conda_build/render.py
@@ -213,7 +213,7 @@ def get_pin_from_build(m, dep, build_dep_versions):
     if (
         version
         and dep_name in m.config.variant.get("pin_run_as_build", {})
-        and not (dep_name == "python" and (m.noarch or m.noarch_python))
+        and not (dep_name == "python" and m.python_version_independent)
         and dep_name in build_dep_versions
     ):
         pin_cfg = m.config.variant["pin_run_as_build"][dep_name]
@@ -410,6 +410,8 @@ def get_upstream_pins(m: MetaData, precs, env):
     precs = [prec for prec in precs if prec.name in explicit_specs]
 
     ignore_pkgs_list = utils.ensure_list(m.get_value("build/ignore_run_exports_from"))
+    if m.python_version_independent and not m.noarch:
+        ignore_pkgs_list.append("python")
     ignore_list = utils.ensure_list(m.get_value("build/ignore_run_exports"))
     additional_specs = {}
     for prec in precs:

diff --git a/conda_build/utils.py b/conda_build/utils.py
@@ -1060,6 +1060,7 @@ def iter_entry_points(items):
 
 
 def create_entry_point(path, module, func, config):
+    """Creates an entry point for legacy noarch_python builds"""
     import_name = func.split(".")[0]
     pyscript = PY_TMPL % {"module": module, "func": func, "import_name": import_name}
     if on_win:
@@ -1083,6 +1084,7 @@ def create_entry_point(path, module, func, config):
 
 
 def create_entry_points(items, config):
+    """Creates entry points for legacy noarch_python builds"""
     if not items:
         return
     bin_dir = join(config.host_prefix, bin_dirname)

diff --git a/conda_build/windows.py b/conda_build/windows.py
@@ -246,7 +246,7 @@ def build_vcvarsall_cmd(cmd, arch=arch_selector):
 
 def write_build_scripts(m, env, bld_bat):
     env_script = join(m.config.work_dir, "build_env_setup.bat")
-    if m.noarch == "python":
+    if m.python_version_independent:
         env["PYTHONDONTWRITEBYTECODE"] = True
     import codecs
 

diff --git a/docs/source/resources/define-metadata.rst b/docs/source/resources/define-metadata.rst
@@ -701,6 +701,61 @@ conda >=4.3 to install.
    it was built, which probably will result in incorrect/incomplete installation in other
    platforms.
 
+Python version independent packages
+-----------------------------------
+
+Allows you to specify "no python version" when building a Python
+package thus making it compatible with a user specified range of Python
+versions. Main use-case for this is to create ABI3 packages as specified
+in [CEP 20](https://github.com/conda/ceps/blob/main/cep-0020.md).
+
+ABI3 packages support building a native Python extension using a
+specific Python version and running it against any later Python version.
+ABI3 or stable ABI is supported by only CPython - the reference Python
+implementation with the Global Interpreter Lock (GIL) enabled. Therefore
+package builders who wishes to support the free-threaded python build
+or another implementation like PyPy still has to build a conda package
+specific to that ABI as they don't support ABI3. There are other
+proposed standards like HPy and ABI4 (work-in-progress) that tries
+to address all python implementations.
+
+conda-build can indicate that a conda package works for any python version
+by adding
+
+.. code-block:: yaml
+
+   build:
+     python_version_independent: true
+
+A package builder also has to indicate which standard is supported by
+the package, i.e., for ABI3,
+
+.. code-block:: yaml
+
+   requirements:
+     host:
+       - python-abi3
+       - python
+     run:
+       - python
+
+
+In order to support ABI3 with Python 3.9 and onwards and
+free-threaded builds you can do
+
+.. code-block:: yaml
+   build:
+     python_version_independent: true   # [py == 39]
+     skip: true                         # [py > 39 and not python.endswith("t")]
+
+   requirements:
+     host:
+       - python-abi3                    # [py == 39]
+       - python
+     run:
+       - python
+
+
 Include build recipe
 --------------------
 

diff --git a/news/5456-python-version-independent.rst b/news/5456-python-version-independent.rst
@@ -0,0 +1,21 @@
+### Enhancements
+
+* Added an option `build.python_version_independent` to recipes to support
+  building ABI3 for one CPython version and using the package in any
+  later version. (#5456)
+
+### Bug fixes
+
+* <news item>
+
+### Deprecations
+
+* <news item>
+
+### Docs
+
+* <news item>
+
+### Other
+
+* <news item>
diff --git a/tests/test-recipes/metadata/_python_version_independent/meta.yaml b/tests/test-recipes/metadata/_python_version_independent/meta.yaml
@@ -0,0 +1,28 @@
+package:
+  name: python_version_independent_test_package
+  version: "1.0"
+
+source:
+  path: ../_noarch_python_with_tests/noarch_python_test_package
+
+build:
+  script: python setup.py install --single-version-externally-managed --record=record.txt
+  python_version_independent: true
+  entry_points:
+    - noarch_python_test_package_script = noarch_python_test_package:main
+
+requirements:
+  build:
+  host:
+    - python 3.11.*
+    - setuptools
+  run:
+    - python >=3.11
+
+test:
+  requires:
+    - python 3.12.*
+  imports:
+    - noarch_python_test_package
+  commands:
+    - noarch_python_test_package_script
diff --git a/tests/test_api_build.py b/tests/test_api_build.py
@@ -146,6 +146,21 @@ def test_recipe_builds(
     api.build(str(recipe), config=testing_config)
 
 
+@pytest.mark.slow
+@pytest.mark.serial
+def test_python_version_independent(
+    testing_config,
+    monkeypatch: pytest.MonkeyPatch,
+):
+    recipe = os.path.join(metadata_dir, "_python_version_independent")
+    testing_config.activate = True
+    monkeypatch.setenv("CONDA_TEST_VAR", "conda_test")
+    monkeypatch.setenv("CONDA_TEST_VAR_2", "conda_test_2")
+    output = api.build(str(recipe), config=testing_config)[0]
+    subdir = os.path.basename(os.path.dirname(output))
+    assert subdir != "noarch"
+
+
 @pytest.mark.serial
 @pytest.mark.skipif(
     "CI" in os.environ and "GITHUB_WORKFLOW" in os.environ,