[#3] Download dev headers if not present

python/setup.py

@@ -1,6 +1,15 @@
 # setup.py
+import os
+
 from setuptools import Extension, setup
 
+try:
+    include_dirs = os.environ["PYTHON_INCLUDE_DIRS"].split(os.pathsep)
+except KeyError:
+    raise RuntimeError(
+        "PYTHON_INCLUDE_DIRS environment variable not set."
+    )
+
 module = Extension(
     "ubeacon",
     sources=[
@@ -10,6 +19,7 @@
         "src/ext/cJSON/cJSON.c",
     ],
     extra_compile_args=["-O0", "-Isrc/ext/cJSON", "-std=c99"],
+    include_dirs=include_dirs,
 )
 
 setup(

python/src/ubeacon/extension/download_python_headers.py

@@ -0,0 +1,140 @@
+"""
+Download Python development headers for a given version into a temporary directory.
+
+Works without root on Ubuntu, RHEL, Fedora, and OpenSUSE by downloading (not installing)
+the appropriate package and extracting it locally.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+import distro
+
+
+def _run_command(command: list[str], **kwargs: object) -> subprocess.CompletedProcess:
+    """
+    Run a command with subprocess.run and check for errors.
+
+    command: The command to run, as a list of arguments.
+    kwargs: Additional keyword arguments to pass to subprocess.run.
+
+    Returns the CompletedProcess object returned by subprocess.run.
+    """
+    return subprocess.run(command, check=True, capture_output=True, text=True, **kwargs)
+
+
+def _get_download_package_uri(version: str) -> str:
+    """
+    Return the URL of a distribution package file.
+    """
+
+    match distro.id():
+        case dist if dist in {"ubuntu", "debian"}:
+
+            return (
+                _run_command(
+                    ["apt-get", "--print-uris", "download", f"libpython{version}-dev"]
+                )
+                .stdout.splitlines()[0]
+                .split()[0]
+                .strip("'")
+            )
+
+        case dist if dist in {"fedora", "rhel", "rocky", "centos", "amzn"}:
+
+            def dnf(package_name: str) -> str:
+                # TODO support on ARM
+                return _run_command(
+                    [
+                        "dnf",
+                        "repoquery",
+                        "--location",
+                        package_name,
+                        "--archlist",
+                        "x86_64",
+                    ]
+                ).stdout.strip()
+
+            rpms = dnf(f"python{version}-devel")
+            if not rpms:
+                # The package may be named python3-devel if it's the default
+                # Python version for the current distro version.
+                rpms = dnf("python3-devel")
+                if version not in rpms:
+                    raise ValueError(
+                        f"Could not find a suitable python-devel package for version {version} in dnf output: {rpms}"
+                    )
+            return rpms.splitlines()[
+                0
+            ]  # Take the first result if there are multiple matches.
+
+        case _:
+            raise ValueError(f"Unsupported distribution: {distro.id()}")
+
+
+def _extract_deb(package_path: Path, extract_dir: Path) -> None:
+    """Extract a .deb package into the given directory."""
+    _run_command(["dpkg-deb", "-x", str(package_path), str(extract_dir)])
+
+
+def _extract_rpm(package_path: Path, extract_dir: Path) -> None:
+    """Extract a .rpm package into the given directory."""
+    with subprocess.Popen(
+        ["rpm2cpio", str(package_path)], stdout=subprocess.PIPE
+    ) as rpm2cpio:
+        _run_command(["cpio", "-idm"], stdin=rpm2cpio.stdout, cwd=extract_dir)
+        rpm2cpio.wait()
+        if rpm2cpio.returncode != 0:
+            raise subprocess.CalledProcessError(
+                rpm2cpio.returncode, ["rpm2cpio", str(package_path)]
+            )
+
+
+def python_dev_headers(
+    version: str, storage_dir: Path, uri_override: str | None = None
+) -> Path:
+    """
+    Download the appropriate python-dev/python-devel package for the current Linux
+    distribution if necessary, extract it, and return the path to the
+    extracted headers.
+
+    Args:
+        version: The Python version string, e.g. "3.11".
+        storage_dir: The directory to use for downloading and extracting packages.
+        uri_override: If provided, this URI will be used instead of determining
+        the package URL based on the distribution. This is intended for testing.
+
+    Returns:
+        The root of the unpacked headers.
+
+    Raises:
+        ValueError: If the current distribution is not supported.
+        subprocess.CalledProcessError: If downloading or extracting the package fails.
+    """
+    download_dir = storage_dir / "packages"
+    download_dir.mkdir(exist_ok=True)
+
+    uri = uri_override or _get_download_package_uri(version)
+
+    # Fetch the package file to the download directory.
+    package_path = download_dir / uri.split("/")[-1]
+    if not package_path.exists():
+        _run_command(["curl", "--fail","-L", "-o", str(package_path), uri])
+
+    extract_dir = storage_dir / (package_path.stem + "_extracted")
+    if not extract_dir.exists():
+        if package_path.suffix == ".deb":
+            extract = _extract_deb
+        elif package_path.suffix == ".rpm":
+            extract = _extract_rpm
+        else:
+            raise ValueError(
+                f"""Unknown package format: {package_path.suffix}. Expected .deb or .rpm."""
+            )
+
+        extract_dir.mkdir(exist_ok=True)
+        extract(package_path, extract_dir)
+
+    return extract_dir

python/src/ubeacon/extension/ubeacon.py

@@ -12,6 +12,7 @@
 import functools
 import json
 import os
+import re
 import subprocess
 import tempfile
 from pathlib import Path
@@ -23,9 +24,11 @@
 import pygments.formatters
 import pygments.lexers
 from src.udbpy import locations, report  # pyright: ignore[reportMissingModuleSource]
-from src.udbpy.gdb_extensions import gdbutils  # pyright: ignore[reportMissingModuleSource]
+from src.udbpy.gdb_extensions import (
+    gdbutils,
+)  # pyright: ignore[reportMissingModuleSource]
 
-from . import debuggee, messages
+from . import debuggee, download_python_headers, messages
 
 PREFIX = "s_ubeacon"
 STATE_STRUCT = PREFIX
@@ -36,6 +39,84 @@
 EXCEPTION_FN = f"{TRACE_PREFIX}_exception"
 
 
+def wrap_build(python_executable: str, cache_dir: Path, addon_root: Path) -> None:
+    """Build the UBeacon library using the specified Python executable.
+
+    If the Python development headers are not available on the system, they are
+    downloaded using the distro's package manager.   
+
+    python_executable:
+        The path to the Python executable to build against.
+    cache_dir:
+        The directory to use for caching build artifacts and downloaded headers.
+    addon_root:
+        The root directory of the addon, where `setup.py` is located.
+    """
+
+    version_string = subprocess.check_output(
+        [
+            python_executable,
+            "-c",
+            "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}')",
+        ],
+        text=True,
+        cwd=addon_root,
+        stderr=subprocess.STDOUT,
+    ).strip()
+
+    # Does this Python executable have development headers installed already?
+    include_paths = subprocess.check_output(
+        [
+            python_executable,
+            "-c",
+            "import sysconfig; print(sysconfig.get_path('include'))",
+        ],
+        text=True,
+    ).strip()
+
+    if not Path(include_paths).exists():
+        # Fetch headers from a distro package
+        header_dir = download_python_headers.python_dev_headers(
+            version_string, storage_dir=cache_dir
+        )
+        include_paths = f"{header_dir}/usr/include:{header_dir}/usr/include/python{version_string}"
+
+    try:
+        subprocess.run(
+            [
+                python_executable,
+                "setup.py",
+                "build",
+                "--quiet",
+                f"--build-base={cache_dir}",
+            ],
+            text=True,
+            cwd=addon_root,
+            check=True,
+            capture_output=True,
+            env={**os.environ, "PYTHON_INCLUDE_DIRS": include_paths},)
+
+    except subprocess.CalledProcessError as exc:
+        output = ""
+        if exc.output:
+            with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf:
+                tf.write(exc.output)
+                tf.flush()
+                output += f"Saved stdout to {tf.name}.\n"
+        if exc.stderr:
+            with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf:
+                tf.write(exc.stderr)
+                tf.flush()
+                output += f"Saved stderr to {tf.name}.\n"
+        raise report.ReportableError(
+            f"""Error occurred in Python: could not debug this version of Python.
+
+            You may need to install Python development headers for this version.
+
+            {output}"""
+        )
+
+
 @functools.cache
 def build() -> Path:
     """
@@ -81,40 +162,8 @@ def build() -> Path:
                 return lib_path
 
     # Not found, build it
-    try:
-        subprocess.run(
-            [
-                python_executable,
-                "setup.py",
-                "build",
-                "--quiet",
-                f"--build-base={cache_dir}",
-            ],
-            text=True,
-            cwd=root,
-            check=True,
-            capture_output=True,
-        )
-    except subprocess.CalledProcessError as exc:
-        if exc.output:
-            with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf:
-                tf.write(exc.output)
-                tf.flush()
-                report.user(
-                    f"Saved stdout to {tf.name}.\n"
-                )
-        if exc.stderr:
-            with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf:
-                tf.write(exc.stderr)
-                tf.flush()
-                report.user(
-                    f"Saved stderr to {tf.name}.\n"
-                )
-        raise report.ReportableError(
-            """Error occurred in Python: could not debug this version of Python.
-
-            You may need to install Python development headers for this version."""
-        )
+    wrap_build(python_executable, cache_dir, root)
+
     output = subprocess.check_output(
         [python_executable, "find_so.py", cache_dir], text=True, cwd=root
     )