From 1e170cd238b417f056e387af933abb48d55e7c61 Mon Sep 17 00:00:00 2001 From: Nick Bull Date: Fri, 27 Mar 2026 17:36:15 +0000 Subject: [PATCH] [#3] Download dev headers if not present --- python/setup.py | 10 ++ .../extension/download_python_headers.py | 140 ++++++++++++++++++ python/src/ubeacon/extension/ubeacon.py | 121 ++++++++++----- 3 files changed, 235 insertions(+), 36 deletions(-) create mode 100644 python/src/ubeacon/extension/download_python_headers.py diff --git a/python/setup.py b/python/setup.py index 05c2ca3..4a516eb 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,6 +1,15 @@ # setup.py +import os + from setuptools import Extension, setup +try: + include_dirs = os.environ["PYTHON_INCLUDE_DIRS"].split(os.pathsep) +except KeyError: + raise RuntimeError( + "PYTHON_INCLUDE_DIRS environment variable not set." + ) + module = Extension( "ubeacon", sources=[ @@ -10,6 +19,7 @@ "src/ext/cJSON/cJSON.c", ], extra_compile_args=["-O0", "-Isrc/ext/cJSON", "-std=c99"], + include_dirs=include_dirs, ) setup( diff --git a/python/src/ubeacon/extension/download_python_headers.py b/python/src/ubeacon/extension/download_python_headers.py new file mode 100644 index 0000000..0ea20c2 --- /dev/null +++ b/python/src/ubeacon/extension/download_python_headers.py @@ -0,0 +1,140 @@ +""" +Download Python development headers for a given version into a temporary directory. + +Works without root on Ubuntu, RHEL, Fedora, and OpenSUSE by downloading (not installing) +the appropriate package and extracting it locally. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import distro + + +def _run_command(command: list[str], **kwargs: object) -> subprocess.CompletedProcess: + """ + Run a command with subprocess.run and check for errors. + + command: The command to run, as a list of arguments. + kwargs: Additional keyword arguments to pass to subprocess.run. + + Returns the CompletedProcess object returned by subprocess.run. + """ + return subprocess.run(command, check=True, capture_output=True, text=True, **kwargs) + + +def _get_download_package_uri(version: str) -> str: + """ + Return the URL of a distribution package file. + """ + + match distro.id(): + case dist if dist in {"ubuntu", "debian"}: + + return ( + _run_command( + ["apt-get", "--print-uris", "download", f"libpython{version}-dev"] + ) + .stdout.splitlines()[0] + .split()[0] + .strip("'") + ) + + case dist if dist in {"fedora", "rhel", "rocky", "centos", "amzn"}: + + def dnf(package_name: str) -> str: + # TODO support on ARM + return _run_command( + [ + "dnf", + "repoquery", + "--location", + package_name, + "--archlist", + "x86_64", + ] + ).stdout.strip() + + rpms = dnf(f"python{version}-devel") + if not rpms: + # The package may be named python3-devel if it's the default + # Python version for the current distro version. + rpms = dnf("python3-devel") + if version not in rpms: + raise ValueError( + f"Could not find a suitable python-devel package for version {version} in dnf output: {rpms}" + ) + return rpms.splitlines()[ + 0 + ] # Take the first result if there are multiple matches. + + case _: + raise ValueError(f"Unsupported distribution: {distro.id()}") + + +def _extract_deb(package_path: Path, extract_dir: Path) -> None: + """Extract a .deb package into the given directory.""" + _run_command(["dpkg-deb", "-x", str(package_path), str(extract_dir)]) + + +def _extract_rpm(package_path: Path, extract_dir: Path) -> None: + """Extract a .rpm package into the given directory.""" + with subprocess.Popen( + ["rpm2cpio", str(package_path)], stdout=subprocess.PIPE + ) as rpm2cpio: + _run_command(["cpio", "-idm"], stdin=rpm2cpio.stdout, cwd=extract_dir) + rpm2cpio.wait() + if rpm2cpio.returncode != 0: + raise subprocess.CalledProcessError( + rpm2cpio.returncode, ["rpm2cpio", str(package_path)] + ) + + +def python_dev_headers( + version: str, storage_dir: Path, uri_override: str | None = None +) -> Path: + """ + Download the appropriate python-dev/python-devel package for the current Linux + distribution if necessary, extract it, and return the path to the + extracted headers. + + Args: + version: The Python version string, e.g. "3.11". + storage_dir: The directory to use for downloading and extracting packages. + uri_override: If provided, this URI will be used instead of determining + the package URL based on the distribution. This is intended for testing. + + Returns: + The root of the unpacked headers. + + Raises: + ValueError: If the current distribution is not supported. + subprocess.CalledProcessError: If downloading or extracting the package fails. + """ + download_dir = storage_dir / "packages" + download_dir.mkdir(exist_ok=True) + + uri = uri_override or _get_download_package_uri(version) + + # Fetch the package file to the download directory. + package_path = download_dir / uri.split("/")[-1] + if not package_path.exists(): + _run_command(["curl", "--fail","-L", "-o", str(package_path), uri]) + + extract_dir = storage_dir / (package_path.stem + "_extracted") + if not extract_dir.exists(): + if package_path.suffix == ".deb": + extract = _extract_deb + elif package_path.suffix == ".rpm": + extract = _extract_rpm + else: + raise ValueError( + f"""Unknown package format: {package_path.suffix}. Expected .deb or .rpm.""" + ) + + extract_dir.mkdir(exist_ok=True) + extract(package_path, extract_dir) + + return extract_dir diff --git a/python/src/ubeacon/extension/ubeacon.py b/python/src/ubeacon/extension/ubeacon.py index ec69a69..fe7688b 100644 --- a/python/src/ubeacon/extension/ubeacon.py +++ b/python/src/ubeacon/extension/ubeacon.py @@ -12,6 +12,7 @@ import functools import json import os +import re import subprocess import tempfile from pathlib import Path @@ -23,9 +24,11 @@ import pygments.formatters import pygments.lexers from src.udbpy import locations, report # pyright: ignore[reportMissingModuleSource] -from src.udbpy.gdb_extensions import gdbutils # pyright: ignore[reportMissingModuleSource] +from src.udbpy.gdb_extensions import ( + gdbutils, +) # pyright: ignore[reportMissingModuleSource] -from . import debuggee, messages +from . import debuggee, download_python_headers, messages PREFIX = "s_ubeacon" STATE_STRUCT = PREFIX @@ -36,6 +39,84 @@ EXCEPTION_FN = f"{TRACE_PREFIX}_exception" +def wrap_build(python_executable: str, cache_dir: Path, addon_root: Path) -> None: + """Build the UBeacon library using the specified Python executable. + + If the Python development headers are not available on the system, they are + downloaded using the distro's package manager. + + python_executable: + The path to the Python executable to build against. + cache_dir: + The directory to use for caching build artifacts and downloaded headers. + addon_root: + The root directory of the addon, where `setup.py` is located. + """ + + version_string = subprocess.check_output( + [ + python_executable, + "-c", + "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}')", + ], + text=True, + cwd=addon_root, + stderr=subprocess.STDOUT, + ).strip() + + # Does this Python executable have development headers installed already? + include_paths = subprocess.check_output( + [ + python_executable, + "-c", + "import sysconfig; print(sysconfig.get_path('include'))", + ], + text=True, + ).strip() + + if not Path(include_paths).exists(): + # Fetch headers from a distro package + header_dir = download_python_headers.python_dev_headers( + version_string, storage_dir=cache_dir + ) + include_paths = f"{header_dir}/usr/include:{header_dir}/usr/include/python{version_string}" + + try: + subprocess.run( + [ + python_executable, + "setup.py", + "build", + "--quiet", + f"--build-base={cache_dir}", + ], + text=True, + cwd=addon_root, + check=True, + capture_output=True, + env={**os.environ, "PYTHON_INCLUDE_DIRS": include_paths},) + + except subprocess.CalledProcessError as exc: + output = "" + if exc.output: + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf: + tf.write(exc.output) + tf.flush() + output += f"Saved stdout to {tf.name}.\n" + if exc.stderr: + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf: + tf.write(exc.stderr) + tf.flush() + output += f"Saved stderr to {tf.name}.\n" + raise report.ReportableError( + f"""Error occurred in Python: could not debug this version of Python. + + You may need to install Python development headers for this version. + + {output}""" + ) + + @functools.cache def build() -> Path: """ @@ -81,40 +162,8 @@ def build() -> Path: return lib_path # Not found, build it - try: - subprocess.run( - [ - python_executable, - "setup.py", - "build", - "--quiet", - f"--build-base={cache_dir}", - ], - text=True, - cwd=root, - check=True, - capture_output=True, - ) - except subprocess.CalledProcessError as exc: - if exc.output: - with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf: - tf.write(exc.output) - tf.flush() - report.user( - f"Saved stdout to {tf.name}.\n" - ) - if exc.stderr: - with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tf: - tf.write(exc.stderr) - tf.flush() - report.user( - f"Saved stderr to {tf.name}.\n" - ) - raise report.ReportableError( - """Error occurred in Python: could not debug this version of Python. - - You may need to install Python development headers for this version.""" - ) + wrap_build(python_executable, cache_dir, root) + output = subprocess.check_output( [python_executable, "find_so.py", cache_dir], text=True, cwd=root )