From 5c69c2147aa67946a232d99fcad7043482e0ede4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 11 Mar 2026 20:25:28 +0000 Subject: [PATCH 1/3] Add an "upgrade from previous" test We have tests that do upgrades, but they start from the *new* bootc. Add a `just`+CI workflow that starts from a stable shipped image (we just need to inject tmt deps + nu). The readonly test then gains a helper which optionally performs an upgrade to the new target. IOW the flow is - deploy stock image with unmodified bootc etc - upgrade - run readonly tests It could of course make sense to run *all* of the tests this way as an optional thing, nothing blocks that, but it would be *another* entry in our matrix and we're going to need to figure out how to wrangle that matrix size. Also do to that we'd need to abstract over TMT_REBOOT_COUNT. Assisted-by: OpenCode (Claude claude-opus-4-6) Signed-off-by: Colin Walters --- .github/workflows/ci.yml | 47 +++++++++++++++++++++++++- Justfile | 14 ++++++++ tmt/tests/Dockerfile.upgrade-source | 18 ++++++++++ tmt/tests/booted/bootc_testlib.nu | 49 +++++++++++++++++++++++++++- tmt/tests/booted/test-01-readonly.nu | 6 ++++ 5 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 tmt/tests/Dockerfile.upgrade-source diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04485ea59..ad5233ab5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -262,6 +262,50 @@ jobs: ${{ env.ARCH }}" path: /var/tmp/tmt + # Test the upgrade path: boot from published base image, upgrade to locally-built image, + # then run readonly tests to verify the upgrade worked. + # Excluded: centos-9 (lacks systemd.extra-unit.* support needed for --bind-storage-ro) + test-upgrade: + needs: package + strategy: + fail-fast: false + matrix: + test_os: [fedora-43, centos-10] + + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v6 + - name: Bootc Ubuntu Setup + uses: bootc-dev/actions/bootc-ubuntu-setup@main + with: + libvirt: true + - name: Install tmt + run: pip install --user "tmt[provision-virtual]" + + - name: Setup env + run: | + BASE=$(just pullspec-for-os base ${{ matrix.test_os }}) + echo "BOOTC_base=${BASE}" >> $GITHUB_ENV + echo "BOOTC_SKIP_PACKAGE=1" >> $GITHUB_ENV + echo "RUST_BACKTRACE=full" >> $GITHUB_ENV + + - name: Download package artifacts + uses: actions/download-artifact@v8 + with: + name: packages-${{ matrix.test_os }} + path: target/packages/ + + - name: Run upgrade test + run: just test-upgrade + + - name: Archive TMT logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: "tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-upgrade-${{ env.ARCH }}" + path: /var/tmp/tmt + # Test bootc install on Fedora CoreOS (separate job to avoid disk space issues # when run in the same job as test-integration). # Uses fedora-43 as it's the current stable Fedora release matching CoreOS. @@ -342,7 +386,7 @@ jobs: # Sentinel job for required checks - configure this job name in repository settings required-checks: if: always() - needs: [cargo-deny, validate, package, test-integration, test-coreos, test-container-export] + needs: [cargo-deny, validate, package, test-integration, test-upgrade, test-coreos, test-container-export] runs-on: ubuntu-latest steps: - run: exit 1 @@ -351,5 +395,6 @@ jobs: needs.validate.result != 'success' || needs.package.result != 'success' || needs.test-integration.result != 'success' || + needs.test-upgrade.result != 'success' || needs.test-coreos.result != 'success' || needs.test-container-export.result != 'success' diff --git a/Justfile b/Justfile index 9b853d926..a6e33da9f 100644 --- a/Justfile +++ b/Justfile @@ -17,6 +17,8 @@ base_img := "localhost/bootc" # Synthetic upgrade image for testing upgrade_img := base_img + "-upgrade" +# Base image with tmt dependencies added, used as the boot source for upgrade tests +upgrade_source_img := base_img + "-upgrade-source" # Build variant: ostree (default) or composefs variant := env("BOOTC_variant", "ostree") @@ -141,6 +143,14 @@ test-composefs bootloader filesystem boot_type seal_state *ARGS: {{ARGS}} \ $(if [ "{{boot_type}}" = "uki" ]; then echo "readonly"; else echo "integration"; fi) +# Run upgrade test: boot VM from published base image (with tmt deps added), +# upgrade to locally-built image, reboot, then run readonly tests to verify. +# The --upgrade-image flag triggers --bind-storage-ro in bcvk, making the +# locally-built image available inside the VM via containers-storage transport. +[group('core')] +test-upgrade *ARGS: build _build-upgrade-source-image + cargo xtask run-tmt --env=BOOTC_variant={{variant}} --env=BOOTC_test_upgrade_image={{base_img}} --upgrade-image={{base_img}} {{upgrade_source_img}} {{ARGS}} readonly + # Run cargo fmt and clippy checks in container [group('core')] validate: @@ -339,6 +349,10 @@ _keygen: _build-upgrade-image: cat tmt/tests/Dockerfile.upgrade | podman build -t {{upgrade_img}} --from={{base_img}} - +# Build the upgrade source image: base image + tmt dependencies (rsync, nu, cloud-init) +_build-upgrade-source-image: + podman build --build-arg=base={{base}} -t {{upgrade_source_img}} -f tmt/tests/Dockerfile.upgrade-source . + # Copy an image from user podman storage to root's podman storage # This allows building as regular user then running privileged tests [group('testing')] diff --git a/tmt/tests/Dockerfile.upgrade-source b/tmt/tests/Dockerfile.upgrade-source new file mode 100644 index 000000000..1eceb069c --- /dev/null +++ b/tmt/tests/Dockerfile.upgrade-source @@ -0,0 +1,18 @@ +# Build an image suitable for upgrade testing: takes the published base image +# and adds the minimal dependencies needed by tmt (rsync, nu, etc.) +# so we can boot it in a VM, then upgrade into the locally-built image. +# +# Note: we do NOT pass `cloudinit` to provision-derived.sh because bcvk +# handles SSH key injection itself; cloud-init interferes with that. +ARG base +FROM ${base} +COPY hack/provision-derived.sh hack/packages.txt /run/provision/ +RUN --mount=type=tmpfs,target=/tmp < { + if not (have_hostexports) { + error make { msg: "BOOTC_test_upgrade_image is set but host exports (--bind-storage-ro) are not available" } + } + print $"Upgrade image specified: ($upgrade_image)" + print "Performing upgrade switch..." + bootc switch --transport containers-storage $upgrade_image + print "Switch complete, rebooting..." + tmt-reboot + }, + "1" => { + print $"Second boot after upgrade to ($upgrade_image)" + let st = bootc status --json | from json + let booted = $st.status.booted.image + assert equal $booted.image.transport "containers-storage" + assert equal $booted.image.image $upgrade_image + print "Upgrade verified, continuing with tests..." + }, + $o => { + # For higher reboot counts, just continue - the caller + # may have its own reboot logic + }, + } +} diff --git a/tmt/tests/booted/test-01-readonly.nu b/tmt/tests/booted/test-01-readonly.nu index 2b72e37cf..d7662bddf 100644 --- a/tmt/tests/booted/test-01-readonly.nu +++ b/tmt/tests/booted/test-01-readonly.nu @@ -7,9 +7,15 @@ # # Run all readonly tests in sequence use tap.nu +use bootc_testlib.nu tap begin "readonly tests" +# If an upgrade image is specified (via BOOTC_test_upgrade_image env var), +# perform the upgrade and reboot first. On the second boot after upgrade, +# this returns and we continue with the readonly tests below. +bootc_testlib maybe_upgrade + # Get all readonly test files and run them in order let tests = (ls booted/readonly/*-test-*.nu | get name | sort) From 119f458bd6ac709b35c922da29c63750958da3bb Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 18 Mar 2026 00:22:25 +0000 Subject: [PATCH 2/3] ci/upgrade: Add composefs variant and upgrade --check test Extend the upgrade test matrix with an ostree/composefs variant dimension, so we test upgrading both storage backends. For composefs, the upgrade source image needs: - The bootc dracut module in the initramfs (auto-excluded by check()) - A fsverity-capable root filesystem (ext4 or btrfs via configure-rootfs) - enforcing=0 karg (base image SELinux policy lacks composefs contexts) The enforcing=0 karg is also added to the target image test-kargs so the post-upgrade boot works. A new readonly test (031-test-upgrade-check) runs `bootc upgrade --check` after the upgrade to verify the running system can still self-update. For composefs with pre-upgrade bootc <= 1.13, this is a known failure (#2074) due to incompatible on-disk state. The pre-upgrade bootc version is saved to /var/bootc-pre-upgrade-version during first boot. Assisted-by: OpenCode (Claude claude-opus-4-6) Signed-off-by: Colin Walters --- .github/workflows/ci.yml | 4 +- Justfile | 19 ++++++++- crates/xtask/src/tmt.rs | 4 ++ crates/xtask/src/xtask.rs | 4 ++ tmt/tests/Dockerfile.upgrade-source | 16 +++++++- tmt/tests/booted/bootc_testlib.nu | 6 +++ .../booted/readonly/031-test-upgrade-check.nu | 41 +++++++++++++++++++ 7 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 tmt/tests/booted/readonly/031-test-upgrade-check.nu diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad5233ab5..bd7fe28d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -271,6 +271,7 @@ jobs: fail-fast: false matrix: test_os: [fedora-43, centos-10] + variant: [ostree, composefs] runs-on: ubuntu-24.04 @@ -287,6 +288,7 @@ jobs: run: | BASE=$(just pullspec-for-os base ${{ matrix.test_os }}) echo "BOOTC_base=${BASE}" >> $GITHUB_ENV + echo "BOOTC_variant=${{ matrix.variant }}" >> $GITHUB_ENV echo "BOOTC_SKIP_PACKAGE=1" >> $GITHUB_ENV echo "RUST_BACKTRACE=full" >> $GITHUB_ENV @@ -303,7 +305,7 @@ jobs: if: always() uses: actions/upload-artifact@v7 with: - name: "tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-upgrade-${{ env.ARCH }}" + name: "tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ matrix.variant }}-upgrade-${{ env.ARCH }}" path: /var/tmp/tmt # Test bootc install on Fedora CoreOS (separate job to avoid disk space issues diff --git a/Justfile b/Justfile index a6e33da9f..76b63389a 100644 --- a/Justfile +++ b/Justfile @@ -149,7 +149,22 @@ test-composefs bootloader filesystem boot_type seal_state *ARGS: # locally-built image available inside the VM via containers-storage transport. [group('core')] test-upgrade *ARGS: build _build-upgrade-source-image - cargo xtask run-tmt --env=BOOTC_variant={{variant}} --env=BOOTC_test_upgrade_image={{base_img}} --upgrade-image={{base_img}} {{upgrade_source_img}} {{ARGS}} readonly + #!/bin/bash + set -xeuo pipefail + composefs_args=() + if [[ "{{variant}}" = composefs ]]; then + composefs_args=(--composefs-backend \ + --bootloader={{bootloader}} \ + --filesystem={{filesystem}} \ + --seal-state={{seal_state}} \ + --boot-type={{boot_type}} \ + --karg=enforcing=0) + fi + cargo xtask run-tmt --env=BOOTC_variant={{variant}} \ + --env=BOOTC_test_upgrade_image={{base_img}} \ + --upgrade-image={{base_img}} \ + "${composefs_args[@]}" \ + {{upgrade_source_img}} {{ARGS}} readonly # Run cargo fmt and clippy checks in container [group('core')] @@ -351,7 +366,7 @@ _build-upgrade-image: # Build the upgrade source image: base image + tmt dependencies (rsync, nu, cloud-init) _build-upgrade-source-image: - podman build --build-arg=base={{base}} -t {{upgrade_source_img}} -f tmt/tests/Dockerfile.upgrade-source . + podman build --build-arg=base={{base}} --build-arg=variant={{variant}} -t {{upgrade_source_img}} -f tmt/tests/Dockerfile.upgrade-source . # Copy an image from user podman storage to root's podman storage # This allows building as regular user then running privileged tests diff --git a/crates/xtask/src/tmt.rs b/crates/xtask/src/tmt.rs index 0766c8a93..f0242ca06 100644 --- a/crates/xtask/src/tmt.rs +++ b/crates/xtask/src/tmt.rs @@ -492,6 +492,10 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { opts.push(format!("--bootloader={b}")); } + for k in &args.karg { + opts.push(format!("--karg={k}")); + } + opts }; diff --git a/crates/xtask/src/xtask.rs b/crates/xtask/src/xtask.rs index 1e0fc0d44..2fc7c10fb 100644 --- a/crates/xtask/src/xtask.rs +++ b/crates/xtask/src/xtask.rs @@ -176,6 +176,10 @@ pub(crate) struct RunTmtArgs { // Required to send kargs to only bls installs #[arg(long, default_value_t, requires = "composefs_backend")] pub(crate) boot_type: BootType, + + /// Additional kernel arguments to pass to bcvk + #[arg(long)] + pub(crate) karg: Vec, } /// Arguments for tmt-provision command diff --git a/tmt/tests/Dockerfile.upgrade-source b/tmt/tests/Dockerfile.upgrade-source index 1eceb069c..3ce0ae74e 100644 --- a/tmt/tests/Dockerfile.upgrade-source +++ b/tmt/tests/Dockerfile.upgrade-source @@ -6,7 +6,8 @@ # handles SSH key injection itself; cloud-init interferes with that. ARG base FROM ${base} -COPY hack/provision-derived.sh hack/packages.txt /run/provision/ +COPY hack/provision-derived.sh hack/packages.txt contrib/packaging/configure-rootfs /run/provision/ +ARG variant=ostree RUN --mount=type=tmpfs,target=/tmp < /etc/dracut.conf.d/50-bootc-composefs.conf + kver=$(cd /usr/lib/modules && echo *) + dracut --force --kver "$kver" "/usr/lib/modules/$kver/initramfs.img" +fi rm -rf /run/provision EORUN diff --git a/tmt/tests/booted/bootc_testlib.nu b/tmt/tests/booted/bootc_testlib.nu index 3b879f029..2560d9b13 100644 --- a/tmt/tests/booted/bootc_testlib.nu +++ b/tmt/tests/booted/bootc_testlib.nu @@ -50,6 +50,12 @@ export def maybe_upgrade [] { if not (have_hostexports) { error make { msg: "BOOTC_test_upgrade_image is set but host exports (--bind-storage-ro) are not available" } } + # Save the pre-upgrade bootc version so post-upgrade tests + # can detect known incompatibilities with older versions. + let pre_ver = (bootc --version | parse "bootc {v}" | get 0.v) + $pre_ver | save /var/bootc-pre-upgrade-version + print $"Pre-upgrade bootc version: ($pre_ver)" + print $"Upgrade image specified: ($upgrade_image)" print "Performing upgrade switch..." bootc switch --transport containers-storage $upgrade_image diff --git a/tmt/tests/booted/readonly/031-test-upgrade-check.nu b/tmt/tests/booted/readonly/031-test-upgrade-check.nu new file mode 100644 index 000000000..31b3fd412 --- /dev/null +++ b/tmt/tests/booted/readonly/031-test-upgrade-check.nu @@ -0,0 +1,41 @@ +use std assert +use tap.nu + +tap begin "verify bootc upgrade --check succeeds after upgrade" + +# After an upgrade (bootc switch), verify that the running system can +# still query for further upgrades. This catches regressions where +# on-disk state created by an older bootc is incompatible with the +# new bootc's upgrade machinery (e.g. #2074). +# Only meaningful when we actually performed an upgrade. +let upgrade_image = $env.BOOTC_test_upgrade_image? | default "" +if $upgrade_image == "" { + print "# skip: not an upgrade test (BOOTC_test_upgrade_image not set)" + tap ok + exit 0 +} + +# Read the pre-upgrade bootc version saved during first boot. +let old_version_str = (open /var/bootc-pre-upgrade-version | str trim) +let old_ver = ($old_version_str | split row "." | each { into int }) +print $"Pre-upgrade bootc version: ($old_version_str)" + +let is_composefs = (tap is_composefs) + +print "Running bootc upgrade --check to verify upgrade machinery works..." +let result = do -i { bootc upgrade --check } | complete +if $result.exit_code == 0 { + print "bootc upgrade --check succeeded" +} else { + print $"bootc upgrade --check failed: ($result.stderr)" + # Known failure: composefs upgrades from bootc <= 1.13 have + # incompatible on-disk state (see #2074). + let old_bootc_le_1_13 = ($old_ver.0 < 1) or (($old_ver.0 == 1) and ($old_ver.1 <= 13)) + if $is_composefs and $old_bootc_le_1_13 { + print $"# known failure: composefs upgrade --check from bootc ($old_version_str) \(see #2074\)" + } else { + error make { msg: $"bootc upgrade --check failed unexpectedly: ($result.stderr)" } + } +} + +tap ok From f2f03e93f46f7560fac68efd707c41630e0ebab6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 18 Mar 2026 21:41:38 +0000 Subject: [PATCH 3/3] tests/readonly: Add SELinux enforcing check Verify that SELinux is in enforcing mode at the start of the readonly test suite. This catches accidental regressions where test images ship with permissive mode or enforcing=0 on the kernel command line. The check is skipped for composefs upgrade tests where enforcing=0 is intentionally needed due to the base image SELinux policy not yet covering composefs file contexts. Assisted-by: OpenCode (Claude claude-opus-4-6) Signed-off-by: Colin Walters --- .../readonly/000-test-selinux-enforcing.nu | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tmt/tests/booted/readonly/000-test-selinux-enforcing.nu diff --git a/tmt/tests/booted/readonly/000-test-selinux-enforcing.nu b/tmt/tests/booted/readonly/000-test-selinux-enforcing.nu new file mode 100644 index 000000000..7a6a66b84 --- /dev/null +++ b/tmt/tests/booted/readonly/000-test-selinux-enforcing.nu @@ -0,0 +1,21 @@ +use std assert +use tap.nu + +tap begin "verify SELinux is enforcing" + +# Composefs upgrade source images boot with enforcing=0 because the +# base image's SELinux policy doesn't yet cover composefs file contexts. +# Skip this check in that case. +let upgrade_image = $env.BOOTC_test_upgrade_image? | default "" +let is_composefs = (tap is_composefs) +if $upgrade_image != "" and $is_composefs { + print "# skip: composefs upgrade boots with enforcing=0 (base image SELinux policy gap)" + tap ok + exit 0 +} + +let enforce = (open /sys/fs/selinux/enforce | str trim) +assert equal $enforce "1" "SELinux should be in enforcing mode" +print "SELinux is enforcing" + +tap ok