From 60fbe93328f14d0f4dfc518dd52c51f4d8f045e1 Mon Sep 17 00:00:00 2001 From: Michele Baldessari Date: Sun, 22 Mar 2026 16:35:15 +0100 Subject: [PATCH] WIP: Allow to use a prebuilt container with collections --- Makefile | 5 ++- README.md | 42 +++++++++++++++++++------ agof_vault_template.yml | 6 ---- collections-container/Containerfile | 22 +++++++++++++ collections-container/ansible.cfg.build | 13 ++++++++ collections-container/build.sh | 36 +++++++++++++++++++++ pre_init/main.yml | 37 ++++++++-------------- pre_init/templates/ansible.cfg.j2 | 10 +----- pre_init/vars/preinit_vars.yml | 11 +------ scripts/pattern-util.sh | 4 ++- site.yml | 2 -- vars/main.yml | 1 - 12 files changed, 125 insertions(+), 64 deletions(-) create mode 100644 collections-container/Containerfile create mode 100644 collections-container/ansible.cfg.build create mode 100755 collections-container/build.sh diff --git a/Makefile b/Makefile index a3775e6..b68e815 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,10 @@ help: ## This help message @echo "Pattern: $(NAME)" @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^(\s|[a-zA-Z_0-9-])+:.*?##/ { printf " \033[36m%-35s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) -preinit: ## Setup ansible environemnt - configure ansible.cfg and download collections +build-collections-image: ## Build the pre-built collections container image + collections-container/build.sh + +preinit: ## Setup ansible environment - configure ansible.cfg and verify pre-built collections ansible-playbook pre_init/main.yml $(EXTRA_PLAYBOOK_OPTS) fix_aws_dns: ## Update public DNS for AWS - needed when a VM cold starts diff --git a/README.md b/README.md index 9f20178..aa5e878 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ * 4.3. [Automation Hub Specific Configuration](#AutomationHubSpecificConfiguration) * 4.4. [AWS-Specific Configuration](#AWS-SpecificConfiguration) * 4.5. [ImageBuilder-Specific Configuration](#ImageBuilder-SpecificConfiguration) +* 6. [Pre-built Collections Container](#PrebuiltCollectionsContainer) * 5. [What the Framework Does, Step-by-Step](#WhattheFrameworkDoesStep-by-Step) * 5.1. [Pre-GitOps Steps](#Pre-GitOpsSteps) * 5.1.1. [[Pre-init](init_env/pre_init_env.yml) (mandatory)](#Pre-initinit_envpre_init_env.ymlmandatory) @@ -172,9 +173,6 @@ The variables to include builds for the extra components are all Ansible boolean | redhat_registry_username_vault | Red Hat Subscriber Username | true | | | | redhat_registry_password_vault | Red Hat Subscriber Password | true | | | | manifest_content | Base64 encoded Manifest to Entitle | true | | Can be loaded directly from a file using a construct like this: `"{{ lookup('file', '~/Downloads/manifest.zip') | b64encode }}"` | -| automation_hub_certified_url | URL for Certified Content | false | | This refers to the automation hub section on [https://console.redhat.com](https://console.redhat.com). It is the endpoint that is used to download Validated Content in addition to any public Galaxy content needed | -| automation_hub_validated_url | URL for Validated Content | false | | This refers to the automation hub section on [https://console.redhat.com](https://console.redhat.com). It is the endpoint that is used to download Certified Content in addition to any public Galaxy content needed | -| automation_hub_token_vault| Subscriber-specific token for Content | true | | | automation_hub | Flag to build and enable Automation Hub | false | false | | Building a Private Automation Hub is necessary if your pattern builds an Execution Environment that is not hosted on a public container registry. | | eda | Flag to build and enable Event Driven Automation controller | false | true | | | | agof_controller_config_dir | Directory to pass to controller_configuration | false | | This directory is the key one to load all other AAP Controller configuration. The framework will populate it by checking out the `agof_iac_repo` version `agof_iac_repo_version` (default: `main`) | @@ -184,9 +182,6 @@ The variables to include builds for the extra components are all Ansible boolean | Name | Description | Required | Default | Notes | | ------------------------- | ------------------------------------ | -------- | -------- | ------------------ | -| init_env_collection_install | Whether to install collections required by the framework | false | true | | -| init_env_collection_install_force | Whether to use the `force` argument when installing collections | false | false | Forces the installation of declared dependencies if true | -| special_collection_installs | "Bundled" collection installations (references files in repodir) | false | `[]` | A mechanism to allow the installation of collections bundled into the pattern, if the ones published in galaxy and/or Automation Hub are not sufficient | | offline_token | Red Hat Offline Token | false | | Used to build the imagebuilder image | ### 4.3. Automation Hub Specific Configuration @@ -233,11 +228,11 @@ Pre-initialization, for our purposes, refers to things that have to be installed ##### Build ansible.cfg -Because ansible.cfg needs to be configured with the automation hub endpooint, we generate it from a template. The ansible.cfg includes endpoints for both public and Automation Hub galaxy servers and a vault password path (for `ansible-vault`). The template used is [here](init_env/templates/ansible.cfg.j2). The automation hub token is read from the vault and injected into the environment for the collection installation phase. +The ansible.cfg is generated from a template to configure collection paths and other Ansible settings. The template used is [here](init_env/templates/ansible.cfg.j2). -##### Collection dependency install +##### Pre-built collections -The next thing the pre-init play does is install dependency collections based on what's contained in [requirements.yml](requirements.yml). If there are any "special" collections contained in the root of the framework they can also be installed at this time. When the framework was under development this was necessary with redhat.satellite but the necessary changes have been upstreamed as of 3.11; 3.12 has been released since then, so this mechanism is no longer needed but is left in as it might be necessary in the future. +Ansible collections required by the framework are pre-installed in the `agof-collections` container image (see [Pre-built Collections Container](#PrebuiltCollectionsContainer)). The pre-init play verifies that the collections are available at `/usr/share/ansible/collections/ansible_collections`. This eliminates the need for Automation Hub tokens at runtime. #### 5.1.2. [Environment Initialization](init_env/main.yml) (optional; enabled by default; `make install` entry point) @@ -347,7 +342,34 @@ Meanwhile, the helm values that have been passed into the aap-config application The framework is only truly in GitOps mode once the configuration has been fully applied to the AAP controller and the AAP controller has taken responsibility for managing the environment. Prior to that point, there are many opportunities (by design) to inject non-declarative elements into the environment. Beyond that point, changes should be made to the environment or the workflows that configure it by pushing git commits to the repositories the pattern uses. -## 6. Acknowledgements +## 6. Pre-built Collections Container + +All required Ansible collections (listed in [requirements.yml](requirements.yml)) are shipped pre-installed in the `agof-collections` container image. This eliminates the need for Automation Hub tokens at runtime. + +### Using the Container + +Set the `AGOF_COLLECTIONS_CONTAINER` environment variable to use the pre-built image: + +```shell +export AGOF_COLLECTIONS_CONTAINER=quay.io/validatedpatterns/agof-collections:latest +``` + +### Building the Container (for maintainers) + +Requires a valid Automation Hub token: + +```shell +export AUTOMATION_HUB_TOKEN="your-token-here" +make build-collections-image +``` + +To build and push to the registry: + +```shell +AUTOMATION_HUB_TOKEN="your-token" collections-container/build.sh latest --push +``` + +## 7. Acknowledgements This repository represents an interpretation of GitOps principles, as developed in the Hybrid Cloud Patterns GitOps framework for Kubernetes, and an adaptation and fusion of two previous ongoing efforts at Red Hat: [Ansible-Workshops](https://github.com/ansible/workshops) and [LabBuilder2/RHISbuilder](https://github.com/parmstro/labbuilder2). diff --git a/agof_vault_template.yml b/agof_vault_template.yml index 2858369..3098c08 100644 --- a/agof_vault_template.yml +++ b/agof_vault_template.yml @@ -28,9 +28,3 @@ activation_key_vault: "The name of an Activation Key to embed in the imagebuilde #skip_imagebuilder_build: 'boolean: skips imagebuilder build process' #imagebuilder_ami: 'The ID of an AWS AMI image, preferably one that was built with this toolkit' -automation_hub_token_vault: 'A token associated with your AAP subscription used to retrieve Automation Hub content' - -# These variables can be set but are optional. The previous (before AAP 2.4) conncept of sync-list was private -# to an account. -#automation_hub_url_certified_vault: 'The private automation hub URL for certified content' -#automation_hub_url_validated_vault: 'The private automation hub URL for validated content' diff --git a/collections-container/Containerfile b/collections-container/Containerfile new file mode 100644 index 0000000..4e5f9c9 --- /dev/null +++ b/collections-container/Containerfile @@ -0,0 +1,22 @@ +# Stage 1: Install collections (this stage is discarded — token never in final image) +FROM quay.io/validatedpatterns/utility-container:v1.0.2 AS builder + +ARG AUTOMATION_HUB_TOKEN + +COPY requirements.yml /tmp/requirements.yml +COPY collections-container/ansible.cfg.build /etc/ansible/ansible.cfg + +ENV ANSIBLE_GALAXY_SERVER_AUTOMATION_HUB_CERTIFIED_TOKEN=${AUTOMATION_HUB_TOKEN} +ENV ANSIBLE_GALAXY_SERVER_AUTOMATION_HUB_VALIDATED_TOKEN=${AUTOMATION_HUB_TOKEN} + +RUN ansible-galaxy collection install \ + -p /usr/share/ansible/collections \ + -r /tmp/requirements.yml + +# Stage 2: Clean image with only the collections +FROM quay.io/validatedpatterns/utility-container:v1.0.2 + +LABEL maintainer="Validated Patterns Team" +LABEL description="AGOF utility container with pre-installed Ansible collections" + +COPY --from=builder /usr/share/ansible/collections /usr/share/ansible/collections diff --git a/collections-container/ansible.cfg.build b/collections-container/ansible.cfg.build new file mode 100644 index 0000000..b681c64 --- /dev/null +++ b/collections-container/ansible.cfg.build @@ -0,0 +1,13 @@ +[galaxy] +server_list = automation_hub_certified,automation_hub_validated,galaxy_pub + +[galaxy_server.automation_hub_certified] +url=https://console.redhat.com/api/automation-hub/content/published/ +auth_url=https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token + +[galaxy_server.automation_hub_validated] +url=https://console.redhat.com/api/automation-hub/content/validated/ +auth_url=https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token + +[galaxy_server.galaxy_pub] +url=https://galaxy.ansible.com/ diff --git a/collections-container/build.sh b/collections-container/build.sh new file mode 100755 index 0000000..0de7fe2 --- /dev/null +++ b/collections-container/build.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +IMAGE_NAME="${AGOF_COLLECTIONS_IMAGE:-quay.io/validatedpatterns/agof-collections}" +TAG="${1:-latest}" +PUSH=false + +for arg in "$@"; do + if [ "$arg" = "--push" ]; then + PUSH=true + fi +done + +if [ -z "${AUTOMATION_HUB_TOKEN:-}" ]; then + echo "Error: AUTOMATION_HUB_TOKEN environment variable must be set" + echo "Get your token from https://console.redhat.com/ansible/automation-hub/token" + exit 1 +fi + +echo "Building ${IMAGE_NAME}:${TAG}..." +podman build \ + --build-arg "AUTOMATION_HUB_TOKEN=${AUTOMATION_HUB_TOKEN}" \ + -t "${IMAGE_NAME}:${TAG}" \ + -f collections-container/Containerfile \ + "${REPO_ROOT}" + +echo "Built ${IMAGE_NAME}:${TAG}" + +if [ "$PUSH" = true ]; then + echo "Pushing ${IMAGE_NAME}:${TAG}..." + podman push "${IMAGE_NAME}:${TAG}" + echo "Pushed ${IMAGE_NAME}:${TAG}" +fi diff --git a/pre_init/main.yml b/pre_init/main.yml index ec9f926..6a2ed3b 100644 --- a/pre_init/main.yml +++ b/pre_init/main.yml @@ -23,32 +23,21 @@ when: - offline_token is defined - - name: Check if automation_hub_token is older than 30 days - ansible.builtin.import_tasks: jwt_check.yml - vars: - local_token_name: "automation_hub_token" - local_token: "{{ automation_hub_token }}" - failed_when: false - when: - - automation_hub_token is defined - - - name: "Install collections for ansible environment {{ 'forcefully' if init_env_collection_install_force }}" - when: init_env_collection_install + - name: "Verify pre-built collections are available" block: - - name: "Install local patched files" - ansible.builtin.command: - cmd: "ansible-galaxy collection install {{ '--force' if init_env_collection_install_force }} -p '{{ ansible_cfg_patch_collection_dir }}' '{{ item }}'" - chdir: ".." - loop: '{{ special_collection_installs }}' + - name: "Check that pre-built collections directory exists" + ansible.builtin.stat: + path: "/usr/share/ansible/collections/ansible_collections" + register: prebuilt_collections_dir - - name: "Install requirements file based collections {{ 'forcefully' if init_env_collection_install_force }}" - ansible.builtin.command: - cmd: |- - ansible-galaxy collection install {{ '--force' if init_env_collection_install_force }} -p '{{ ansible_cfg_patch_collection_dir }}' -r requirements.yml - chdir: ".." - environment: - ANSIBLE_GALAXY_SERVER_AUTOMATION_HUB_CERTIFIED_TOKEN: "{{ automation_hub_token }}" - ANSIBLE_GALAXY_SERVER_AUTOMATION_HUB_VALIDATED_TOKEN: "{{ automation_hub_token }}" + - name: "Assert pre-built collections are present" + ansible.builtin.assert: + that: + - prebuilt_collections_dir.stat.exists + - prebuilt_collections_dir.stat.isdir + fail_msg: > + Pre-built collections not found at /usr/share/ansible/collections/ansible_collections. + Are you running inside the agof-collections container? - name: "Assert that we have a valid target platform" ansible.builtin.assert: diff --git a/pre_init/templates/ansible.cfg.j2 b/pre_init/templates/ansible.cfg.j2 index 94ca865..a5aab6d 100644 --- a/pre_init/templates/ansible.cfg.j2 +++ b/pre_init/templates/ansible.cfg.j2 @@ -46,15 +46,7 @@ timeout = 10 control_path = %(directory)s/%%h-%%r [galaxy] -server_list = automation_hub_certified,automation_hub_validated,galaxy_pub - -[galaxy_server.automation_hub_certified] -url={{ automation_hub_certified_url | d('https://console.redhat.com/api/automation-hub/content/published/') }} -auth_url={{ auth_url | d('https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token') }} - -[galaxy_server.automation_hub_validated] -url={{ automation_hub_validated_url | d('https://console.redhat.com/api/automation-hub/content/validated/') }} -auth_url={{ auth_url | d('https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token') }} +server_list = galaxy_pub [galaxy_server.galaxy_pub] url=https://galaxy.ansible.com/ diff --git a/pre_init/vars/preinit_vars.yml b/pre_init/vars/preinit_vars.yml index 1a1cdbd..35cce07 100644 --- a/pre_init/vars/preinit_vars.yml +++ b/pre_init/vars/preinit_vars.yml @@ -7,13 +7,4 @@ bootstrap_allowable_values: agof_bootstrap_target: "aws" ansible_cfg_log_path: '~/lab_builder_ansible.log' -ansible_cfg_collections_path: '~/.ansible/collections/ansible_collections:/usr/share/ansible/collections/ansible_collections' -ansible_cfg_patch_collection_dir: '~/.ansible/collections/ansible_collections' - -automation_hub_url: '{{ automation_hub_url_vault }}' -automation_hub_token: '{{ automation_hub_token_vault }}' - -init_env_collection_install: true -init_env_collection_install_force: false - -special_collection_installs: [] +ansible_cfg_collections_path: '/usr/share/ansible/collections/ansible_collections:~/.ansible/collections/ansible_collections' diff --git a/scripts/pattern-util.sh b/scripts/pattern-util.sh index 1d59ef2..4c19d57 100755 --- a/scripts/pattern-util.sh +++ b/scripts/pattern-util.sh @@ -8,7 +8,9 @@ function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }' } -if [ -z "$PATTERN_UTILITY_CONTAINER" ]; then +if [ -n "$AGOF_COLLECTIONS_CONTAINER" ]; then + PATTERN_UTILITY_CONTAINER="$AGOF_COLLECTIONS_CONTAINER" +elif [ -z "$PATTERN_UTILITY_CONTAINER" ]; then PATTERN_UTILITY_CONTAINER="quay.io/validatedpatterns/utility-container:v1.0.2" fi # If PATTERN_DISCONNECTED_HOME is set it will be used to populate both PATTERN_UTILITY_CONTAINER diff --git a/site.yml b/site.yml index 20c9534..84031cf 100644 --- a/site.yml +++ b/site.yml @@ -7,8 +7,6 @@ vars_files: - 'vars/main.yml' - '~/agof_vault.yml' - vars: - init_env_collection_install_force: true tasks: - name: "Show vars" ansible.builtin.debug: diff --git a/vars/main.yml b/vars/main.yml index 8c044f6..29bae69 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -7,4 +7,3 @@ containerized_install: true containerized_installer_user: 'aap' containerized_installer_user_home: '/home/aap' -init_env_collection_install_force: false