From 8a198e3b149a1fccded30df7a49bdb6f7b1f4c2b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 18 Mar 2026 17:45:43 +0100 Subject: [PATCH 01/11] Add support for RISC-V repository --- .github/workflows/docs.yml | 7 +++++++ scripts/generate_data_files.py | 30 +++++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 399f7ef..8415aae 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -60,6 +60,13 @@ jobs: # Merge all these results together python scripts/merge_data_files.py out.yaml eessi*.yaml mv out.yaml docs/data/eessi_software_metadata.yaml + # Also do RISC-V repository + rm eessi*.yaml + export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" + export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" + ( module load EESSI/2025.06 && module load EasyBuild/5 && python generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) + python scripts/merge_data_files.py out_riscv.yaml eessi*.yaml + mv out_riscv.yaml docs/data/eessi_software_metadata_riscv.yaml # Generate json data files and markdown index/description for them cd docs/data python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata.yaml eessi_api_metadata diff --git a/scripts/generate_data_files.py b/scripts/generate_data_files.py index b16e405..a70d27c 100644 --- a/scripts/generate_data_files.py +++ b/scripts/generate_data_files.py @@ -18,7 +18,16 @@ from easybuild.tools.include import include_easyblocks from contextlib import contextmanager -VALID_EESSI_VERSIONS = ["2025.06", "2023.06"] +SUPPORTED_REPOSITORIES = { + 'software.eessi.io': ["2025.06", "2023.06"], + 'dev.eessi.io/riscv': ["2025.06-001"], +} + +VALID_EESSI_VERSIONS = [ + version + for versions in SUPPORTED_REPOSITORIES.values() + for version in versions +] # Give order to my toolchains so I can easily figure out what "latest" means EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS = OrderedDict( @@ -205,20 +214,31 @@ def merge_dicts(d1, d2): "-e", required=True, choices=VALID_EESSI_VERSIONS, - help=f"Allowed versions: {', '.join(VALID_EESSI_VERSIONS)}", + help=f"Allowed versions (also dependent on repository): {', '.join(VALID_EESSI_VERSIONS)}", + ) + parser.add_argument( + "--repository", + "-r", + default="software.eessi.io", + choices=SUPPORTED_REPOSITORIES.keys(), + help=f"Repository to scan: {', '.join(SUPPORTED_REPOSITORIES)} (default: %(default)s)", ) args = parser.parse_args() eessi_version = args.eessi_version + repository = args.repository + + if eessi_version not in SUPPORTED_REPOSITORIES[repository]: + raise ValueError(f"You must choose an EESSI version supported by the repository: {SUPPORTED_REPOSITORIES[repository]}") print(f"Using EESSI version: {eessi_version}") # We use a single architecture path to gather information about the software versions eessi_reference_architecture = os.getenv("EESSI_ARCHDETECT_OPTIONS_OVERRIDE", False) if not eessi_reference_architecture: - print("You must have selected a CPU architecture via EESSI_ARCHDETECT_OPTIONS_OVERRIDE") + print("You must have selected a CPU architecture via EESSI_ARCHDETECT_OPTIONS_OVERRIDE environment variable") exit() - base_path = f"/cvmfs/software.eessi.io/versions/{eessi_version}/software/linux/{eessi_reference_architecture}" + base_path = f"/cvmfs/{repository}/versions/{eessi_version}/software/linux/{eessi_reference_architecture}" cpu_easyconfig_files_dict = collect_eb_files(os.path.join(base_path, "software")) # We also gather all the acclerator installations for NVIDIA-enabled packages # We're not typically running this script on a node with a GPU so an override must have been set @@ -243,7 +263,7 @@ def merge_dicts(d1, d2): # Store the toolchain hierarchies supported by the EESSI version eessi_software["eessi_version"][eessi_version]["toolchain_hierarchy"] = {} - for top_level_toolchain in EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS[eessi_version]: + for top_level_toolchain in EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS[eessi_version.split("-")[0]]: # versions are typically 2024a/2024b etc. for top level toolchains # so let's use that to make sorting easy toolchain_family = f"{top_level_toolchain['version']}_{top_level_toolchain['name']}" From 7252c8760f205eb8001f71340c8ada47979d7584 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 18 Mar 2026 18:04:38 +0100 Subject: [PATCH 02/11] Work towards support for RISCV in process_eessi_software_metadata.py --- scripts/process_eessi_software_metadata.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/process_eessi_software_metadata.py b/scripts/process_eessi_software_metadata.py index e2042a2..01f9c3a 100644 --- a/scripts/process_eessi_software_metadata.py +++ b/scripts/process_eessi_software_metadata.py @@ -24,6 +24,10 @@ "x86_64/intel/cascadelake", ] +RISCV_ARCHITECTURES = [ + "riscv64/generic", +] + NVIDIA_ARCHITECTURES = [ "accel/nvidia/cc70", "accel/nvidia/cc80", @@ -63,7 +67,11 @@ def get_software_information_by_filename(file_metadata, original_path=None, tool # 1) Detect the architecture substring inside the path base_version_dict["cpu_arch"] = [] detected_arch = None - for arch in ARCHITECTURES: + if '/riscv64/' in original_path: + architecture_group = RISCV_ARCHITECTURES + else: + architecture_group = ARCHITECTURES + for arch in architecture_group: if f"/{arch}/" in original_path: detected_arch = arch break @@ -89,7 +97,7 @@ def get_software_information_by_filename(file_metadata, original_path=None, tool spider_cache = before_arch + detected_arch + "/.lmod/cache/spiderT.lua" # 3) Substitute each architecture and test module file existence in spider cache - for arch in ARCHITECTURES: + for arch in architecture_group: substituted_modulefile = modulefile.replace(detected_arch, arch) substituted_spider_cache = spider_cache.replace(detected_arch, arch) # os.path.exists is very expensive for CVMFS so we just look for the file in the spider cache @@ -368,7 +376,7 @@ def main(): base_json_metadata["architectures_map"] = {} for eessi_version in eessi_versions: base_json_metadata["architectures_map"][eessi_version] = {} - for architecture in ARCHITECTURES: + for architecture in ARCHITECTURES + RISCV_ARCHITECTURES: base_json_metadata["architectures_map"][eessi_version][architecture] = architecture base_json_metadata["gpu_architectures_map"] = {} base_json_metadata["category_details"] = {} From 2f769a5e583df601ee024df1b492abaeb0644d7e Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 18 Mar 2026 21:15:18 +0100 Subject: [PATCH 03/11] No spider cache for RISC-V --- .github/workflows/docs.yml | 10 +++++++++- .github/workflows/prs.yml | 20 ++++++++++++++++++-- scripts/process_eessi_software_metadata.py | 7 +++++-- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8415aae..55d2ba5 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -60,6 +60,7 @@ jobs: # Merge all these results together python scripts/merge_data_files.py out.yaml eessi*.yaml mv out.yaml docs/data/eessi_software_metadata.yaml + # Also do RISC-V repository rm eessi*.yaml export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" @@ -67,11 +68,18 @@ jobs: ( module load EESSI/2025.06 && module load EasyBuild/5 && python generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) python scripts/merge_data_files.py out_riscv.yaml eessi*.yaml mv out_riscv.yaml docs/data/eessi_software_metadata_riscv.yaml + # Generate json data files and markdown index/description for them cd docs/data python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata.yaml eessi_api_metadata + python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata_riscv.yaml eessi_api_metadata_riscv python ../../scripts/calculate_hashes.py - for json_file in *.json; do + for json_file in *metadata.json; do + python ../../scripts/generate_schema_md.py $json_file >> index.md + done + echo "" >> index.md + echo "### RISC-V datafiles" >> index.md + for json_file in *metadata_riscv.json; do python ../../scripts/generate_schema_md.py $json_file >> index.md done - run: | diff --git a/.github/workflows/prs.yml b/.github/workflows/prs.yml index da0d590..20c87bc 100644 --- a/.github/workflows/prs.yml +++ b/.github/workflows/prs.yml @@ -39,13 +39,29 @@ jobs: wait python scripts/merge_data_files.py out.yaml eessi*.yaml mv out.yaml docs/data/eessi_software_metadata.yaml + + # Also test RISC-V + rm eessi*.yaml + export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" + export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" + ( module load EESSI/2025.06 && module load EasyBuild/5 && python generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) + python scripts/merge_data_files.py out_riscv.yaml eessi*.yaml + mv out_riscv.yaml docs/data/eessi_software_metadata_riscv.yaml + # Generate json data files and markdown index/description for them cd docs/data python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata.yaml eessi_api_metadata + python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata_riscv.yaml eessi_api_metadata_riscv python ../../scripts/calculate_hashes.py - for json_file in *.json; do + for json_file in *metadata.json; do python ../../scripts/generate_schema_md.py $json_file >> index.md done + echo "" >> index.md + echo "### RISC-V datafiles" >> index.md + for json_file in *metadata_riscv.json; do + python ../../scripts/generate_schema_md.py $json_file >> index.md + done + cat index.md - name: Test building the website run: | source /tmp/venv_docs/bin/activate @@ -54,4 +70,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: eessi-api-metadata - path: docs/data/eessi_api_metadata_software.json + path: docs/data/eessi_api_metadata_software*.json diff --git a/scripts/process_eessi_software_metadata.py b/scripts/process_eessi_software_metadata.py index 01f9c3a..2d96206 100644 --- a/scripts/process_eessi_software_metadata.py +++ b/scripts/process_eessi_software_metadata.py @@ -100,8 +100,11 @@ def get_software_information_by_filename(file_metadata, original_path=None, tool for arch in architecture_group: substituted_modulefile = modulefile.replace(detected_arch, arch) substituted_spider_cache = spider_cache.replace(detected_arch, arch) - # os.path.exists is very expensive for CVMFS so we just look for the file in the spider cache - found = subprocess.run(["grep", "-q", substituted_modulefile, substituted_spider_cache]).returncode == 0 + # os.path.exists is very expensive for CVMFS so we just look for the file in the spider cache (if we can) + if os.path.exists(substituted_spider_cache): + found = subprocess.run(["grep", "-q", substituted_modulefile, substituted_spider_cache]).returncode == 0 + else: + found = os.path.exists(substituted_modulefile) if found: base_version_dict["cpu_arch"].append(arch) # If we have an accelerator module let's check which architectures are supported From 4f3f2603a9a0d7e218cb761ffca69a7c6fcd3594 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 18 Mar 2026 21:29:37 +0100 Subject: [PATCH 04/11] Be careful about when environment variables are set for RISC-V case --- .github/workflows/docs.yml | 5 ++--- .github/workflows/prs.yml | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 55d2ba5..c09df06 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -63,9 +63,8 @@ jobs: # Also do RISC-V repository rm eessi*.yaml - export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" - export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" - ( module load EESSI/2025.06 && module load EasyBuild/5 && python generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) + # Need to set environment variables after loading EESSI to avoid problems with EESSI module + ( module load EESSI/2025.06 && export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" && export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" && module load EasyBuild/5 && python generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) python scripts/merge_data_files.py out_riscv.yaml eessi*.yaml mv out_riscv.yaml docs/data/eessi_software_metadata_riscv.yaml diff --git a/.github/workflows/prs.yml b/.github/workflows/prs.yml index 20c87bc..571d991 100644 --- a/.github/workflows/prs.yml +++ b/.github/workflows/prs.yml @@ -42,9 +42,8 @@ jobs: # Also test RISC-V rm eessi*.yaml - export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" - export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" - ( module load EESSI/2025.06 && module load EasyBuild/5 && python generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) + # Need to set environment variables after loading EESSI to avoid problems with EESSI module + ( module load EESSI/2025.06 && export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" && export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" && module load EasyBuild/5 && python generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) python scripts/merge_data_files.py out_riscv.yaml eessi*.yaml mv out_riscv.yaml docs/data/eessi_software_metadata_riscv.yaml From 627f612b9a8c967e8e8d38497cb598d863e163df Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 18 Mar 2026 21:32:45 +0100 Subject: [PATCH 05/11] Make sure to upload all files in PR --- .github/workflows/prs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prs.yml b/.github/workflows/prs.yml index 571d991..e6211a4 100644 --- a/.github/workflows/prs.yml +++ b/.github/workflows/prs.yml @@ -69,4 +69,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: eessi-api-metadata - path: docs/data/eessi_api_metadata_software*.json + path: docs/data/eessi_api_metadata_*software*.json From fde7455c60aa418d1ff5f6d759be8e8cc0b18dff Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 18 Mar 2026 21:43:45 +0100 Subject: [PATCH 06/11] Correct path to script --- .github/workflows/docs.yml | 2 +- .github/workflows/prs.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c09df06..eb42680 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -64,7 +64,7 @@ jobs: # Also do RISC-V repository rm eessi*.yaml # Need to set environment variables after loading EESSI to avoid problems with EESSI module - ( module load EESSI/2025.06 && export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" && export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" && module load EasyBuild/5 && python generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) + ( module load EESSI/2025.06 && export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" && export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" && module load EasyBuild/5 && python scripts/generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) python scripts/merge_data_files.py out_riscv.yaml eessi*.yaml mv out_riscv.yaml docs/data/eessi_software_metadata_riscv.yaml diff --git a/.github/workflows/prs.yml b/.github/workflows/prs.yml index e6211a4..130e275 100644 --- a/.github/workflows/prs.yml +++ b/.github/workflows/prs.yml @@ -43,7 +43,7 @@ jobs: # Also test RISC-V rm eessi*.yaml # Need to set environment variables after loading EESSI to avoid problems with EESSI module - ( module load EESSI/2025.06 && export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" && export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" && module load EasyBuild/5 && python generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) + ( module load EESSI/2025.06 && export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" && export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" && module load EasyBuild/5 && python scripts/generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) python scripts/merge_data_files.py out_riscv.yaml eessi*.yaml mv out_riscv.yaml docs/data/eessi_software_metadata_riscv.yaml From 5849475f9e200b5e446b6f77bf4ce4e6914947b5 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 18 Mar 2026 22:26:03 +0100 Subject: [PATCH 07/11] Update prs.yml --- .github/workflows/prs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prs.yml b/.github/workflows/prs.yml index 130e275..fa34ccc 100644 --- a/.github/workflows/prs.yml +++ b/.github/workflows/prs.yml @@ -43,7 +43,7 @@ jobs: # Also test RISC-V rm eessi*.yaml # Need to set environment variables after loading EESSI to avoid problems with EESSI module - ( module load EESSI/2025.06 && export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" && export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" && module load EasyBuild/5 && python scripts/generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) + ( module load EESSI/2025.06 && module load EasyBuild/5 && module load EESSI-extend && export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" && export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" && python scripts/generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) python scripts/merge_data_files.py out_riscv.yaml eessi*.yaml mv out_riscv.yaml docs/data/eessi_software_metadata_riscv.yaml From e97bce2f3054341702fc68448f4e71490211ae2c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 19 Mar 2026 11:11:05 +0100 Subject: [PATCH 08/11] Try harder to initialise a RISC-V repo correctly --- .github/workflows/docs.yml | 25 ++++++++++++++++++++----- .github/workflows/prs.yml | 33 +++++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index eb42680..7c940c8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -64,21 +64,36 @@ jobs: # Also do RISC-V repository rm eessi*.yaml # Need to set environment variables after loading EESSI to avoid problems with EESSI module - ( module load EESSI/2025.06 && export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" && export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" && module load EasyBuild/5 && python scripts/generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) + ( + module purge + export EESSI_DEBUG_INIT=1 + export EESSI_VERSION_OVERRIDE=2025.06-001 + export EESSI_ARCHDETECT_OPTIONS_OVERRIDE=riscv64/generic + export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" + # Need to fake uname to convince EESSI we are on a RISC-V system + uname() { echo "riscv64"; } + export -f uname + module load EESSI/2025.06 + # Can't run binaries from compat layer so remove them + export PATH=$(echo "$PATH" | /usr/bin/tr ':' '\n' | /usr/bin/grep -v '^/cvmfs' | /usr/bin/paste -sd ':' -) + module load EasyBuild/5 + module load EESSI-extend + python scripts/generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" + ) python scripts/merge_data_files.py out_riscv.yaml eessi*.yaml - mv out_riscv.yaml docs/data/eessi_software_metadata_riscv.yaml + mv out_riscv.yaml docs/data/eessi_software_metadata-riscv.yaml # Generate json data files and markdown index/description for them cd docs/data python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata.yaml eessi_api_metadata - python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata_riscv.yaml eessi_api_metadata_riscv + python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata-riscv.yaml eessi_api_metadata-riscv python ../../scripts/calculate_hashes.py - for json_file in *metadata.json; do + for json_file in eessi_api_metadata_*.json; do python ../../scripts/generate_schema_md.py $json_file >> index.md done echo "" >> index.md echo "### RISC-V datafiles" >> index.md - for json_file in *metadata_riscv.json; do + for json_file in eessi_api_metadata-riscv*.json; do python ../../scripts/generate_schema_md.py $json_file >> index.md done - run: | diff --git a/.github/workflows/prs.yml b/.github/workflows/prs.yml index fa34ccc..e8c2963 100644 --- a/.github/workflows/prs.yml +++ b/.github/workflows/prs.yml @@ -34,7 +34,13 @@ jobs: export EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/nvidia/cc90" export EESSI_OVERRIDE_GPU_CHECK=1 # Only do 2023.06 for EB 5 since this is just a test - ( module load EESSI/2023.06 && module load EasyBuild/5 && module load EESSI-extend && python scripts/generate_data_files.py --eessi-version=2023.06 ) & + ( + module purge + module load EESSI/2023.06 + module load EasyBuild/5 + module load EESSI-extend + python scripts/generate_data_files.py --eessi-version=2023.06 + ) & # Merge all these results together wait python scripts/merge_data_files.py out.yaml eessi*.yaml @@ -43,21 +49,36 @@ jobs: # Also test RISC-V rm eessi*.yaml # Need to set environment variables after loading EESSI to avoid problems with EESSI module - ( module load EESSI/2025.06 && module load EasyBuild/5 && module load EESSI-extend && export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="riscv64/generic" && export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" && python scripts/generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" ) + ( + module purge + export EESSI_DEBUG_INIT=1 + export EESSI_VERSION_OVERRIDE=2025.06-001 + export EESSI_ARCHDETECT_OPTIONS_OVERRIDE=riscv64/generic + export EESSI_ACCELERATOR_TARGET_OVERRIDE="doesnotexist" + # Need to fake uname to convince EESSI we are on a RISC-V system + uname() { echo "riscv64"; } + export -f uname + module load EESSI/2025.06 + # Can't run binaries from compat layer so remove them + export PATH=$(echo "$PATH" | /usr/bin/tr ':' '\n' | /usr/bin/grep -v '^/cvmfs' | /usr/bin/paste -sd ':' -) + module load EasyBuild/5 + module load EESSI-extend + python scripts/generate_data_files.py --eessi-version 2025.06-001 --repository "dev.eessi.io/riscv" + ) python scripts/merge_data_files.py out_riscv.yaml eessi*.yaml - mv out_riscv.yaml docs/data/eessi_software_metadata_riscv.yaml + mv out_riscv.yaml docs/data/eessi_software_metadata-riscv.yaml # Generate json data files and markdown index/description for them cd docs/data python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata.yaml eessi_api_metadata - python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata_riscv.yaml eessi_api_metadata_riscv + python ../../scripts/process_eessi_software_metadata.py eessi_software_metadata-riscv.yaml eessi_api_metadata-riscv python ../../scripts/calculate_hashes.py - for json_file in *metadata.json; do + for json_file in eessi_api_metadata_*.json; do python ../../scripts/generate_schema_md.py $json_file >> index.md done echo "" >> index.md echo "### RISC-V datafiles" >> index.md - for json_file in *metadata_riscv.json; do + for json_file in eessi_api_metadata-riscv*.json; do python ../../scripts/generate_schema_md.py $json_file >> index.md done cat index.md From 97457139656431af944610e703af30dc07bb2a11 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 19 Mar 2026 11:38:57 +0100 Subject: [PATCH 09/11] Tidy up, and a little less noisy --- .github/workflows/docs.yml | 6 +++--- .github/workflows/prs.yml | 7 +++---- scripts/process_eessi_software_metadata.py | 10 +++++++++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7c940c8..dbae2e7 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -63,7 +63,7 @@ jobs: # Also do RISC-V repository rm eessi*.yaml - # Need to set environment variables after loading EESSI to avoid problems with EESSI module + # Need to set environment variables to load EESSI module for RISC-V ( module purge export EESSI_DEBUG_INIT=1 @@ -74,7 +74,7 @@ jobs: uname() { echo "riscv64"; } export -f uname module load EESSI/2025.06 - # Can't run binaries from compat layer so remove them + # Can't run RISC-V binaries from compat layer so remove them export PATH=$(echo "$PATH" | /usr/bin/tr ':' '\n' | /usr/bin/grep -v '^/cvmfs' | /usr/bin/paste -sd ':' -) module load EasyBuild/5 module load EESSI-extend @@ -93,7 +93,7 @@ jobs: done echo "" >> index.md echo "### RISC-V datafiles" >> index.md - for json_file in eessi_api_metadata-riscv*.json; do + for json_file in eessi_api_metadata-riscv_*.json; do python ../../scripts/generate_schema_md.py $json_file >> index.md done - run: | diff --git a/.github/workflows/prs.yml b/.github/workflows/prs.yml index e8c2963..a646108 100644 --- a/.github/workflows/prs.yml +++ b/.github/workflows/prs.yml @@ -48,7 +48,7 @@ jobs: # Also test RISC-V rm eessi*.yaml - # Need to set environment variables after loading EESSI to avoid problems with EESSI module + # Need to set environment variables to load EESSI module for RISC-V ( module purge export EESSI_DEBUG_INIT=1 @@ -59,7 +59,7 @@ jobs: uname() { echo "riscv64"; } export -f uname module load EESSI/2025.06 - # Can't run binaries from compat layer so remove them + # Can't run RISC-V binaries from compat layer so remove them export PATH=$(echo "$PATH" | /usr/bin/tr ':' '\n' | /usr/bin/grep -v '^/cvmfs' | /usr/bin/paste -sd ':' -) module load EasyBuild/5 module load EESSI-extend @@ -81,7 +81,6 @@ jobs: for json_file in eessi_api_metadata-riscv*.json; do python ../../scripts/generate_schema_md.py $json_file >> index.md done - cat index.md - name: Test building the website run: | source /tmp/venv_docs/bin/activate @@ -90,4 +89,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: eessi-api-metadata - path: docs/data/eessi_api_metadata_*software*.json + path: docs/data/eessi_api_metadata*software*.json diff --git a/scripts/process_eessi_software_metadata.py b/scripts/process_eessi_software_metadata.py index 2d96206..30b2bc7 100644 --- a/scripts/process_eessi_software_metadata.py +++ b/scripts/process_eessi_software_metadata.py @@ -117,7 +117,15 @@ def get_software_information_by_filename(file_metadata, original_path=None, tool # Let's not include the "accel/" part of the accel_arch base_version_dict["gpu_arch"][arch].append(accel_arch.replace("accel/", "", 1)) else: - print(f"No module {accel_substituted_modulefile}...not adding software for architecture {arch}/{accel_arch}") + # Let's not be too noisy here, we know we don't have some CUDA archs in 2023.06 + if not ( + accel_substituted_modulefile.startswith('/cvmfs/software.eessi.io/versions/2023.06') + and accel_arch in ["accel/nvidia/cc100", "accel/nvidia/cc120"] + ): + print( + f"No module {accel_substituted_modulefile}... " + f"not adding software for architecture {arch}/{accel_arch}" + ) continue else: print(f"No module {substituted_modulefile}...not adding software for architecture {arch}") From 78213ae0b96dfead21cd624df84cb99b895afd4c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 19 Mar 2026 12:15:21 +0100 Subject: [PATCH 10/11] Final tidy up --- .github/workflows/docs.yml | 1 + .github/workflows/prs.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index dbae2e7..1b107d0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -93,6 +93,7 @@ jobs: done echo "" >> index.md echo "### RISC-V datafiles" >> index.md + echo "" >> index.md for json_file in eessi_api_metadata-riscv_*.json; do python ../../scripts/generate_schema_md.py $json_file >> index.md done diff --git a/.github/workflows/prs.yml b/.github/workflows/prs.yml index a646108..25ca3ce 100644 --- a/.github/workflows/prs.yml +++ b/.github/workflows/prs.yml @@ -78,6 +78,7 @@ jobs: done echo "" >> index.md echo "### RISC-V datafiles" >> index.md + echo "" >> index.md for json_file in eessi_api_metadata-riscv*.json; do python ../../scripts/generate_schema_md.py $json_file >> index.md done From 1841c679e9e67a34e14a2b8fe0c01ba13142ef72 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 19 Mar 2026 12:34:33 +0100 Subject: [PATCH 11/11] Add a comment about removing stub for RISC-V --- scripts/generate_data_files.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/generate_data_files.py b/scripts/generate_data_files.py index a70d27c..852fb75 100644 --- a/scripts/generate_data_files.py +++ b/scripts/generate_data_files.py @@ -263,6 +263,7 @@ def merge_dicts(d1, d2): # Store the toolchain hierarchies supported by the EESSI version eessi_software["eessi_version"][eessi_version]["toolchain_hierarchy"] = {} + # RISC-V versions have a stub like -001 at the end, make sure to drop it for top_level_toolchain in EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS[eessi_version.split("-")[0]]: # versions are typically 2024a/2024b etc. for top level toolchains # so let's use that to make sorting easy