From 3aa5ffa53440d86e3a8cd5c0a36975dc229b0248 Mon Sep 17 00:00:00 2001 From: AnHeuermann <38031952+AnHeuermann@users.noreply.github.com> Date: Tue, 17 Mar 2026 15:43:52 +0100 Subject: [PATCH 1/3] Overhaul reference comparison: new error metrics, DataFrames CSV, richer HTML table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the per-point `_check_point` / `error_fn` dispatch with two vectorised helpers (`_absolute_error`, `_scaled_relative_error`) and a globally-scaled relative tolerance check. `abs_tol` is now optional (`nothing` = disabled, shown as ∞ in the HTML); when set it acts as an independent hard threshold rather than a per-point floor. Switch diff-CSV output from manual text-mode writing to DataFrames.jl + CSV.jl; the new format includes `_abserr` and `_relerr` columns for every failing signal. Improve the `_diff.html` variable-coverage table: - Add "Max Abs Error" (scientific notation) and "Max Rel Error" (percent) columns for both passing and failing signals. - Read per-signal max errors for failing signals directly from the diff CSV. - Hide the abs-tolerance from the meta line when `abs_tol` is `nothing`. Thread `settings::CompareSettings` explicitly through `test_model` and `main`; remove the module-level `_CMP_SETTINGS` global and the `configure_comparison!` / `compare_settings` API. Co-Authored-By: claude-sonnet-4-6 --- Project.toml | 2 + src/BaseModelicaLibraryTesting.jl | 5 +- src/compare.jl | 263 +++++++++++------------------- src/pipeline.jl | 17 +- src/types.jl | 22 +-- 5 files changed, 112 insertions(+), 197 deletions(-) diff --git a/Project.toml b/Project.toml index 75a73180f..94d1ffecd 100644 --- a/Project.toml +++ b/Project.toml @@ -5,6 +5,8 @@ authors = ["AnHeuermann"] [deps] BaseModelica = "a17d5099-185d-4ff5-b5d3-51aa4569e56d" +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" DifferentialEquations = "0c46a032-eb83-5123-abaf-570d42b7fbaa" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" diff --git a/src/BaseModelicaLibraryTesting.jl b/src/BaseModelicaLibraryTesting.jl index 7610640fa..71af4f1c5 100644 --- a/src/BaseModelicaLibraryTesting.jl +++ b/src/BaseModelicaLibraryTesting.jl @@ -22,10 +22,7 @@ include("pipeline.jl") # Shared types and constants export ModelResult, CompareSettings, RunInfo -export LIBRARY, LIBRARY_VERSION, CMP_REL_TOL, CMP_ABS_TOL - -# Comparison configuration -export configure_comparison!, compare_settings +export CMP_REL_TOL, CMP_ABS_TOL # Pipeline phases export run_export # Phase 1: Base Modelica export via OMC diff --git a/src/compare.jl b/src/compare.jl index 51568c1e1..6f17e745a 100644 --- a/src/compare.jl +++ b/src/compare.jl @@ -1,5 +1,7 @@ # ── Variable name utilities ──────────────────────────────────────────────────── +import CSV +import DataFrames import ModelingToolkit import Printf: @sprintf @@ -102,120 +104,6 @@ function _install_assets(results_root::String) end end -# ── Comparison settings and error functions ──────────────────────────────────── - -"""Module-level default comparison settings. Modify via `configure_comparison!`.""" -const _CMP_SETTINGS = CompareSettings() - -""" - _check_relative(s, r, ref_scale, cfg) → Bool - -Classic relative-error check. Passes when - - |s − r| ≤ max(rel_tol · |r|, abs_tol) - -This is the traditional approach used by many validation tools. It works well -when the signal stays well away from zero, but may produce false failures at -zero crossings because the per-point tolerance shrinks to `abs_tol ≈ 0` when -`r ≈ 0`. -""" -function _check_relative(s::Real, r::Real, ::Real, cfg::CompareSettings)::Bool - abs(s - r) <= max(cfg.rel_tol * abs(r), cfg.abs_tol) -end - -""" - _check_mixed(s, r, ref_scale, cfg) → Bool - -Scale-aware relative-error check (default). Passes when - - |s − r| ≤ max(rel_tol · |r|, rel_tol · ref_scale, abs_tol) - -The middle term (`rel_tol · ref_scale`) provides an amplitude-proportional -absolute floor. Near zero crossings the tolerance is set by the peak magnitude -of the reference signal rather than the near-zero instantaneous value, so -physically correct simulations are not falsely rejected. -""" -function _check_mixed(s::Real, r::Real, ref_scale::Real, cfg::CompareSettings)::Bool - abs(s - r) <= max(cfg.rel_tol * abs(r), cfg.rel_tol * ref_scale, cfg.abs_tol) -end - -""" - _check_absolute(s, r, ref_scale, cfg) → Bool - -Pure absolute check. Passes when - - |s − r| ≤ abs_tol - -Useful when all compared signals have known, small magnitudes or when a -signal-independent tolerance threshold is required. -""" -function _check_absolute(s::Real, r::Real, ::Real, cfg::CompareSettings)::Bool - abs(s - r) <= cfg.abs_tol -end - -""" - _check_point(s, r, ref_scale, cfg) → Bool - -Dispatch to the error function selected by `cfg.error_fn`. - -| `error_fn` | Description | -|:--------------|:----------------------------------------------------| -| `:mixed` | Scale-aware relative error (default, recommended) | -| `:relative` | Classic relative error (may fail at zero crossings) | -| `:absolute` | Pure absolute error | -""" -function _check_point(s::Real, r::Real, ref_scale::Real, cfg::CompareSettings)::Bool - fn = cfg.error_fn - fn === :mixed && return _check_mixed(s, r, ref_scale, cfg) - fn === :relative && return _check_relative(s, r, ref_scale, cfg) - fn === :absolute && return _check_absolute(s, r, ref_scale, cfg) - throw(ArgumentError( - "Unknown error_fn $(repr(fn)); choose :mixed, :relative, or :absolute")) -end - -""" - configure_comparison!(; rel_tol, abs_tol, error_fn) → CompareSettings - -Update the module-level comparison settings in-place and return them. - -# Keyword arguments - -- `rel_tol` — maximum allowed relative error. Default: `$(CMP_REL_TOL)` (2 %). -- `abs_tol` — hard absolute-error floor applied when signals are near zero. - Default: `$(CMP_ABS_TOL)`. -- `error_fn` — selects the point-wise check function. One of: - - `:mixed` — scale-aware relative error (default, recommended); - - `:relative` — classic relative error (may reject valid zero-crossing signals); - - `:absolute` — pure absolute error. - -# Example - -```julia -configure_comparison!(rel_tol = 0.01, error_fn = :relative) -``` -""" -function configure_comparison!(; - rel_tol :: Union{Float64,Nothing} = nothing, - abs_tol :: Union{Float64,Nothing} = nothing, - error_fn :: Union{Symbol,Nothing} = nothing, -) - isnothing(rel_tol) || (_CMP_SETTINGS.rel_tol = rel_tol) - isnothing(abs_tol) || (_CMP_SETTINGS.abs_tol = abs_tol) - isnothing(error_fn) || (_CMP_SETTINGS.error_fn = error_fn) - return _CMP_SETTINGS -end - -""" - compare_settings() → CompareSettings - -Return the current module-level comparison settings. - -Pass the returned object (or a freshly constructed `CompareSettings(...)`) to -`compare_with_reference` via the `settings` keyword to override the defaults -for a single call without changing the global state. -""" -compare_settings() = _CMP_SETTINGS - # ── Interactive diff HTML ────────────────────────────────────────────────────── """ @@ -235,32 +123,42 @@ The page references `../../assets/dygraph.min.*` relative to its location. `_install_assets` is called automatically. """ function write_diff_html(model_dir::String, model::String; - diff_csv_path::String = "", - pass_sigs::Vector{String} = String[], - skip_sigs::Vector{String} = String[]) + diff_csv_path::String = "", + pass_sigs::Vector{String} = String[], + skip_sigs::Vector{String} = String[], + pass_max_abs_error::Dict{String,Float64} = Dict{String,Float64}(), + pass_max_rel_error::Dict{String,Float64} = Dict{String,Float64}(), + settings::CompareSettings = CompareSettings()) short_name = split(model, ".")[end] html_path = joinpath(model_dir, "$(short_name)_diff.html") results_root = dirname(dirname(abspath(model_dir))) # …/files/ → … _install_assets(results_root) - # Read fail_sigs and CSV content from the diff CSV (may not exist). - fail_sigs = String[] - csv_js = "" + # Read fail_sigs, per-signal max errors, and CSV content from the diff CSV. + fail_sigs = String[] + max_abs_error = Dict{String,Float64}() + max_rel_error = Dict{String,Float64}() + csv_js = "" if !isempty(diff_csv_path) && isfile(diff_csv_path) - lines = readlines(diff_csv_path) - if length(lines) >= 1 - headers = [replace(strip(h), "\"" => "") for h in split(lines[1], ",")] - for h in headers - length(h) > 4 && h[end-3:end] == "_ref" && push!(fail_sigs, h[1:end-4]) - end - csv_text = read(diff_csv_path, String) - csv_js = replace(replace(csv_text, "\\" => "\\\\"), "`" => "\\`") + df = CSV.read(diff_csv_path, DataFrames.DataFrame) + for col in names(df) + endswith(col, "_ref") && push!(fail_sigs, col[1:end-4]) + end + for sig in fail_sigs + max_abs_error[sig] = maximum(df[!, "$(sig)_abserr"]) + max_rel_error[sig] = maximum(df[!, "$(sig)_relerr"]) end + csv_text = read(diff_csv_path, String) + csv_js = replace(replace(csv_text, "\\" => "\\\\"), "`" => "\\`") end # ── Meta block ────────────────────────────────────────────────────────────── - tol_str = "(rel ≤ $(round(Int, _CMP_SETTINGS.rel_tol * 100))%," * - " abs ≤ $(_CMP_SETTINGS.abs_tol))" + tol_str = if settings.abs_tol === nothing + "(rel ≤ $(round(Int, settings.rel_tol * 100))%)" + else + "(rel ≤ $(round(Int, settings.rel_tol * 100))%," * + " abs ≤ $(settings.abs_tol))" + end csv_link = isempty(fail_sigs) ? "" : """  ·  Download diff CSV""" skip_note = isempty(skip_sigs) ? "" : @@ -277,23 +175,30 @@ function write_diff_html(model_dir::String, model::String; n_total = n_found + length(skip_sigs) th = "border:1px solid #ccc;padding:3px 10px;background:#eee;text-align:left;" td = "border:1px solid #ccc;padding:3px 10px;" + tdr = td * "text-align:right;" rows = String[] for sig in pass_sigs push!(rows, "$sig" * - "✓ pass") + "✓ pass" * + "$(@sprintf("%.4e", pass_max_abs_error[sig]))" * + "$(@sprintf("%.2f%%", pass_max_rel_error[sig] * 100))") end for sig in fail_sigs push!(rows, "$sig" * - "✗ fail") + "✗ fail" * + "$(@sprintf("%.4e", max_abs_error[sig]))" * + "$(@sprintf("%.2f%%", max_rel_error[sig] * 100))") end for sig in skip_sigs push!(rows, "$sig" * - "not found in simulation") + "not found in simulation" * + "——") end """

Variable Coverage """ * """— $n_found of $n_total reference signal(s) found

""" * """""" * - """""" * + """""" * + """""" * """$(join(rows))
SignalStatus
SignalStatusMax Abs ErrorMax Rel Error
""" end @@ -312,6 +217,27 @@ end # ── Reference comparison ─────────────────────────────────────────────────────── +""" + _absolute_error(actual, reference) -> Vector{Real} + +Return the element-wise absolute error between `actual` and `reference`. +""" +function _absolute_error(actual::AbstractVector{<:Real}, reference::AbstractVector{<:Real}) + return abs.(actual .- reference) +end + +""" + _scaled_relative_error(actual, reference) -> Vector{Real} + +Return the element-wise absolute error between `actual` and `reference`, scaled by the +maximum absolute value of `reference` (or `eps()` if that maximum is smaller, to avoid +division by zero). +""" +function _scaled_relative_error(actual::AbstractVector{<:Real}, reference::AbstractVector{<:Real}) + reference_scale = max( maximum(abs.(reference)), eps() ) + return abs.(actual .- reference) ./ reference_scale +end + """ _eval_sim(sol, accessor, t) → Float64 @@ -354,16 +280,14 @@ is written whenever there are failures or skipped signals. # Keyword arguments - `settings` — a `CompareSettings` instance controlling tolerances and the - error function. Defaults to the module-level settings returned - by `compare_settings()`. Use `configure_comparison!` to change - the defaults, or pass a local `CompareSettings(...)` here. + error function. """ function compare_with_reference( sol, ref_csv_path::String, model_dir::String, model::String; - settings::CompareSettings = _CMP_SETTINGS, + settings::CompareSettings = CompareSettings(), )::Tuple{Int,Int,Int,String} times, ref_data = _read_ref_csv(ref_csv_path) @@ -409,7 +333,12 @@ function compare_with_reference( pass_sigs = String[] fail_sigs = String[] skip_sigs = String[] - fail_scales = Dict{String,Float64}() + pass_max_abs_error = Dict{String, Float64}() + pass_max_rel_error = Dict{String, Float64}() + fail_ref_vals = Dict{String, Vector{Float64}}() + fail_sim_vals = Dict{String, Vector{Float64}}() + fail_abs_error = Dict{String, Vector{Float64}}() + fail_scaled_rel_error = Dict{String, Vector{Float64}}() for sig in signals haskey(ref_data, sig) || continue # signal absent from ref CSV entirely @@ -424,10 +353,6 @@ function compare_with_reference( ref_vals = ref_data[sig][valid_mask] n_total += 1 - # Peak |ref| — used as amplitude floor so relative error stays finite - # near zero crossings. - ref_scale = isempty(ref_vals) ? 0.0 : maximum(abs, ref_vals) - # Interpolate simulation at reference time points. sim_vals = [_eval_sim(sol, accessor, t) for t in t_ref] @@ -438,16 +363,24 @@ function compare_with_reference( continue end - pass = all(zip(sim_vals, ref_vals)) do (s, r) - _check_point(s, r, ref_scale, settings) - end + # Check absolute error and globally scaled relative error + abs_error = _absolute_error(sim_vals, ref_vals) + scaled_rel_error = _scaled_relative_error(sim_vals, ref_vals) + + pass = (settings.abs_tol === nothing || maximum(abs_error) < settings.abs_tol) && + maximum(scaled_rel_error) < settings.rel_tol if pass n_pass += 1 push!(pass_sigs, sig) + pass_max_abs_error[sig] = maximum(abs_error) + pass_max_rel_error[sig] = maximum(scaled_rel_error) else push!(fail_sigs, sig) - fail_scales[sig] = ref_scale + fail_ref_vals[sig] = ref_vals + fail_sim_vals[sig] = sim_vals + fail_abs_error[sig] = abs_error + fail_scaled_rel_error[sig] = scaled_rel_error end end @@ -457,35 +390,25 @@ function compare_with_reference( diff_csv = "" if !isempty(fail_sigs) diff_csv = joinpath(model_dir, "$(short_name)_diff.csv") - open(diff_csv, "w") do f - cols = ["time"] - for sig in fail_sigs - push!(cols, "$(sig)_ref", "$(sig)_sim", "$(sig)_relerr") - end - println(f, join(cols, ",")) - for (ti, t) in enumerate(t_ref) - row = [@sprintf("%.10g", t)] - for sig in fail_sigs - ref_vals = ref_data[sig][valid_mask] - r = ref_vals[ti] - s = _eval_sim(sol, var_access[_normalize_var(sig)], t) - ref_scale = get(fail_scales, sig, 0.0) - relerr = abs(s - r) / max(abs(r), ref_scale, settings.abs_tol) - push!(row, @sprintf("%.10g", r), - @sprintf("%.10g", s), - @sprintf("%.6g", relerr)) - end - println(f, join(row, ",")) - end + df = DataFrames.DataFrame("time" => t_ref) + for sig in fail_sigs + df[!, "$(sig)_ref"] = fail_ref_vals[sig] + df[!, "$(sig)_sim"] = fail_sim_vals[sig] + df[!, "$(sig)_abserr"] = fail_abs_error[sig] + df[!, "$(sig)_relerr"] = fail_scaled_rel_error[sig] end + CSV.write(diff_csv, df) end # ── Write detail HTML whenever there is anything worth showing ─────────────── if !isempty(fail_sigs) || !isempty(skip_sigs) write_diff_html(model_dir, model; - diff_csv_path = diff_csv, - pass_sigs = pass_sigs, - skip_sigs = skip_sigs) + diff_csv_path = diff_csv, + pass_sigs = pass_sigs, + skip_sigs = skip_sigs, + pass_max_abs_error = pass_max_abs_error, + pass_max_rel_error = pass_max_rel_error, + settings = settings) end return n_total, n_pass, length(skip_sigs), diff_csv diff --git a/src/pipeline.jl b/src/pipeline.jl index b1ee54237..d2dc8171a 100644 --- a/src/pipeline.jl +++ b/src/pipeline.jl @@ -57,8 +57,12 @@ end Run the four-phase pipeline for a single model and return its result. """ -function test_model(omc::OMJulia.OMCSession, model::String, results_root::String, - ref_root::String; csv_max_size_mb::Int = CSV_MAX_SIZE_MB)::ModelResult +function test_model(omc::OMJulia.OMCSession, + model::String, + results_root::String, + ref_root::String; + csv_max_size_mb::Int = CSV_MAX_SIZE_MB, + settings::CompareSettings = CompareSettings())::ModelResult model_dir = joinpath(results_root, "files", model) mkpath(model_dir) @@ -86,7 +90,7 @@ function test_model(omc::OMJulia.OMCSession, model::String, results_root::String if ref_csv !== nothing try cmp_total, cmp_pass, cmp_skip, cmp_csv = - compare_with_reference(sol, ref_csv, model_dir, model) + compare_with_reference(sol, ref_csv, model_dir, model; settings) catch e @warn "Reference comparison failed for $model: $(sprint(showerror, e))" end @@ -111,14 +115,15 @@ Discovers models via OMC, runs `test_model` for each, then writes the HTML report. Returns a `Vector{ModelResult}`. """ function main(; - library :: String = LIBRARY, - version :: String = LIBRARY_VERSION, + library :: String, + version :: String, filter :: Union{String,Nothing} = nothing, omc_exe :: String = get(ENV, "OMC_EXE", "omc"), results_root :: String = "", ref_root :: String = get(ENV, "MAPLIB_REF", ""), bm_options :: String = get(ENV, "BM_OPTIONS", "scalarize,moveBindings,inlineFunctions"), csv_max_size_mb :: Int = CSV_MAX_SIZE_MB, + settings :: CompareSettings = CompareSettings(), ) t0 = time() @@ -186,7 +191,7 @@ function main(; for (i, model) in enumerate(models) @info "[$i/$(length(models))] $model" - result = test_model(omc, model, results_root, ref_root; csv_max_size_mb) + result = test_model(omc, model, results_root, ref_root; csv_max_size_mb, settings) push!(results, result) phase = result.sim_success ? "SIM OK" : diff --git a/src/types.jl b/src/types.jl index b073c7843..1666ae666 100644 --- a/src/types.jl +++ b/src/types.jl @@ -1,12 +1,9 @@ # ── Shared constants ─────────────────────────────────────────────────────────── -const LIBRARY = "Modelica" -const LIBRARY_VERSION = "4.1.0" - -# Comparison tolerances (2 % relative, 1e-6 absolute — matches Modelica +# Comparison tolerances (2 % relative, 1e-4 absolute — matches Modelica # Association compliance tooling defaults). const CMP_REL_TOL = 0.02 -const CMP_ABS_TOL = 1e-6 +const CMP_ABS_TOL = 1e-4 # CSV files larger than this limit are not committed to gh-pages (GitHub # enforces a 100 MB hard cap; we use a conservative 20 MB soft limit). @@ -20,21 +17,12 @@ const CSV_MAX_SIZE_MB = 20 Mutable configuration struct for signal comparison. # Fields -- `rel_tol` — maximum allowed relative error (default: `$(CMP_REL_TOL)`, i.e. 2 %). -- `abs_tol` — hard absolute-error floor used when signals are near zero - (default: `$(CMP_ABS_TOL)`). -- `error_fn` — selects the point-wise pass/fail function. One of: - - `:mixed` — scale-aware relative error (default, recommended); - - `:relative` — classic relative error (may reject valid zero-crossing signals); - - `:absolute` — pure absolute error. - -Use `configure_comparison!` to update the module-level defaults, or construct a -local instance to pass to `compare_with_reference` for a single run. +- `abs_tol` — hard absolute-error (default: `nothing`) +- `rel_tol` — maximum allowed globally scaled relative error (default: `$(CMP_REL_TOL)`, i.e. 2 %). """ Base.@kwdef mutable struct CompareSettings + abs_tol :: Union{Float64,Nothing} = nothing rel_tol :: Float64 = CMP_REL_TOL - abs_tol :: Float64 = CMP_ABS_TOL - error_fn :: Symbol = :mixed end # ── Run metadata ─────────────────────────────────────────────────────────────── From 2a7dd3bcd2a416528b2404e906584464e2ff0f6b Mon Sep 17 00:00:00 2001 From: AnHeuermann <38031952+AnHeuermann@users.noreply.github.com> Date: Tue, 17 Mar 2026 17:20:21 +0100 Subject: [PATCH 2/3] Undo update of abs tol --- src/types.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.jl b/src/types.jl index 1666ae666..64418f150 100644 --- a/src/types.jl +++ b/src/types.jl @@ -3,7 +3,7 @@ # Comparison tolerances (2 % relative, 1e-4 absolute — matches Modelica # Association compliance tooling defaults). const CMP_REL_TOL = 0.02 -const CMP_ABS_TOL = 1e-4 +const CMP_ABS_TOL = 1e-6 # CSV files larger than this limit are not committed to gh-pages (GitHub # enforces a 100 MB hard cap; we use a conservative 20 MB soft limit). From c4f808582926759750551d5fdb30a417944f6f68 Mon Sep 17 00:00:00 2001 From: AnHeuermann <38031952+AnHeuermann@users.noreply.github.com> Date: Wed, 18 Mar 2026 16:53:40 +0100 Subject: [PATCH 3/3] Add cmp_time field, signal pre-filtering, and simulation CSV optimizations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add `cmp_time` to `ModelResult` to track reference comparison wall time; report it in the per-model status line - Read comparison signals in `pipeline.jl` before phase 3 and pass them to both `run_simulate` and `compare_with_reference`, avoiding redundant file reads and enabling observed-variable filtering in the CSV writer - Filter observed variables written to the simulation CSV to only those needed for comparison, keeping large models (e.g. LightningSegmented- TransmissionLine) within the CSV size limit - Write CSV at fixed intervals (from `ode_prob.kwargs[:saveat]` or tspan/500) instead of at every solver step; use dense interpolation for accuracy - Fix `Nothing → Float64` for models where BaseModelica sets `saveat=nothing` by using `something(get(..., nothing), default)` - Accept caller-supplied signal list in `compare_with_reference`; fall back to `comparisonSignals.txt` / all ref-CSV columns when none supplied - Delete versioned results directory before rsync in CI to remove stale files - Use qualified `Dates.now()` / `DifferentialEquations.*` after switching to module-level imports Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/msl-test.yml | 6 +++- src/BaseModelicaLibraryTesting.jl | 4 +-- src/compare.jl | 30 ++++++++++++------- src/pipeline.jl | 48 ++++++++++++++++++++++--------- src/report.jl | 4 +-- src/simulate.jl | 37 ++++++++++++++++-------- src/types.jl | 1 + 7 files changed, 88 insertions(+), 42 deletions(-) diff --git a/.github/workflows/msl-test.yml b/.github/workflows/msl-test.yml index 853bae16d..001fb493d 100644 --- a/.github/workflows/msl-test.yml +++ b/.github/workflows/msl-test.yml @@ -145,7 +145,11 @@ jobs: fi - name: Copy new results into gh-pages tree - run: rsync -a results/ site/results/ + env: + BM_VERSION: ${{ steps.versions.outputs.bm_version }} + run: | + rm -rf "site/results/${BM_VERSION}/${LIB_NAME}/${LIB_VERSION}" + rsync -a results/ site/results/ - name: Generate landing page run: python3 .github/scripts/gen_landing_page.py site diff --git a/src/BaseModelicaLibraryTesting.jl b/src/BaseModelicaLibraryTesting.jl index 71af4f1c5..493658aeb 100644 --- a/src/BaseModelicaLibraryTesting.jl +++ b/src/BaseModelicaLibraryTesting.jl @@ -4,9 +4,9 @@ import Pkg import OMJulia import OMJulia: sendExpression import BaseModelica -import DifferentialEquations: solve, Rodas5P, ReturnCode +import DifferentialEquations import ModelingToolkit -import Dates: now +import Dates import Printf: @sprintf include("types.jl") diff --git a/src/compare.jl b/src/compare.jl index df9f37e73..bf1176111 100644 --- a/src/compare.jl +++ b/src/compare.jl @@ -287,24 +287,32 @@ function compare_with_reference( ref_csv_path::String, model_dir::String, model::String; - settings::CompareSettings = CompareSettings(), + settings::CompareSettings = CompareSettings(), + signals::Vector{String} = String[], )::Tuple{Int,Int,Int,String} times, ref_data = _read_ref_csv(ref_csv_path) isempty(times) && return 0, 0, 0, "" - # Determine which signals to compare: prefer comparisonSignals.txt - sig_file = joinpath(dirname(ref_csv_path), "comparisonSignals.txt") - using_sig_file = isfile(sig_file) - signals = if using_sig_file - sigs = filter(s -> lowercase(s) != "time" && !isempty(s), strip.(readlines(sig_file))) - sigs_missing = filter(s -> !haskey(ref_data, s), sigs) - isempty(sigs_missing) || error("Signal(s) listed in comparisonSignals.txt not present in reference CSV: $(join(sigs_missing, ", "))") - sigs + # Determine which signals to compare. + # Prefer the caller-supplied list; fall back to comparisonSignals.txt, then + # all columns in the reference CSV. + signals = if !isempty(signals) + sigs_missing = filter(s -> !haskey(ref_data, s), signals) + isempty(sigs_missing) || error("Signal(s) not present in reference CSV: $(join(sigs_missing, ", "))") + signals else - filter(k -> lowercase(k) != "time", collect(keys(ref_data))) + sig_file = joinpath(dirname(ref_csv_path), "comparisonSignals.txt") + if isfile(sig_file) + sigs = String.(filter(s -> lowercase(s) != "time" && !isempty(s), strip.(readlines(sig_file)))) + sigs_missing = filter(s -> !haskey(ref_data, s), sigs) + isempty(sigs_missing) || error("Signal(s) listed in comparisonSignals.txt not present in reference CSV: $(join(sigs_missing, ", "))") + sigs + else + filter(k -> lowercase(k) != "time", collect(keys(ref_data))) + end end - n_total = length(signals) + n_total = length(signals) # ── Build variable accessor map ────────────────────────────────────────────── # var_access: normalized name → Int (state index) or MTK symbolic (observed). diff --git a/src/pipeline.jl b/src/pipeline.jl index ed8b21989..7d7807773 100644 --- a/src/pipeline.jl +++ b/src/pipeline.jl @@ -73,27 +73,45 @@ function test_model(omc::OMJulia.OMCSession, # Phase 1 ────────────────────────────────────────────────────────────────── exp_ok, exp_t, exp_err = run_export(omc, model, model_dir, bm_path) exp_ok || return ModelResult( - model, false, exp_t, exp_err, false, 0.0, "", false, 0.0, "", 0, 0, 0, "") + model, false, exp_t, exp_err, false, 0.0, "", false, 0.0, "", 0, 0, 0, 0.0, "") # Phase 2 ────────────────────────────────────────────────────────────────── par_ok, par_t, par_err, ode_prob = run_parse(bm_path, model_dir, model) par_ok || return ModelResult( - model, true, exp_t, exp_err, false, par_t, par_err, false, 0.0, "", 0, 0, 0, "") + model, true, exp_t, exp_err, false, par_t, par_err, false, 0.0, "", 0, 0, 0, 0.0, "") + + # Resolve reference CSV and comparison signals early so phase 3 can filter + # the CSV output to only the signals that will actually be verified. + ref_csv = isempty(ref_root) ? nothing : _ref_csv_path(ref_root, model) + cmp_signals = if ref_csv !== nothing + sig_file = joinpath(dirname(ref_csv), "comparisonSignals.txt") + if isfile(sig_file) + String.(filter(s -> lowercase(s) != "time" && !isempty(s), strip.(readlines(sig_file)))) + else + _, ref_data = _read_ref_csv(ref_csv) + filter(k -> lowercase(k) != "time", collect(keys(ref_data))) + end + else + String[] + end # Phase 3 ────────────────────────────────────────────────────────────────── - sim_ok, sim_t, sim_err, sol = run_simulate(ode_prob, model_dir, model; csv_max_size_mb) + sim_ok, sim_t, sim_err, sol = run_simulate(ode_prob, model_dir, model; + csv_max_size_mb, cmp_signals) # Phase 4 (optional) ─────────────────────────────────────────────────────── cmp_total, cmp_pass, cmp_skip, cmp_csv = 0, 0, 0, "" - if sim_ok && !isempty(ref_root) - ref_csv = _ref_csv_path(ref_root, model) - if ref_csv !== nothing - try - cmp_total, cmp_pass, cmp_skip, cmp_csv = - compare_with_reference(sol, ref_csv, model_dir, model; settings) - catch e - @warn "Reference comparison failed for $model: $(sprint(showerror, e))" - end + cmp_t = 0.0 + if sim_ok && ref_csv !== nothing + try + t0_cmp = time() + cmp_total, cmp_pass, cmp_skip, cmp_csv = + compare_with_reference(sol, ref_csv, model_dir, model; + settings, signals = cmp_signals) + cmp_t = time() - t0_cmp + catch e + cmp_t = time() - t0_cmp + @warn "Reference comparison failed for $model: $(sprint(showerror, e))" end end @@ -102,7 +120,7 @@ function test_model(omc::OMJulia.OMCSession, true, exp_t, exp_err, true, par_t, par_err, sim_ok, sim_t, sim_err, - cmp_total, cmp_pass, cmp_skip, cmp_csv) + cmp_total, cmp_pass, cmp_skip, cmp_t, cmp_csv) end # ── Main ─────────────────────────────────────────────────────────────────────── @@ -207,7 +225,9 @@ function main(; end cmp_info = if result.cmp_total > 0 skip_note = result.cmp_skip > 0 ? " skip=$(result.cmp_skip)" : "" - " cmp=$(result.cmp_pass)/$(result.cmp_total)$skip_note" + " cmp=$(result.cmp_pass)/$(result.cmp_total)$skip_note ($(round(result.cmp_time;digits=2))s)" + elseif result.cmp_time > 0 + " cmp=n/a ($(round(result.cmp_time;digits=2))s)" else "" end diff --git a/src/report.jl b/src/report.jl index 63f81b320..882c7d6f6 100644 --- a/src/report.jl +++ b/src/report.jl @@ -1,6 +1,6 @@ # ── HTML report generation ───────────────────────────────────────────────────── -import Dates: now +import Dates import Printf: @sprintf function _status_cell(ok::Bool, t::Float64, logFile::Union{String,Nothing}) @@ -145,7 +145,7 @@ function generate_report(results::Vector{ModelResult}, results_root::String,

$(info.library) $(info.lib_version) — Base Modelica / MTK Pipeline Test Results

-

Generated: $(now())
+

Generated: $(Dates.now())
OpenModelica: $(info.omc_version)
OMC options: $(info.omc_options)
BaseModelica.jl: $(basemodelica_jl_version)
diff --git a/src/simulate.jl b/src/simulate.jl index 18c40923e..d4a1632bc 100644 --- a/src/simulate.jl +++ b/src/simulate.jl @@ -1,6 +1,6 @@ # ── Phase 3: ODE simulation with DifferentialEquations / MTK ────────────────── -import DifferentialEquations: solve, Rodas5P, ReturnCode +import DifferentialEquations import Logging import ModelingToolkit import Printf: @sprintf @@ -9,15 +9,17 @@ import Printf: @sprintf run_simulate(ode_prob, model_dir, model; csv_max_size_mb) → (success, time, error, sol) Solve `ode_prob` with Rodas5P (stiff solver). On success, also writes the -full solution as a CSV file `_sim.csv` in `model_dir`. +solution as a CSV file `_sim.csv` in `model_dir`. Writes a `_sim.log` file in `model_dir`. Returns `nothing` as the fourth element on failure. CSV files larger than `csv_max_size_mb` MiB are deleted and replaced with a `_sim.csv.toobig` marker so that the report can note the omission. """ -function run_simulate(ode_prob, model_dir::String, model::String; - csv_max_size_mb::Int = CSV_MAX_SIZE_MB)::Tuple{Bool,Float64,String,Any} +function run_simulate(ode_prob, model_dir::String, + model::String; + cmp_signals ::Vector{String} = String[], + csv_max_size_mb::Int = CSV_MAX_SIZE_MB)::Tuple{Bool,Float64,String,Any} sim_success = false sim_time = 0.0 sim_error = "" @@ -27,16 +29,20 @@ function run_simulate(ode_prob, model_dir::String, model::String; println(log_file, "Model: $model") logger = Logging.SimpleLogger(log_file, Logging.Debug) t0 = time() + + # Read interval before overwriting it + interval = something(get(ode_prob.kwargs, :saveat, nothing), + (ode_prob.tspan[end] - ode_prob.tspan[1]) / 500) + try # Rodas5P handles stiff DAE-like systems well. # Redirect all library log output (including Symbolics/MTK warnings) # to the log file so they don't clutter stdout. sol = Logging.with_logger(logger) do - # Overwrite saveat, always use dense output. - solve(ode_prob, Rodas5P(); saveat = Float64[], dense = true) + DifferentialEquations.solve(ode_prob, DifferentialEquations.Rodas5P(); saveat = Float64[], dense = true) end sim_time = time() - t0 - if sol.retcode == ReturnCode.Success + if sol.retcode == DifferentialEquations.ReturnCode.Success sys = sol.prob.f.sys n_vars = length(ModelingToolkit.unknowns(sys)) n_obs = length(ModelingToolkit.observed(sys)) @@ -67,20 +73,28 @@ function run_simulate(ode_prob, model_dir::String, model::String; sys = sol.prob.f.sys vars = ModelingToolkit.unknowns(sys) obs_eqs = ModelingToolkit.observed(sys) - obs_syms = [eq.lhs for eq in obs_eqs] + # Only save observed variables that appear in cmp_signals. + # This avoids writing thousands of algebraic variables to disk when + # only a handful are actually verified during comparison. + norm_cmp = Set(_normalize_var(s) for s in cmp_signals) + obs_eqs_filtered = isempty(norm_cmp) ? obs_eqs : + filter(eq -> _normalize_var(string(eq.lhs)) in norm_cmp, obs_eqs) + obs_syms = [eq.lhs for eq in obs_eqs_filtered] col_names = vcat( [_clean_var_name(string(v)) for v in vars], [_clean_var_name(string(s)) for s in obs_syms], ) open(sim_csv, "w") do f println(f, join(["time"; col_names], ",")) - for (ti, t) in enumerate(sol.t) + t_csv = range(ode_prob.tspan[1], ode_prob.tspan[end]; step = interval) + for t in t_csv row = [@sprintf("%.10g", t)] + u = sol(Float64(t)) for vi in eachindex(vars) - push!(row, @sprintf("%.10g", sol[vi, ti])) + push!(row, @sprintf("%.10g", u[vi])) end for sym in obs_syms - val = try Float64(sol(t; idxs = sym)) catch; NaN end + val = try Float64(sol(Float64(t); idxs = sym)) catch; NaN end push!(row, @sprintf("%.10g", val)) end println(f, join(row, ",")) @@ -90,7 +104,6 @@ function run_simulate(ode_prob, model_dir::String, model::String; if csv_bytes > csv_max_size_mb * 1024^2 csv_mb = round(csv_bytes / 1024^2; digits=1) @warn "Simulation CSV for $model is $(csv_mb) MB (> $(csv_max_size_mb) MB limit); skipping." - rm(sim_csv) write(sim_csv * ".toobig", string(csv_bytes)) end catch e diff --git a/src/types.jl b/src/types.jl index 64418f150..f4fbdb011 100644 --- a/src/types.jl +++ b/src/types.jl @@ -82,5 +82,6 @@ struct ModelResult cmp_total :: Int # signals actually compared (found in simulation) cmp_pass :: Int cmp_skip :: Int # reference signals not found in simulation + cmp_time :: Float64 # wall time for comparison phase (0.0 if skipped) cmp_csv :: String # absolute path to diff CSV; "" if all pass or no comparison end