Skip to content

Commit 5f95ebd

Browse files
AnHeuermannclaude
andauthored
Improve comparison coverage, reporting, and HTML diff page (#20)
- Expand variable matching to include MTK observed (algebraically eliminated) variables via sol(t; idxs=sym), raising coverage beyond state variables alone - Track skipped signals (reference vars not found in simulation) and surface them in a new cmp_skip field on ModelResult - Add variable-coverage table to _diff.html listing every reference signal as pass/fail/not-found; move table above the charts - Colour index.html "Ref Cmp" cells orange (partial) when all compared signals pass but some reference variables were not found - Always link the sim CSV from the "Ref Cmp" cell; show "(CSV N/A)" when the file was skipped for exceeding CSV_MAX_SIZE_MB - Fix time-column lookup in reference CSVs to be case-insensitive ("Time" vs "time"), fixing missing comparisons for some models - Replace fixed meta placeholders in diff_template.html with a single META_BLOCK substitution; guard JS against empty CSV data Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 6dc1f2c commit 5f95ebd

5 files changed

Lines changed: 236 additions & 100 deletions

File tree

assets/diff_template.html

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,15 @@
1919
</head>
2020
<body>
2121
<h1>Diff &#x2014; {{MODEL}}</h1>
22-
<p class="meta">
23-
{{N_FAIL}} signal(s) outside tolerance
24-
(rel &#x2264; {{REL_TOL_PCT}}%,&nbsp; abs &#x2264; {{ABS_TOL}})
25-
&nbsp;&middot;&nbsp;
26-
<a href="{{CSV_NAME}}">Download CSV</a>
27-
</p>
22+
{{META_BLOCK}}
23+
{{VAR_TABLE}}
2824
<div id="charts"></div>
2925
<script>
3026
var csvRaw = `{{CSV_DATA}}`;
3127

3228
(function () {
33-
var lines = csvRaw.trim().split('\n');
29+
var lines = csvRaw.trim().split('\n');
30+
if (lines.length < 2) return;
3431
var headers = lines[0].split(',').map(function (s) { return s.trim(); });
3532
var rows = lines.slice(1).filter(function (l) { return l.trim() !== ''; })
3633
.map(function (l) { return l.split(',').map(Number); });

src/compare.jl

Lines changed: 171 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,8 @@ function _read_ref_csv(path::String)::Tuple{Vector{Float64}, Dict{String,Vector{
7575
end
7676
end
7777

78-
times = get(data, "time", Float64[])
78+
time_key = something(findfirst(h -> lowercase(h) == "time", headers), nothing)
79+
times = time_key === nothing ? Float64[] : data[headers[time_key]]
7980
return times, data
8081
end
8182

@@ -218,74 +219,138 @@ compare_settings() = _CMP_SETTINGS
218219
# ── Interactive diff HTML ──────────────────────────────────────────────────────
219220

220221
"""
221-
write_diff_html(diff_csv_path, model)
222+
write_diff_html(model_dir, model; diff_csv_path, pass_sigs, skip_sigs)
222223
223-
Generate an interactive HTML page for a `_diff.csv` file using the bundled
224-
Dygraphs library. One zoomable chart is created per failing signal, showing
225-
the reference trace, the simulation trace, and the relative error on a second
226-
y-axis. The HTML file is written next to the CSV with a `.html` extension.
224+
Generate an interactive HTML page for a comparison result using the bundled
225+
Dygraphs library. The page always includes a variable-coverage table listing
226+
every reference signal and whether it was found in the simulation. When there
227+
are failing signals a zoomable chart is added per signal showing the reference
228+
trace, the simulation trace, and the relative error on a second y-axis.
227229
228-
The page references `../../assets/dygraph.min.*` relative to its location
229-
(`<results_root>/files/<model>/`). `_install_assets` is called automatically
230-
to copy the library files to `<results_root>/assets/` if not already present.
231-
"""
232-
function write_diff_html(diff_csv_path::String, model::String)
233-
lines = readlines(diff_csv_path)
234-
isempty(lines) && return
230+
`model_dir` is `<results_root>/files/<model>`. The HTML is written to
231+
`<model_dir>/<short>_diff.html`. `diff_csv_path` is the absolute path to the
232+
diff CSV (empty string when all comparable signals pass).
235233
236-
headers = [replace(strip(h), "\"" => "") for h in split(lines[1], ",")]
234+
The page references `../../assets/dygraph.min.*` relative to its location.
235+
`_install_assets` is called automatically.
236+
"""
237+
function write_diff_html(model_dir::String, model::String;
238+
diff_csv_path::String = "",
239+
pass_sigs::Vector{String} = String[],
240+
skip_sigs::Vector{String} = String[])
241+
short_name = split(model, ".")[end]
242+
html_path = joinpath(model_dir, "$(short_name)_diff.html")
243+
results_root = dirname(dirname(abspath(model_dir))) # …/files/<model> → …
244+
_install_assets(results_root)
237245

238-
# Extract unique signal names from headers like "C1.v_ref", "C1.v_sim", …
246+
# Read fail_sigs and CSV content from the diff CSV (may not exist).
239247
fail_sigs = String[]
240-
for h in headers
241-
if length(h) > 4 && h[end-3:end] == "_ref"
242-
push!(fail_sigs, h[1:end-4])
248+
csv_js = ""
249+
if !isempty(diff_csv_path) && isfile(diff_csv_path)
250+
lines = readlines(diff_csv_path)
251+
if length(lines) >= 1
252+
headers = [replace(strip(h), "\"" => "") for h in split(lines[1], ",")]
253+
for h in headers
254+
length(h) > 4 && h[end-3:end] == "_ref" && push!(fail_sigs, h[1:end-4])
255+
end
256+
csv_text = read(diff_csv_path, String)
257+
csv_js = replace(replace(csv_text, "\\" => "\\\\"), "`" => "\\`")
243258
end
244259
end
245-
isempty(fail_sigs) && return
246260

247-
short_name = split(model, ".")[end]
248-
249-
# Escape CSV content for embedding as a JS template literal.
250-
# The only characters that would break a template literal are \ and `.
251-
csv_text = read(diff_csv_path, String)
252-
csv_js = replace(replace(csv_text, "\\" => "\\\\"), "`" => "\\`")
253-
254-
# Derive results_root from diff_csv_path: <results_root>/files/<model>/<file>
255-
results_root = dirname(dirname(dirname(abspath(diff_csv_path))))
256-
_install_assets(results_root)
261+
# ── Meta block ──────────────────────────────────────────────────────────────
262+
tol_str = "(rel &#x2264; $(round(Int, _CMP_SETTINGS.rel_tol * 100))%," *
263+
" abs &#x2264; $(_CMP_SETTINGS.abs_tol))"
264+
csv_link = isempty(fail_sigs) ? "" :
265+
""" &nbsp;&middot;&nbsp; <a href="$(short_name)_diff.csv">Download diff CSV</a>"""
266+
skip_note = isempty(skip_sigs) ? "" :
267+
""" &nbsp;&middot;&nbsp; $(length(skip_sigs)) signal(s) not found in simulation"""
268+
meta_block = """<p class="meta">$(length(fail_sigs)) signal(s) outside tolerance """ *
269+
"""$tol_str$(skip_note)$(csv_link)</p>"""
270+
271+
# ── Variable-coverage table ──────────────────────────────────────────────────
272+
all_sigs = vcat(pass_sigs, fail_sigs, skip_sigs)
273+
var_table = if isempty(all_sigs)
274+
""
275+
else
276+
n_found = length(pass_sigs) + length(fail_sigs)
277+
n_total = n_found + length(skip_sigs)
278+
th = "border:1px solid #ccc;padding:3px 10px;background:#eee;text-align:left;"
279+
td = "border:1px solid #ccc;padding:3px 10px;"
280+
rows = String[]
281+
for sig in pass_sigs
282+
push!(rows, "<tr style=\"background:#d4edda\"><td style=\"$td\">$sig</td>" *
283+
"<td style=\"$td\">&#10003; pass</td></tr>")
284+
end
285+
for sig in fail_sigs
286+
push!(rows, "<tr style=\"background:#f8d7da\"><td style=\"$td\">$sig</td>" *
287+
"<td style=\"$td\">&#10007; fail</td></tr>")
288+
end
289+
for sig in skip_sigs
290+
push!(rows, "<tr style=\"background:#fff3cd\"><td style=\"$td\">$sig</td>" *
291+
"<td style=\"$td\">not found in simulation</td></tr>")
292+
end
293+
"""<h2 style="font-size:1.1em;margin-top:2em;">Variable Coverage """ *
294+
"""&#x2014; $n_found of $n_total reference signal(s) found</h2>""" *
295+
"""<table style="border-collapse:collapse;font-size:13px;">""" *
296+
"""<thead><tr><th style="$th">Signal</th><th style="$th">Status</th></tr></thead>""" *
297+
"""<tbody>$(join(rows))</tbody></table>"""
298+
end
257299

258-
# Fill template placeholders
300+
# ── Fill template ────────────────────────────────────────────────────────────
259301
template = read(joinpath(_ASSETS_DIR, "diff_template.html"), String)
260302
html = replace(
261303
template,
262304
"{{TITLE}}" => short_name,
263305
"{{MODEL}}" => model,
264-
"{{N_FAIL}}" => string(length(fail_sigs)),
265-
"{{REL_TOL_PCT}}" => string(round(Int, _CMP_SETTINGS.rel_tol * 100)),
266-
"{{ABS_TOL}}" => string(_CMP_SETTINGS.abs_tol),
267-
"{{CSV_NAME}}" => "$(short_name)_diff.csv",
306+
"{{META_BLOCK}}" => meta_block,
268307
"{{CSV_DATA}}" => csv_js,
308+
"{{VAR_TABLE}}" => var_table,
269309
)
270-
271-
html_path = replace(diff_csv_path, r"\.csv$" => ".html")
272310
write(html_path, html)
273311
end
274312

275313
# ── Reference comparison ───────────────────────────────────────────────────────
276314

315+
"""
316+
_eval_sim(sol, accessor, t) → Float64
317+
318+
Evaluate the simulation solution at time `t` for a single signal. `accessor`
319+
is either an `Int` (index into the state vector, for unknowns) or an MTK
320+
symbolic variable (for observed variables, evaluated via `sol(t; idxs=sym)`).
321+
Returns `NaN` if the observed-variable evaluation fails.
322+
"""
323+
function _eval_sim(sol, accessor, t::Float64)::Float64
324+
if accessor isa Integer
325+
return Float64(sol(t)[accessor])
326+
else
327+
try
328+
return Float64(sol(t; idxs = accessor))
329+
catch
330+
return NaN
331+
end
332+
end
333+
end
334+
277335
"""
278336
compare_with_reference(sol, ref_csv_path, model_dir, model;
279-
settings) → (total, pass, diff_csv)
337+
settings) → (total, pass, skip, diff_csv)
280338
281-
Compare a DifferentialEquations solution against the MAP-LIB reference CSV.
339+
Compare a DifferentialEquations / MTK solution against the MAP-LIB reference CSV.
282340
283341
Returns:
284-
total — number of signals successfully compared
285-
pass — number of signals within tolerance
342+
total — number of reference signals successfully compared
343+
pass — number of compared signals within tolerance
344+
skip — number of reference signals not found in the simulation
286345
diff_csv — absolute path to the written diff CSV (empty string if all pass)
287346
288-
Signals that cannot be matched to an MTK state variable are skipped.
347+
The lookup covers both MTK state variables (`ModelingToolkit.unknowns`) and
348+
observed (algebraically eliminated) variables (`ModelingToolkit.observed`),
349+
so signals that MTK removed during structural simplification are still matched
350+
and compared via continuous interpolation of the observed function.
351+
352+
A `_diff.html` detail page with zoomable charts and a variable-coverage table
353+
is written whenever there are failures or skipped signals.
289354
290355
# Keyword arguments
291356
- `settings` — a `CompareSettings` instance controlling tolerances and the
@@ -299,83 +364,111 @@ function compare_with_reference(
299364
model_dir::String,
300365
model::String;
301366
settings::CompareSettings = _CMP_SETTINGS,
302-
)::Tuple{Int,Int,String}
367+
)::Tuple{Int,Int,Int,String}
303368

304369
times, ref_data = _read_ref_csv(ref_csv_path)
305-
isempty(times) && return 0, 0, ""
370+
isempty(times) && return 0, 0, 0, ""
306371

307372
# Determine which signals to compare: prefer comparisonSignals.txt
308373
sig_file = joinpath(dirname(ref_csv_path), "comparisonSignals.txt")
309374
signals = if isfile(sig_file)
310-
filter(s -> s != "time" && !isempty(s), strip.(readlines(sig_file)))
375+
filter(s -> lowercase(s) != "time" && !isempty(s), strip.(readlines(sig_file)))
311376
else
312-
filter(k -> k != "time", collect(keys(ref_data)))
377+
filter(k -> lowercase(k) != "time", collect(keys(ref_data)))
313378
end
314379

315-
# Build normalized-name → state-variable-index map from the MTK system
316-
sys = sol.prob.f.sys
317-
vars = ModelingToolkit.unknowns(sys)
318-
var_norm = Dict(_normalize_var(string(v)) => i for (i, v) in enumerate(vars))
380+
# ── Build variable accessor map ──────────────────────────────────────────────
381+
# var_access: normalized name → Int (state index) or MTK symbolic (observed).
382+
# State variables come first so they take priority over any observed alias.
383+
sys = sol.prob.f.sys
384+
var_access = Dict{String,Any}()
385+
for (i, v) in enumerate(ModelingToolkit.unknowns(sys))
386+
var_access[_normalize_var(string(v))] = i
387+
end
388+
# Observed variables: algebraically eliminated by structural_simplify.
389+
# MTK solution objects support sol(t; idxs=sym) for these via SciML's
390+
# SymbolicIndexingInterface, so they can be interpolated like state vars.
391+
try
392+
for eq in ModelingToolkit.observed(sys)
393+
name = _normalize_var(string(eq.lhs))
394+
haskey(var_access, name) || (var_access[name] = eq.lhs)
395+
end
396+
catch e
397+
@warn "Could not enumerate observed variables: $(sprint(showerror, e))"
398+
end
319399

320-
# Clip reference time points to the simulation interval
400+
# Clip reference time to the simulation interval
321401
t_start = sol.t[1]
322402
t_end = sol.t[end]
323403
valid_mask = (times .>= t_start) .& (times .<= t_end)
324404
t_ref = times[valid_mask]
325-
isempty(t_ref) && return 0, 0, ""
405+
isempty(t_ref) && return 0, 0, 0, ""
326406

327-
n_total = 0
328-
n_pass = 0
329-
fail_sigs = String[]
330-
fail_scales = Dict{String,Float64}() # peak |ref| per failing signal
407+
n_total = 0
408+
n_pass = 0
409+
pass_sigs = String[]
410+
fail_sigs = String[]
411+
skip_sigs = String[]
412+
fail_scales = Dict{String,Float64}()
331413

332414
for sig in signals
333-
haskey(ref_data, sig) || continue
334-
haskey(var_norm, _normalize_var(sig)) || continue # skip non-state signals
415+
haskey(ref_data, sig) || continue # signal absent from ref CSV entirely
335416

417+
norm = _normalize_var(sig)
418+
if !haskey(var_access, norm)
419+
push!(skip_sigs, sig)
420+
continue
421+
end
422+
423+
accessor = var_access[norm]
336424
ref_vals = ref_data[sig][valid_mask]
337-
idx = var_norm[_normalize_var(sig)]
338425
n_total += 1
339426

340-
# Peak magnitude of the reference signal — used as the absolute-error scale
341-
# near zero crossings so that relative error does not blow up.
427+
# Peak |ref| — used as amplitude floor so relative error stays finite
428+
# near zero crossings.
342429
ref_scale = isempty(ref_vals) ? 0.0 : maximum(abs, ref_vals)
343430

344-
# Interpolate simulation at reference time points
345-
sim_vals = [sol(t)[idx] for t in t_ref]
431+
# Interpolate simulation at reference time points.
432+
sim_vals = [_eval_sim(sol, accessor, t) for t in t_ref]
433+
434+
# If evaluation returned NaN (observed-var access failed), treat as skip.
435+
if any(isnan, sim_vals)
436+
n_total -= 1
437+
push!(skip_sigs, sig)
438+
continue
439+
end
346440

347441
pass = all(zip(sim_vals, ref_vals)) do (s, r)
348442
_check_point(s, r, ref_scale, settings)
349443
end
350444

351445
if pass
352446
n_pass += 1
447+
push!(pass_sigs, sig)
353448
else
354449
push!(fail_sigs, sig)
355450
fail_scales[sig] = ref_scale
356451
end
357452
end
358453

359-
# Write diff CSV for failing signals (wide format: ref + sim + relerr per signal)
360-
diff_csv = ""
454+
# ── Write diff CSV for failing signals ──────────────────────────────────────
455+
# Wide format: time, <sig>_ref, <sig>_sim, <sig>_relerr per failing signal.
456+
short_name = split(model, ".")[end]
457+
diff_csv = ""
361458
if !isempty(fail_sigs)
362-
short_name = split(model, ".")[end]
363-
diff_csv = joinpath(model_dir, "$(short_name)_diff.csv")
364-
459+
diff_csv = joinpath(model_dir, "$(short_name)_diff.csv")
365460
open(diff_csv, "w") do f
366461
cols = ["time"]
367462
for sig in fail_sigs
368463
push!(cols, "$(sig)_ref", "$(sig)_sim", "$(sig)_relerr")
369464
end
370465
println(f, join(cols, ","))
371-
372466
for (ti, t) in enumerate(t_ref)
373467
row = [@sprintf("%.10g", t)]
374468
for sig in fail_sigs
375469
ref_vals = ref_data[sig][valid_mask]
376470
r = ref_vals[ti]
377-
idx = var_norm[_normalize_var(sig)]
378-
s = sol(t)[idx]
471+
s = _eval_sim(sol, var_access[_normalize_var(sig)], t)
379472
ref_scale = get(fail_scales, sig, 0.0)
380473
relerr = abs(s - r) / max(abs(r), ref_scale, settings.abs_tol)
381474
push!(row, @sprintf("%.10g", r),
@@ -385,9 +478,15 @@ function compare_with_reference(
385478
println(f, join(row, ","))
386479
end
387480
end
481+
end
388482

389-
write_diff_html(diff_csv, model)
483+
# ── Write detail HTML whenever there is anything worth showing ───────────────
484+
if !isempty(fail_sigs) || !isempty(skip_sigs)
485+
write_diff_html(model_dir, model;
486+
diff_csv_path = diff_csv,
487+
pass_sigs = pass_sigs,
488+
skip_sigs = skip_sigs)
390489
end
391490

392-
return n_total, n_pass, diff_csv
491+
return n_total, n_pass, length(skip_sigs), diff_csv
393492
end

0 commit comments

Comments
 (0)