@@ -75,7 +75,8 @@ function _read_ref_csv(path::String)::Tuple{Vector{Float64}, Dict{String,Vector{
7575 end
7676 end
7777
78- times = get (data, " time" , Float64[])
78+ time_key = something (findfirst (h -> lowercase (h) == " time" , headers), nothing )
79+ times = time_key === nothing ? Float64[] : data[headers[time_key]]
7980 return times, data
8081end
8182
@@ -218,74 +219,138 @@ compare_settings() = _CMP_SETTINGS
218219# ── Interactive diff HTML ──────────────────────────────────────────────────────
219220
220221"""
221- write_diff_html(diff_csv_path , model)
222+ write_diff_html(model_dir , model; diff_csv_path, pass_sigs, skip_sigs )
222223
223- Generate an interactive HTML page for a `_diff.csv` file using the bundled
224- Dygraphs library. One zoomable chart is created per failing signal, showing
225- the reference trace, the simulation trace, and the relative error on a second
226- y-axis. The HTML file is written next to the CSV with a `.html` extension.
224+ Generate an interactive HTML page for a comparison result using the bundled
225+ Dygraphs library. The page always includes a variable-coverage table listing
226+ every reference signal and whether it was found in the simulation. When there
227+ are failing signals a zoomable chart is added per signal showing the reference
228+ trace, the simulation trace, and the relative error on a second y-axis.
227229
228- The page references `../../assets/dygraph.min.*` relative to its location
229- (`<results_root>/files/<model>/`). `_install_assets` is called automatically
230- to copy the library files to `<results_root>/assets/` if not already present.
231- """
232- function write_diff_html (diff_csv_path:: String , model:: String )
233- lines = readlines (diff_csv_path)
234- isempty (lines) && return
230+ `model_dir` is `<results_root>/files/<model>`. The HTML is written to
231+ `<model_dir>/<short>_diff.html`. `diff_csv_path` is the absolute path to the
232+ diff CSV (empty string when all comparable signals pass).
235233
236- headers = [replace (strip (h), " \" " => " " ) for h in split (lines[1 ], " ," )]
234+ The page references `../../assets/dygraph.min.*` relative to its location.
235+ `_install_assets` is called automatically.
236+ """
237+ function write_diff_html (model_dir:: String , model:: String ;
238+ diff_csv_path:: String = " " ,
239+ pass_sigs:: Vector{String} = String[],
240+ skip_sigs:: Vector{String} = String[])
241+ short_name = split (model, " ." )[end ]
242+ html_path = joinpath (model_dir, " $(short_name) _diff.html" )
243+ results_root = dirname (dirname (abspath (model_dir))) # …/files/<model> → …
244+ _install_assets (results_root)
237245
238- # Extract unique signal names from headers like "C1.v_ref", "C1.v_sim", …
246+ # Read fail_sigs and CSV content from the diff CSV (may not exist).
239247 fail_sigs = String[]
240- for h in headers
241- if length (h) > 4 && h[end - 3 : end ] == " _ref"
242- push! (fail_sigs, h[1 : end - 4 ])
248+ csv_js = " "
249+ if ! isempty (diff_csv_path) && isfile (diff_csv_path)
250+ lines = readlines (diff_csv_path)
251+ if length (lines) >= 1
252+ headers = [replace (strip (h), " \" " => " " ) for h in split (lines[1 ], " ," )]
253+ for h in headers
254+ length (h) > 4 && h[end - 3 : end ] == " _ref" && push! (fail_sigs, h[1 : end - 4 ])
255+ end
256+ csv_text = read (diff_csv_path, String)
257+ csv_js = replace (replace (csv_text, " \\ " => " \\\\ " ), " `" => " \\ `" )
243258 end
244259 end
245- isempty (fail_sigs) && return
246260
247- short_name = split (model, " ." )[end ]
248-
249- # Escape CSV content for embedding as a JS template literal.
250- # The only characters that would break a template literal are \ and `.
251- csv_text = read (diff_csv_path, String)
252- csv_js = replace (replace (csv_text, " \\ " => " \\\\ " ), " `" => " \\ `" )
253-
254- # Derive results_root from diff_csv_path: <results_root>/files/<model>/<file>
255- results_root = dirname (dirname (dirname (abspath (diff_csv_path))))
256- _install_assets (results_root)
261+ # ── Meta block ──────────────────────────────────────────────────────────────
262+ tol_str = " (rel ≤ $(round (Int, _CMP_SETTINGS. rel_tol * 100 )) %," *
263+ " abs ≤ $(_CMP_SETTINGS. abs_tol) )"
264+ csv_link = isempty (fail_sigs) ? " " :
265+ """ · <a href="$(short_name) _diff.csv">Download diff CSV</a>"""
266+ skip_note = isempty (skip_sigs) ? " " :
267+ """ · $(length (skip_sigs)) signal(s) not found in simulation"""
268+ meta_block = """ <p class="meta">$(length (fail_sigs)) signal(s) outside tolerance """ *
269+ """ $tol_str$(skip_note)$(csv_link) </p>"""
270+
271+ # ── Variable-coverage table ──────────────────────────────────────────────────
272+ all_sigs = vcat (pass_sigs, fail_sigs, skip_sigs)
273+ var_table = if isempty (all_sigs)
274+ " "
275+ else
276+ n_found = length (pass_sigs) + length (fail_sigs)
277+ n_total = n_found + length (skip_sigs)
278+ th = " border:1px solid #ccc;padding:3px 10px;background:#eee;text-align:left;"
279+ td = " border:1px solid #ccc;padding:3px 10px;"
280+ rows = String[]
281+ for sig in pass_sigs
282+ push! (rows, " <tr style=\" background:#d4edda\" ><td style=\" $td \" >$sig </td>" *
283+ " <td style=\" $td \" >✓ pass</td></tr>" )
284+ end
285+ for sig in fail_sigs
286+ push! (rows, " <tr style=\" background:#f8d7da\" ><td style=\" $td \" >$sig </td>" *
287+ " <td style=\" $td \" >✗ fail</td></tr>" )
288+ end
289+ for sig in skip_sigs
290+ push! (rows, " <tr style=\" background:#fff3cd\" ><td style=\" $td \" >$sig </td>" *
291+ " <td style=\" $td \" >not found in simulation</td></tr>" )
292+ end
293+ """ <h2 style="font-size:1.1em;margin-top:2em;">Variable Coverage """ *
294+ """ — $n_found of $n_total reference signal(s) found</h2>""" *
295+ """ <table style="border-collapse:collapse;font-size:13px;">""" *
296+ """ <thead><tr><th style="$th ">Signal</th><th style="$th ">Status</th></tr></thead>""" *
297+ """ <tbody>$(join (rows)) </tbody></table>"""
298+ end
257299
258- # Fill template placeholders
300+ # ── Fill template ────────────────────────────────────────────────────────────
259301 template = read (joinpath (_ASSETS_DIR, " diff_template.html" ), String)
260302 html = replace (
261303 template,
262304 " {{TITLE}}" => short_name,
263305 " {{MODEL}}" => model,
264- " {{N_FAIL}}" => string (length (fail_sigs)),
265- " {{REL_TOL_PCT}}" => string (round (Int, _CMP_SETTINGS. rel_tol * 100 )),
266- " {{ABS_TOL}}" => string (_CMP_SETTINGS. abs_tol),
267- " {{CSV_NAME}}" => " $(short_name) _diff.csv" ,
306+ " {{META_BLOCK}}" => meta_block,
268307 " {{CSV_DATA}}" => csv_js,
308+ " {{VAR_TABLE}}" => var_table,
269309 )
270-
271- html_path = replace (diff_csv_path, r" \. csv$" => " .html" )
272310 write (html_path, html)
273311end
274312
275313# ── Reference comparison ───────────────────────────────────────────────────────
276314
315+ """
316+ _eval_sim(sol, accessor, t) → Float64
317+
318+ Evaluate the simulation solution at time `t` for a single signal. `accessor`
319+ is either an `Int` (index into the state vector, for unknowns) or an MTK
320+ symbolic variable (for observed variables, evaluated via `sol(t; idxs=sym)`).
321+ Returns `NaN` if the observed-variable evaluation fails.
322+ """
323+ function _eval_sim (sol, accessor, t:: Float64 ):: Float64
324+ if accessor isa Integer
325+ return Float64 (sol (t)[accessor])
326+ else
327+ try
328+ return Float64 (sol (t; idxs = accessor))
329+ catch
330+ return NaN
331+ end
332+ end
333+ end
334+
277335"""
278336 compare_with_reference(sol, ref_csv_path, model_dir, model;
279- settings) → (total, pass, diff_csv)
337+ settings) → (total, pass, skip, diff_csv)
280338
281- Compare a DifferentialEquations solution against the MAP-LIB reference CSV.
339+ Compare a DifferentialEquations / MTK solution against the MAP-LIB reference CSV.
282340
283341Returns:
284- total — number of signals successfully compared
285- pass — number of signals within tolerance
342+ total — number of reference signals successfully compared
343+ pass — number of compared signals within tolerance
344+ skip — number of reference signals not found in the simulation
286345 diff_csv — absolute path to the written diff CSV (empty string if all pass)
287346
288- Signals that cannot be matched to an MTK state variable are skipped.
347+ The lookup covers both MTK state variables (`ModelingToolkit.unknowns`) and
348+ observed (algebraically eliminated) variables (`ModelingToolkit.observed`),
349+ so signals that MTK removed during structural simplification are still matched
350+ and compared via continuous interpolation of the observed function.
351+
352+ A `_diff.html` detail page with zoomable charts and a variable-coverage table
353+ is written whenever there are failures or skipped signals.
289354
290355# Keyword arguments
291356- `settings` — a `CompareSettings` instance controlling tolerances and the
@@ -299,83 +364,111 @@ function compare_with_reference(
299364 model_dir:: String ,
300365 model:: String ;
301366 settings:: CompareSettings = _CMP_SETTINGS,
302- ):: Tuple{Int,Int,String}
367+ ):: Tuple{Int,Int,Int, String}
303368
304369 times, ref_data = _read_ref_csv (ref_csv_path)
305- isempty (times) && return 0 , 0 , " "
370+ isempty (times) && return 0 , 0 , 0 , " "
306371
307372 # Determine which signals to compare: prefer comparisonSignals.txt
308373 sig_file = joinpath (dirname (ref_csv_path), " comparisonSignals.txt" )
309374 signals = if isfile (sig_file)
310- filter (s -> s != " time" && ! isempty (s), strip .(readlines (sig_file)))
375+ filter (s -> lowercase (s) != " time" && ! isempty (s), strip .(readlines (sig_file)))
311376 else
312- filter (k -> k != " time" , collect (keys (ref_data)))
377+ filter (k -> lowercase (k) != " time" , collect (keys (ref_data)))
313378 end
314379
315- # Build normalized-name → state-variable-index map from the MTK system
316- sys = sol. prob. f. sys
317- vars = ModelingToolkit. unknowns (sys)
318- var_norm = Dict (_normalize_var (string (v)) => i for (i, v) in enumerate (vars))
380+ # ── Build variable accessor map ──────────────────────────────────────────────
381+ # var_access: normalized name → Int (state index) or MTK symbolic (observed).
382+ # State variables come first so they take priority over any observed alias.
383+ sys = sol. prob. f. sys
384+ var_access = Dict {String,Any} ()
385+ for (i, v) in enumerate (ModelingToolkit. unknowns (sys))
386+ var_access[_normalize_var (string (v))] = i
387+ end
388+ # Observed variables: algebraically eliminated by structural_simplify.
389+ # MTK solution objects support sol(t; idxs=sym) for these via SciML's
390+ # SymbolicIndexingInterface, so they can be interpolated like state vars.
391+ try
392+ for eq in ModelingToolkit. observed (sys)
393+ name = _normalize_var (string (eq. lhs))
394+ haskey (var_access, name) || (var_access[name] = eq. lhs)
395+ end
396+ catch e
397+ @warn " Could not enumerate observed variables: $(sprint (showerror, e)) "
398+ end
319399
320- # Clip reference time points to the simulation interval
400+ # Clip reference time to the simulation interval
321401 t_start = sol. t[1 ]
322402 t_end = sol. t[end ]
323403 valid_mask = (times .>= t_start) .& (times .<= t_end)
324404 t_ref = times[valid_mask]
325- isempty (t_ref) && return 0 , 0 , " "
405+ isempty (t_ref) && return 0 , 0 , 0 , " "
326406
327- n_total = 0
328- n_pass = 0
329- fail_sigs = String[]
330- fail_scales = Dict {String,Float64} () # peak |ref| per failing signal
407+ n_total = 0
408+ n_pass = 0
409+ pass_sigs = String[]
410+ fail_sigs = String[]
411+ skip_sigs = String[]
412+ fail_scales = Dict {String,Float64} ()
331413
332414 for sig in signals
333- haskey (ref_data, sig) || continue
334- haskey (var_norm, _normalize_var (sig)) || continue # skip non-state signals
415+ haskey (ref_data, sig) || continue # signal absent from ref CSV entirely
335416
417+ norm = _normalize_var (sig)
418+ if ! haskey (var_access, norm)
419+ push! (skip_sigs, sig)
420+ continue
421+ end
422+
423+ accessor = var_access[norm]
336424 ref_vals = ref_data[sig][valid_mask]
337- idx = var_norm[_normalize_var (sig)]
338425 n_total += 1
339426
340- # Peak magnitude of the reference signal — used as the absolute- error scale
341- # near zero crossings so that relative error does not blow up .
427+ # Peak |ref| — used as amplitude floor so relative error stays finite
428+ # near zero crossings.
342429 ref_scale = isempty (ref_vals) ? 0.0 : maximum (abs, ref_vals)
343430
344- # Interpolate simulation at reference time points
345- sim_vals = [sol (t)[idx] for t in t_ref]
431+ # Interpolate simulation at reference time points.
432+ sim_vals = [_eval_sim (sol, accessor, t) for t in t_ref]
433+
434+ # If evaluation returned NaN (observed-var access failed), treat as skip.
435+ if any (isnan, sim_vals)
436+ n_total -= 1
437+ push! (skip_sigs, sig)
438+ continue
439+ end
346440
347441 pass = all (zip (sim_vals, ref_vals)) do (s, r)
348442 _check_point (s, r, ref_scale, settings)
349443 end
350444
351445 if pass
352446 n_pass += 1
447+ push! (pass_sigs, sig)
353448 else
354449 push! (fail_sigs, sig)
355450 fail_scales[sig] = ref_scale
356451 end
357452 end
358453
359- # Write diff CSV for failing signals (wide format: ref + sim + relerr per signal)
360- diff_csv = " "
454+ # ── Write diff CSV for failing signals ──────────────────────────────────────
455+ # Wide format: time, <sig>_ref, <sig>_sim, <sig>_relerr per failing signal.
456+ short_name = split (model, " ." )[end ]
457+ diff_csv = " "
361458 if ! isempty (fail_sigs)
362- short_name = split (model, " ." )[end ]
363- diff_csv = joinpath (model_dir, " $(short_name) _diff.csv" )
364-
459+ diff_csv = joinpath (model_dir, " $(short_name) _diff.csv" )
365460 open (diff_csv, " w" ) do f
366461 cols = [" time" ]
367462 for sig in fail_sigs
368463 push! (cols, " $(sig) _ref" , " $(sig) _sim" , " $(sig) _relerr" )
369464 end
370465 println (f, join (cols, " ," ))
371-
372466 for (ti, t) in enumerate (t_ref)
373467 row = [@sprintf (" %.10g" , t)]
374468 for sig in fail_sigs
375469 ref_vals = ref_data[sig][valid_mask]
376470 r = ref_vals[ti]
377- idx = var_norm[_normalize_var (sig)]
378- s = sol (t)[idx]
471+ s = _eval_sim (sol, var_access[_normalize_var (sig)], t)
379472 ref_scale = get (fail_scales, sig, 0.0 )
380473 relerr = abs (s - r) / max (abs (r), ref_scale, settings. abs_tol)
381474 push! (row, @sprintf (" %.10g" , r),
@@ -385,9 +478,15 @@ function compare_with_reference(
385478 println (f, join (row, " ," ))
386479 end
387480 end
481+ end
388482
389- write_diff_html (diff_csv, model)
483+ # ── Write detail HTML whenever there is anything worth showing ───────────────
484+ if ! isempty (fail_sigs) || ! isempty (skip_sigs)
485+ write_diff_html (model_dir, model;
486+ diff_csv_path = diff_csv,
487+ pass_sigs = pass_sigs,
488+ skip_sigs = skip_sigs)
390489 end
391490
392- return n_total, n_pass, diff_csv
491+ return n_total, n_pass, length (skip_sigs), diff_csv
393492end
0 commit comments