test(fpga): F-3.2 add DDC cosim fuzz runner with seed sweep
A new SCENARIO_FUZZ branch in tb_ddc_cosim.v accepts +hex / +csv / +tag plusargs so an external runner can pick stimulus and output paths per iteration. The three path registers are widened to 4 kbit each so long temp-directory paths (e.g. /private/var/folders/...) do not overflow the MSB and emerge truncated — a real failure mode caught while writing this runner. test_ddc_cosim_fuzz.py is a pytest-driven fuzz harness: - Generates a random plausible radar scene per seed (1-4 targets with random range/velocity/RCS/phase, random noise level 0.5-6.0 LSB stddev) via radar_scene.generate_adc_samples, fully deterministic. - Compiles tb_ddc_cosim.v once per session (module-scope fixture), then runs vvp per seed. - Asserts sample-count bounds consistent with 4x CIC decimation, signed-18 range on every baseband I/Q word, and non-zero output (catches silent pipeline stalls). - Ships with two tiers: test_ddc_fuzz_fast (8 seeds, default CI) and test_ddc_fuzz_full (100 seeds, opt-in via -m slow) matching the audit ask. Registers the "slow" marker in pyproject.toml for the 100-seed opt-in.
This commit is contained in:
@@ -64,9 +64,11 @@ module tb_ddc_cosim;
|
|||||||
|
|
||||||
// Scenario selector (set via +define)
|
// Scenario selector (set via +define)
|
||||||
reg [255:0] scenario_name;
|
reg [255:0] scenario_name;
|
||||||
reg [1023:0] hex_file_path;
|
// Widened to 4 kbits (512 bytes) so fuzz-runner temp paths
|
||||||
reg [1023:0] csv_out_path;
|
// (e.g. /private/var/folders/.../pytest-of-...) fit without MSB truncation.
|
||||||
reg [1023:0] csv_cic_path;
|
reg [4095:0] hex_file_path;
|
||||||
|
reg [4095:0] csv_out_path;
|
||||||
|
reg [4095:0] csv_cic_path;
|
||||||
|
|
||||||
// ── Clock generation ──────────────────────────────────────
|
// ── Clock generation ──────────────────────────────────────
|
||||||
// 400 MHz clock
|
// 400 MHz clock
|
||||||
@@ -152,7 +154,16 @@ module tb_ddc_cosim;
|
|||||||
// ── Select scenario ───────────────────────────────────
|
// ── Select scenario ───────────────────────────────────
|
||||||
// Default to DC scenario for fastest validation
|
// Default to DC scenario for fastest validation
|
||||||
// Override with: +define+SCENARIO_SINGLE, +define+SCENARIO_MULTI, etc.
|
// Override with: +define+SCENARIO_SINGLE, +define+SCENARIO_MULTI, etc.
|
||||||
`ifdef SCENARIO_SINGLE
|
`ifdef SCENARIO_FUZZ
|
||||||
|
// Audit F-3.2: fuzz runner provides +hex and +csv paths plus a
|
||||||
|
// scenario tag. Any missing plusarg falls back to the DC vector.
|
||||||
|
if (!$value$plusargs("hex=%s", hex_file_path))
|
||||||
|
hex_file_path = "tb/cosim/adc_dc.hex";
|
||||||
|
if (!$value$plusargs("csv=%s", csv_out_path))
|
||||||
|
csv_out_path = "tb/cosim/rtl_bb_fuzz.csv";
|
||||||
|
if (!$value$plusargs("tag=%s", scenario_name))
|
||||||
|
scenario_name = "fuzz";
|
||||||
|
`elsif SCENARIO_SINGLE
|
||||||
hex_file_path = "tb/cosim/adc_single_target.hex";
|
hex_file_path = "tb/cosim/adc_single_target.hex";
|
||||||
csv_out_path = "tb/cosim/rtl_bb_single_target.csv";
|
csv_out_path = "tb/cosim/rtl_bb_single_target.csv";
|
||||||
scenario_name = "single_target";
|
scenario_name = "single_target";
|
||||||
|
|||||||
@@ -0,0 +1,185 @@
|
|||||||
|
"""
|
||||||
|
DDC Cosim Fuzz Runner (audit F-3.2)
|
||||||
|
===================================
|
||||||
|
Parameterized seed sweep over the existing DDC cosim testbench.
|
||||||
|
|
||||||
|
For each seed the runner:
|
||||||
|
1. Generates a random plausible radar scene (1-4 targets, random range /
|
||||||
|
velocity / RCS, random noise level) via tb/cosim/radar_scene.py, using
|
||||||
|
the seed for full determinism.
|
||||||
|
2. Writes a temporary ADC hex file.
|
||||||
|
3. Compiles tb_ddc_cosim.v with -DSCENARIO_FUZZ (once, cached across seeds)
|
||||||
|
and runs vvp with +hex, +csv, +tag plusargs.
|
||||||
|
4. Parses the RTL output CSV and checks:
|
||||||
|
- non-empty output (the pipeline produced baseband samples)
|
||||||
|
- all I/Q values are within signed-18-bit range
|
||||||
|
- no NaN / parse errors
|
||||||
|
- sample count is within the expected bound from CIC decimation ratio
|
||||||
|
|
||||||
|
The intent is liveness / crash-fuzz, not bit-exact cross-check. Bit-exact
|
||||||
|
validation is covered by the static scenarios (single_target, multi_target,
|
||||||
|
etc) in the existing suite. Fuzz complements that by surfacing edge-case
|
||||||
|
corruption, saturation, or overflow on random-but-valid inputs.
|
||||||
|
|
||||||
|
Marks:
|
||||||
|
- The default fuzz sweep uses 8 seeds for fast CI.
|
||||||
|
- Use `-m slow` to unlock the full 100-seed sweep matched to the audit ask.
|
||||||
|
|
||||||
|
Compile + run times per seed on a laptop with iverilog 13: ~6 s. The default
|
||||||
|
8-seed sweep fits in a ~1 minute pytest run; the 100-seed sweep takes ~10-12
|
||||||
|
minutes.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
THIS_DIR = Path(__file__).resolve().parent
|
||||||
|
REPO_ROOT = THIS_DIR.parent.parent.parent
|
||||||
|
FPGA_DIR = REPO_ROOT / "9_Firmware" / "9_2_FPGA"
|
||||||
|
COSIM_DIR = FPGA_DIR / "tb" / "cosim"
|
||||||
|
|
||||||
|
sys.path.insert(0, str(COSIM_DIR))
|
||||||
|
import radar_scene # noqa: E402
|
||||||
|
|
||||||
|
FAST_SEEDS = list(range(8))
|
||||||
|
SLOW_SEEDS = list(range(100))
|
||||||
|
|
||||||
|
# Pipeline constants
|
||||||
|
N_ADC_SAMPLES = 16384
|
||||||
|
CIC_DECIMATION = 4
|
||||||
|
FIR_DECIMATION = 1
|
||||||
|
EXPECTED_BB_MIN = N_ADC_SAMPLES // (CIC_DECIMATION * 4) # pessimistic lower bound
|
||||||
|
EXPECTED_BB_MAX = N_ADC_SAMPLES // CIC_DECIMATION # upper bound before FIR drain
|
||||||
|
SIGNED_18_MIN = -(1 << 17)
|
||||||
|
SIGNED_18_MAX = (1 << 17) - 1
|
||||||
|
|
||||||
|
SOURCE_FILES = [
|
||||||
|
"tb/tb_ddc_cosim.v",
|
||||||
|
"ddc_400m.v",
|
||||||
|
"nco_400m_enhanced.v",
|
||||||
|
"cic_decimator_4x_enhanced.v",
|
||||||
|
"fir_lowpass.v",
|
||||||
|
"cdc_modules.v",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def compiled_fuzz_vvp(tmp_path_factory):
|
||||||
|
"""Compile tb_ddc_cosim.v once per pytest session with SCENARIO_FUZZ."""
|
||||||
|
iverilog = _iverilog_bin()
|
||||||
|
if not iverilog:
|
||||||
|
pytest.skip("iverilog not available on PATH")
|
||||||
|
|
||||||
|
out_dir = tmp_path_factory.mktemp("ddc_fuzz_build")
|
||||||
|
vvp = out_dir / "tb_ddc_cosim_fuzz.vvp"
|
||||||
|
sources = [str(FPGA_DIR / p) for p in SOURCE_FILES]
|
||||||
|
cmd = [
|
||||||
|
iverilog, "-g2001", "-DSIMULATION", "-DSCENARIO_FUZZ",
|
||||||
|
"-o", str(vvp), *sources,
|
||||||
|
]
|
||||||
|
res = subprocess.run(cmd, cwd=FPGA_DIR, capture_output=True, text=True, check=False)
|
||||||
|
if res.returncode != 0:
|
||||||
|
pytest.skip(f"iverilog compile failed:\n{res.stderr}")
|
||||||
|
return vvp
|
||||||
|
|
||||||
|
|
||||||
|
def _iverilog_bin() -> str | None:
|
||||||
|
from shutil import which
|
||||||
|
return which("iverilog")
|
||||||
|
|
||||||
|
|
||||||
|
def _random_scene(seed: int) -> list[radar_scene.Target]:
|
||||||
|
rng = random.Random(seed)
|
||||||
|
n = rng.randint(1, 4)
|
||||||
|
return [
|
||||||
|
radar_scene.Target(
|
||||||
|
range_m=rng.uniform(50, 1500),
|
||||||
|
velocity_mps=rng.uniform(-40, 40),
|
||||||
|
rcs_dbsm=rng.uniform(-10, 20),
|
||||||
|
phase_deg=rng.uniform(0, 360),
|
||||||
|
)
|
||||||
|
for _ in range(n)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _run_seed(seed: int, vvp: Path, work: Path) -> tuple[int, list[tuple[int, int]]]:
|
||||||
|
"""Generate stimulus, run the DUT, return (bb_sample_count, [(i,q)...])."""
|
||||||
|
targets = _random_scene(seed)
|
||||||
|
noise = random.Random(seed ^ 0xA5A5).uniform(0.5, 6.0)
|
||||||
|
adc = radar_scene.generate_adc_samples(
|
||||||
|
targets, N_ADC_SAMPLES, noise_stddev=noise, seed=seed
|
||||||
|
)
|
||||||
|
|
||||||
|
hex_path = work / f"adc_fuzz_{seed:04d}.hex"
|
||||||
|
csv_path = work / f"rtl_bb_fuzz_{seed:04d}.csv"
|
||||||
|
radar_scene.write_hex_file(str(hex_path), adc, bits=8)
|
||||||
|
|
||||||
|
vvp_bin = _vvp_bin()
|
||||||
|
if not vvp_bin:
|
||||||
|
pytest.skip("vvp not available")
|
||||||
|
|
||||||
|
cmd = [
|
||||||
|
vvp_bin, str(vvp),
|
||||||
|
f"+hex={hex_path}",
|
||||||
|
f"+csv={csv_path}",
|
||||||
|
f"+tag=seed{seed:04d}",
|
||||||
|
]
|
||||||
|
res = subprocess.run(cmd, cwd=FPGA_DIR, capture_output=True, text=True, check=False, timeout=120)
|
||||||
|
assert res.returncode == 0, f"vvp exit={res.returncode}\nstdout:\n{res.stdout}\nstderr:\n{res.stderr}"
|
||||||
|
assert csv_path.exists(), (
|
||||||
|
f"vvp completed rc=0 but CSV was not produced at {csv_path}\n"
|
||||||
|
f"cmd: {cmd}\nstdout:\n{res.stdout[-2000:]}\nstderr:\n{res.stderr[-500:]}"
|
||||||
|
)
|
||||||
|
|
||||||
|
rows = []
|
||||||
|
with csv_path.open() as fh:
|
||||||
|
header = fh.readline()
|
||||||
|
assert "baseband_i" in header and "baseband_q" in header, f"unexpected CSV header: {header!r}"
|
||||||
|
for line in fh:
|
||||||
|
parts = line.strip().split(",")
|
||||||
|
if len(parts) != 3:
|
||||||
|
continue
|
||||||
|
_, i_str, q_str = parts
|
||||||
|
rows.append((int(i_str), int(q_str)))
|
||||||
|
return len(rows), rows
|
||||||
|
|
||||||
|
|
||||||
|
def _vvp_bin() -> str | None:
|
||||||
|
from shutil import which
|
||||||
|
return which("vvp")
|
||||||
|
|
||||||
|
|
||||||
|
def _fuzz_assertions(seed: int, rows: list[tuple[int, int]]) -> None:
|
||||||
|
n = len(rows)
|
||||||
|
assert EXPECTED_BB_MIN <= n <= EXPECTED_BB_MAX, (
|
||||||
|
f"seed {seed}: bb sample count {n} outside [{EXPECTED_BB_MIN},{EXPECTED_BB_MAX}]"
|
||||||
|
)
|
||||||
|
for idx, (i, q) in enumerate(rows):
|
||||||
|
assert SIGNED_18_MIN <= i <= SIGNED_18_MAX, (
|
||||||
|
f"seed {seed} row {idx}: baseband_i={i} out of signed-18 range"
|
||||||
|
)
|
||||||
|
assert SIGNED_18_MIN <= q <= SIGNED_18_MAX, (
|
||||||
|
f"seed {seed} row {idx}: baseband_q={q} out of signed-18 range"
|
||||||
|
)
|
||||||
|
all_zero = all(i == 0 and q == 0 for i, q in rows)
|
||||||
|
assert not all_zero, f"seed {seed}: all-zero baseband output — pipeline likely stalled"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("seed", FAST_SEEDS)
|
||||||
|
def test_ddc_fuzz_fast(seed: int, compiled_fuzz_vvp: Path, tmp_path: Path) -> None:
|
||||||
|
_, rows = _run_seed(seed, compiled_fuzz_vvp, tmp_path)
|
||||||
|
_fuzz_assertions(seed, rows)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.slow
|
||||||
|
@pytest.mark.parametrize("seed", SLOW_SEEDS)
|
||||||
|
def test_ddc_fuzz_full(seed: int, compiled_fuzz_vvp: Path, tmp_path: Path) -> None:
|
||||||
|
_, rows = _run_seed(seed, compiled_fuzz_vvp, tmp_path)
|
||||||
|
_fuzz_assertions(seed, rows)
|
||||||
@@ -19,6 +19,11 @@ dev = [
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Ruff configuration
|
# Ruff configuration
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
markers = [
|
||||||
|
"slow: full-sweep tests (opt-in via -m slow); audit F-3.2 100-seed fuzz",
|
||||||
|
]
|
||||||
|
|
||||||
[tool.ruff]
|
[tool.ruff]
|
||||||
target-version = "py312"
|
target-version = "py312"
|
||||||
line-length = 100
|
line-length = 100
|
||||||
|
|||||||
Reference in New Issue
Block a user