ATOCore/scripts/retrieval_eval_fixtures.json

[
  {
    "name": "p04-architecture-decision",
    "project": "p04-gigabit",
    "prompt": "what mirror architecture was selected for GigaBIT M1 and why",
    "expect_present": [
      "--- Trusted Project State ---",
      "Option B",
      "conical",
      "--- Project Memories ---"
    ],
    "expect_absent": [
      "p06-polisher",
      "folded-beam"
    ],
    "notes": "Canonical p04 decision — should surface both Trusted Project State and the project-memory band"
  },
  {
    "name": "p04-constraints",
    "project": "p04-gigabit",
    "prompt": "what are the key GigaBIT M1 program constraints",
    "expect_present": [
      "--- Trusted Project State ---",
      "Zerodur",
      "1.2"
    ],
    "expect_absent": [
      "polisher suite"
    ],
    "known_issue": true,
    "notes": "Known content gap as of 2026-04-24: live retrieval surfaces related constraints but not the exact Zerodur / 1.2 strings. Keep visible, but do not make nightly harness red until the source/state gap is fixed."
  },
  {
    "name": "p04-short-ambiguous",
    "project": "p04-gigabit",
    "prompt": "current status",
    "expect_present": [
      "--- Trusted Project State ---"
    ],
    "expect_absent": [],
    "notes": "Short ambiguous prompt — at minimum project state should surface. Hard case: the prompt is generic enough that chunks may not rank well."
  },
  {
    "name": "p05-configuration",
    "project": "p05-interferometer",
    "prompt": "what is the selected interferometer configuration",
    "expect_present": [
      "folded-beam",
      "CGH"
    ],
    "expect_absent": [
      "Option B",
      "conical back",
      "polisher suite"
    ],
    "notes": "P05 architecture memory covers folded-beam + CGH. GigaBIT M1 legitimately appears in p05 source docs."
  },
  {
    "name": "p05-vendor-signal",
    "project": "p05-interferometer",
    "prompt": "what is the current vendor signal for the interferometer procurement",
    "expect_present": [
      "4D",
      "Zygo"
    ],
    "expect_absent": [
      "polisher"
    ],
    "notes": "Vendor memory mentions 4D as strongest technical candidate and Zygo Verifire SV as value path"
  },
  {
    "name": "p05-cgh-calibration",
    "project": "p05-interferometer",
    "prompt": "how does CGH calibration work for the interferometer",
    "expect_present": [
      "CGH"
    ],
    "expect_absent": [
      "polisher-sim",
      "polisher-post"
    ],
    "notes": "CGH is a core p05 concept. Should surface via chunks and possibly the architecture memory. Must not bleed p06 polisher-suite terms."
  },
  {
    "name": "p05-broad-status-no-atomizer",
    "project": "p05-interferometer",
    "prompt": "current status",
    "expect_present": [
      "--- Trusted Project State ---",
      "--- Project Memories ---",
      "Zygo"
    ],
    "expect_absent": [
      "atomizer-v2",
      "ATOMIZER_PODCAST_BRIEFING",
      "[Source: atomizer-v2/",
      "P04-GigaBIT-M1-KB-design"
    ],
    "notes": "Regression guard for the April 24 audit finding: broad p05 status queries must not pull Atomizer/archive context into project-scoped packs."
  },
  {
    "name": "p05-vendor-decision-no-archive-first",
    "project": "p05-interferometer",
    "prompt": "vendor selection decision",
    "expect_present": [
      "Selection-Decision"
    ],
    "expect_absent": [
      "[Source: atomizer-v2/",
      "ATOMIZER_PODCAST_BRIEFING"
    ],
    "notes": "Project-scoped decision query should stay inside p05 and prefer current decision/vendor material over unrelated project archives."
  },
  {
    "name": "p06-suite-split",
    "project": "p06-polisher",
    "prompt": "how is the polisher software suite split across layers",
    "expect_present": [
      "polisher-sim",
      "polisher-post",
      "polisher-control"
    ],
    "expect_absent": [
      "GigaBIT"
    ],
    "notes": "The three-layer split is in multiple p06 memories"
  },
  {
    "name": "p06-control-rule",
    "project": "p06-polisher",
    "prompt": "what is the polisher control design rule",
    "expect_present": [
      "interlocks"
    ],
    "expect_absent": [
      "interferometer"
    ],
    "notes": "Control design rule memory mentions interlocks and state transitions"
  },
  {
    "name": "p06-firmware-interface",
    "project": "p06-polisher",
    "prompt": "what is the firmware interface contract for the polisher machine",
    "expect_present": [
      "controller-job"
    ],
    "expect_absent": [
      "interferometer",
      "GigaBIT"
    ],
    "notes": "New p06 memory from the first triage: firmware interface contract is invariant controller-job.v1 in, run-log.v1 out"
  },
  {
    "name": "p06-z-axis",
    "project": "p06-polisher",
    "prompt": "how does the polisher Z-axis work",
    "expect_present": [
      "engage"
    ],
    "expect_absent": [
      "interferometer"
    ],
    "notes": "New p06 memory: Z-axis is binary engage/retract, not continuous position. The word 'engage' should appear."
  },
  {
    "name": "p06-cam-mechanism",
    "project": "p06-polisher",
    "prompt": "how is cam amplitude controlled on the polisher",
    "expect_present": [
      "encoder"
    ],
    "expect_absent": [
      "GigaBIT"
    ],
    "notes": "New p06 memory: cam set mechanically by operator, read by encoders. The word 'encoder' should appear."
  },
  {
    "name": "p06-telemetry-rate",
    "project": "p06-polisher",
    "prompt": "what is the expected polishing telemetry data rate",
    "expect_present": [
      "29 MB"
    ],
    "expect_absent": [
      "interferometer"
    ],
    "notes": "New p06 knowledge memory: approximately 29 MB per hour at 100 Hz"
  },
  {
    "name": "p06-offline-design",
    "project": "p06-polisher",
    "prompt": "does the polisher machine need network to operate",
    "expect_present": [
      "offline"
    ],
    "expect_absent": [
      "CGH"
    ],
    "notes": "New p06 memory: machine works fully offline and independently; network is for remote access only"
  },
  {
    "name": "p06-short-ambiguous",
    "project": "p06-polisher",
    "prompt": "current status",
    "expect_present": [
      "--- Trusted Project State ---"
    ],
    "expect_absent": [],
    "notes": "Short ambiguous prompt — project state should surface at minimum"
  },
  {
    "name": "cross-project-no-bleed",
    "project": "p04-gigabit",
    "prompt": "what telemetry rate should we target",
    "expect_present": [],
    "expect_absent": [
      "29 MB",
      "polisher"
    ],
    "notes": "Adversarial: telemetry rate is a p06 fact. A p04 query for 'telemetry rate' must NOT surface p06 memories. Tests cross-project gating."
  },
  {
    "name": "no-project-hint",
    "project": "",
    "prompt": "tell me about the current projects",
    "expect_present": [],
    "expect_absent": [
      "--- Project Memories ---"
    ],
    "notes": "Without a project hint, project memories must not appear (cross-project bleed guard). Chunks may appear if any match."
  },
  {
    "name": "p06-usb-ssd",
    "project": "p06-polisher",
    "prompt": "what storage solution is specified for the polisher RPi",
    "expect_present": [
      "USB SSD"
    ],
    "expect_absent": [
      "interferometer"
    ],
    "notes": "New p06 memory from triage: USB SSD mandatory, not SD card"
  },
  {
    "name": "p06-tailscale",
    "project": "p06-polisher",
    "prompt": "how do we access the polisher machine remotely",
    "expect_present": [
      "Tailscale"
    ],
    "expect_absent": [
      "[Source: p04-gigabit/"
    ],
    "notes": "New p06 memory: Tailscale mesh for RPi remote access. Cross-project guard is a source-path check, not a word blacklist: the polisher ARCHITECTURE.md legitimately mentions the GigaBIT M1 mirror (it is what the polisher is built for), so testing for absence of that word produces false positives. The real invariant is that no p04 source chunks are retrieved into p06 context."
  }
]