{
  "study": {
    "date": "2026-11-20",
    "source_pages": 398,
    "questions": 67,
    "answerable_questions": 59,
    "missing_information_questions": 8,
    "test_answerable": 44,
    "test_missing": 6,
    "development_questions": 17,
    "answer_evaluations": 615,
    "retrieval_variants": 8,
    "expert_reviewed": false,
    "scope": "One real PDF. Synthetic administrative records for scale and linked-fact tests."
  },
  "quality": [
    {
      "id": "fixed_dense",
      "label": "Fixed chunks + dense",
      "n": 44,
      "evidence_at_5": 81.81818181818183,
      "evidence_in_context": 90.9090909090909,
      "supported_answers": 97.72727272727273,
      "answer_successes": 43,
      "correct_abstentions": 6,
      "missing_information_cases": 6
    },
    {
      "id": "fixed_hybrid",
      "label": "Fixed chunks + hybrid",
      "n": 44,
      "evidence_at_5": 81.81818181818183,
      "evidence_in_context": 88.63636363636364,
      "supported_answers": 95.45454545454545,
      "answer_successes": 42,
      "correct_abstentions": 6,
      "missing_information_cases": 6
    },
    {
      "id": "fixed_rerank",
      "label": "Fixed chunks + hybrid + rerank",
      "n": 44,
      "evidence_at_5": 97.72727272727273,
      "evidence_in_context": 100.0,
      "supported_answers": 100.0,
      "answer_successes": 44,
      "correct_abstentions": 6,
      "missing_information_cases": 6
    },
    {
      "id": "structure_hybrid",
      "label": "Structure + hybrid",
      "n": 44,
      "evidence_at_5": 86.36363636363636,
      "evidence_in_context": 93.18181818181817,
      "supported_answers": 95.45454545454545,
      "answer_successes": 42,
      "correct_abstentions": 6,
      "missing_information_cases": 6
    },
    {
      "id": "structure_rerank",
      "label": "Structure + hybrid + rerank",
      "n": 44,
      "evidence_at_5": 100.0,
      "evidence_in_context": 100.0,
      "supported_answers": 100.0,
      "answer_successes": 44,
      "correct_abstentions": 6,
      "missing_information_cases": 6
    },
    {
      "id": "enriched_hybrid",
      "label": "Enrichment + hybrid",
      "n": 44,
      "evidence_at_5": 86.36363636363636,
      "evidence_in_context": 88.63636363636364,
      "supported_answers": 95.45454545454545,
      "answer_successes": 42,
      "correct_abstentions": 6,
      "missing_information_cases": 6
    },
    {
      "id": "enriched_rerank",
      "label": "Enrichment + hybrid + rerank",
      "n": 44,
      "evidence_at_5": 97.72727272727273,
      "evidence_in_context": 97.72727272727273,
      "supported_answers": 97.72727272727273,
      "answer_successes": 43,
      "correct_abstentions": 6,
      "missing_information_cases": 6
    },
    {
      "id": "enriched_parent",
      "label": "Enrichment + rerank + expansion",
      "n": 44,
      "evidence_at_5": 97.72727272727273,
      "evidence_in_context": 97.72727272727273,
      "supported_answers": 95.45454545454545,
      "answer_successes": 42,
      "correct_abstentions": 6,
      "missing_information_cases": 6
    }
  ],
  "needle_scale": [
    {
      "distractors": 1000,
      "n": 18,
      "dense_at_5": 55.55555555555556,
      "hybrid_at_5": 100.0
    },
    {
      "distractors": 10000,
      "n": 18,
      "dense_at_5": 50.0,
      "hybrid_at_5": 100.0
    },
    {
      "distractors": 50000,
      "n": 18,
      "dense_at_5": 44.44444444444444,
      "hybrid_at_5": 100.0
    }
  ],
  "needle_answers": [
    {
      "variant": "dense",
      "kind": "direct",
      "n": 18,
      "exact_match_rate": 0.4444444444444444
    },
    {
      "variant": "dense",
      "kind": "two_hop",
      "n": 6,
      "exact_match_rate": 0.0
    },
    {
      "variant": "dense",
      "kind": "absent",
      "n": 6,
      "exact_match_rate": 1.0
    },
    {
      "variant": "hybrid_rerank",
      "kind": "direct",
      "n": 18,
      "exact_match_rate": 1.0
    },
    {
      "variant": "hybrid_rerank",
      "kind": "two_hop",
      "n": 6,
      "exact_match_rate": 0.0
    },
    {
      "variant": "hybrid_rerank",
      "kind": "absent",
      "n": 6,
      "exact_match_rate": 1.0
    },
    {
      "variant": "bounded_two_hop",
      "kind": "two_hop",
      "n": 6,
      "exact_match_rate": 0.0
    }
  ],
  "two_hop": {
    "seed": 20261121,
    "total_passages": 50631,
    "n": 18,
    "design": "New nonce values and new question/record phrasings after diagnosing the initial six two-hop development fixtures.",
    "variants": [
      {
        "variant": "one_hop",
        "n": 18,
        "successes": 0,
        "exact_match_rate": 0.0,
        "context_sufficient": 0.0
      },
      {
        "variant": "two_hop_original_query_rerank",
        "n": 18,
        "successes": 4,
        "exact_match_rate": 0.2222222222222222,
        "context_sufficient": 0.2222222222222222
      },
      {
        "variant": "two_hop_reserved_context",
        "n": 18,
        "successes": 18,
        "exact_match_rate": 1.0,
        "context_sufficient": 1.0
      }
    ]
  },
  "long_context": [
    {
      "tokens": 8000,
      "position": 0.05,
      "n": 2,
      "exact_match_rate": 1.0,
      "p50_api_ms": 2692.461575003108
    },
    {
      "tokens": 8000,
      "position": 0.5,
      "n": 2,
      "exact_match_rate": 1.0,
      "p50_api_ms": 2482.718361512525
    },
    {
      "tokens": 8000,
      "position": 0.95,
      "n": 2,
      "exact_match_rate": 1.0,
      "p50_api_ms": 2360.2318725315854
    },
    {
      "tokens": 32000,
      "position": 0.05,
      "n": 2,
      "exact_match_rate": 1.0,
      "p50_api_ms": 3304.0757839626167
    },
    {
      "tokens": 32000,
      "position": 0.5,
      "n": 2,
      "exact_match_rate": 1.0,
      "p50_api_ms": 3140.041275997646
    },
    {
      "tokens": 32000,
      "position": 0.95,
      "n": 2,
      "exact_match_rate": 1.0,
      "p50_api_ms": 3199.4055375107564
    },
    {
      "tokens": 128000,
      "position": 0.05,
      "n": 2,
      "exact_match_rate": 1.0,
      "p50_api_ms": 3777.4209500057623
    },
    {
      "tokens": 128000,
      "position": 0.5,
      "n": 2,
      "exact_match_rate": 1.0,
      "p50_api_ms": 4432.523898984073
    },
    {
      "tokens": 128000,
      "position": 0.95,
      "n": 2,
      "exact_match_rate": 1.0,
      "p50_api_ms": 4279.813791974448
    }
  ],
  "latency": [
    {
      "id": "fixed_dense",
      "label": "Fixed chunks + dense",
      "n": 8,
      "p50_seconds": 4.158710218995111,
      "p95_seconds": 8.869657125679076
    },
    {
      "id": "structure_rerank",
      "label": "Structure + hybrid + rerank",
      "n": 8,
      "p50_seconds": 5.28273900700151,
      "p95_seconds": 7.41913425962557
    },
    {
      "id": "enriched_rerank",
      "label": "Enrichment + hybrid + rerank",
      "n": 8,
      "p50_seconds": 6.962614810996456,
      "p95_seconds": 13.43851209346321
    },
    {
      "id": "full_document",
      "label": "Full document",
      "n": 8,
      "p50_seconds": 5.109029250481399,
      "p95_seconds": 7.291650728194508
    }
  ],
  "stages": [
    {
      "label": "Query embedding",
      "seconds": 0.6296032209938858
    },
    {
      "label": "Local search",
      "seconds": 0.0011198645224794745
    },
    {
      "label": "Reranking",
      "seconds": 0.7903033424809109
    },
    {
      "label": "Answer generation",
      "seconds": 3.7856638569792267
    }
  ],
  "statistics": {
    "answer_confidence": [
      {
        "variant": "enriched_hybrid",
        "n": 44,
        "successes": 42,
        "wilson_95": [
          0.8486500156237035,
          0.987444890050405
        ]
      },
      {
        "variant": "enriched_parent",
        "n": 44,
        "successes": 42,
        "wilson_95": [
          0.8486500156237035,
          0.987444890050405
        ]
      },
      {
        "variant": "enriched_rerank",
        "n": 44,
        "successes": 43,
        "wilson_95": [
          0.8819229030178671,
          0.9959767479399467
        ]
      },
      {
        "variant": "fixed_dense",
        "n": 44,
        "successes": 43,
        "wilson_95": [
          0.8819229030178671,
          0.9959767479399467
        ]
      },
      {
        "variant": "fixed_hybrid",
        "n": 44,
        "successes": 42,
        "wilson_95": [
          0.8486500156237035,
          0.987444890050405
        ]
      },
      {
        "variant": "fixed_rerank",
        "n": 44,
        "successes": 44,
        "wilson_95": [
          0.9197043962415192,
          1.0
        ]
      },
      {
        "variant": "full_document",
        "n": 8,
        "successes": 8,
        "wilson_95": [
          0.6755924351161198,
          1.0
        ]
      },
      {
        "variant": "oracle",
        "n": 44,
        "successes": 44,
        "wilson_95": [
          0.9197043962415192,
          1.0
        ]
      },
      {
        "variant": "structure_hybrid",
        "n": 44,
        "successes": 42,
        "wilson_95": [
          0.8486500156237035,
          0.987444890050405
        ]
      },
      {
        "variant": "structure_rerank",
        "n": 44,
        "successes": 44,
        "wilson_95": [
          0.9197043962415192,
          1.0
        ]
      }
    ],
    "structure_vs_dense_answer": {
      "n": 44,
      "delta": 0.022727272727272728,
      "paired_bootstrap_95": [
        0.0,
        0.06818181818181818
      ],
      "candidate_only_successes": 1,
      "baseline_only_successes": 0
    },
    "caution": "Exploratory within-document split; cases and LLM judgments are not independent clinical validation."
  },
  "configuration": {
    "embedding": "qwen/qwen3-embedding-8b",
    "embedding_dimensions": 1024,
    "reranker": "qwen/qwen3-reranker-8b",
    "enrichment": "anthropic/claude-haiku-5.5",
    "answer": "openai/gpt-6.1-sol",
    "judge": "anthropic/claude-sonnet-5.5",
    "context_budget_tokens": 4000,
    "max_context_chunks": 10,
    "hybrid": {
      "dense_candidates": 40,
      "bm25_candidates": 40,
      "rrf_k": 60,
      "rerank_candidates": 30
    }
  },
  "limits": [
    "Model-judged answers. Domain-expert review is pending.",
    "The test split was used for diagnosis. It is not a sealed external validation set.",
    "Synthetic records retain exact identifiers. They are not independent real PDFs.",
    "Latency has eight observations per variant. Other benchmark traffic ran concurrently.",
    "No production ANN database, OCR benchmark, or sustained load test was run."
  ]
}
