{
  "answer_path": {
    "answer_rate_answerable": {
      "n": 24,
      "rate": 1.0,
      "successes": 24,
      "wilson_95": [
        0.8620237953250198,
        1.0
      ]
    },
    "citation_precision": {
      "n": 85,
      "rate": 0.17647058823529413,
      "successes": 15,
      "wilson_95": [
        0.10996363561495573,
        0.2709560303823717
      ]
    },
    "configuration": "ablate:vector-only-top3: vector top-3",
    "faithfulness": {
      "n": 85,
      "rate": 0.058823529411764705,
      "successes": 5,
      "wilson_95": [
        0.025385160033412506,
        0.13041438450839762
      ]
    },
    "kind": "rule-based, not a model judge",
    "unanswerable_refusal_rate": {
      "n": 6,
      "rate": 0.16666666666666666,
      "successes": 1,
      "wilson_95": [
        0.030053369748306635,
        0.5635028221864702
      ]
    }
  },
  "gates": [
    {
      "detail": "0.625 < 0.750",
      "n": 24,
      "name": "retrieval_recall_at_3",
      "passed": false,
      "rate": 0.625,
      "successes": 15,
      "threshold": 0.75,
      "wilson_95": [
        0.42709962442674276,
        0.7884063244045122
      ]
    },
    {
      "detail": "1.000 >= 0.500",
      "n": 24,
      "name": "answer_rate_answerable",
      "passed": true,
      "rate": 1.0,
      "successes": 24,
      "threshold": 0.5,
      "wilson_95": [
        0.8620237953250198,
        1.0
      ]
    },
    {
      "detail": "0.167 < 0.800",
      "n": 6,
      "name": "unanswerable_refusal_rate",
      "passed": false,
      "rate": 0.16666666666666666,
      "successes": 1,
      "threshold": 0.8,
      "wilson_95": [
        0.030053369748306635,
        0.5635028221864702
      ]
    },
    {
      "detail": "0.176 < 0.800",
      "n": 85,
      "name": "citation_precision",
      "passed": false,
      "rate": 0.17647058823529413,
      "successes": 15,
      "threshold": 0.8,
      "wilson_95": [
        0.10996363561495573,
        0.2709560303823717
      ]
    },
    {
      "detail": "0.059 < 0.800",
      "n": 85,
      "name": "faithfulness",
      "passed": false,
      "rate": 0.058823529411764705,
      "successes": 5,
      "threshold": 0.8,
      "wilson_95": [
        0.025385160033412506,
        0.13041438450839762
      ]
    }
  ],
  "generated_at": "2026-10-23T02:48:14.843796+00:00",
  "metrics": {
    "answer_rate_answerable": {
      "n": 24,
      "rate": 1.0,
      "successes": 24,
      "wilson_95": [
        0.8620237953250198,
        1.0
      ]
    },
    "citation_precision": {
      "kind": "rule-based, not a model judge",
      "n": 85,
      "rate": 0.17647058823529413,
      "successes": 15,
      "wilson_95": [
        0.10996363561495573,
        0.2709560303823717
      ]
    },
    "faithfulness": {
      "kind": "rule-based, not a model judge",
      "n": 85,
      "rate": 0.058823529411764705,
      "successes": 5,
      "wilson_95": [
        0.025385160033412506,
        0.13041438450839762
      ]
    },
    "retrieval_recall_at_3": {
      "n": 24,
      "rate": 0.625,
      "successes": 15,
      "wilson_95": [
        0.42709962442674276,
        0.7884063244045122
      ]
    },
    "stage_failed_counts": {
      "fusion": 0,
      "generation": 5,
      "rerank": 0,
      "retrieval": 9,
      "schema": 0
    },
    "unanswerable_refusal_rate": {
      "n": 6,
      "rate": 0.16666666666666666,
      "successes": 1,
      "wilson_95": [
        0.030053369748306635,
        0.5635028221864702
      ]
    }
  },
  "passed": false,
  "provenance": {
    "answerable": 24,
    "calibration": {
      "fit_set_sha256": "52278766f27bd7bc78726da5b4a1afb6e2858e8a159abb9f0fca63dfe0275a21",
      "provenance": "provisional-self-retrieval-v1: machine-generated queries from local corpus sentences, hard negatives by reranker score; R5 refits on human gold",
      "threshold": 0.2552458157056597,
      "uncalibrated": false
    },
    "embedder": "fastembed:BAAI/bge-small-en-v1.5",
    "generator": {
      "name": "extractive fallback (no model key)",
      "structured_output": "extractive"
    },
    "gold_path": "eval\\gold.jsonl",
    "gold_provenance": "hand-authored from the 260 ingested chunks; every quote is checked against the live chunk text before measurement",
    "gold_sha256": "3df448b7f080853395d6c9a546c184e8381247998db6b39c8450198a7e911516",
    "judge": {
      "kind": "rule-based, not a model judge",
      "metrics": [
        "citation_precision",
        "faithfulness"
      ],
      "model_judge_used": false,
      "reason": "no judge model is reachable: DOCQA_JUDGE_BASE_URL is unset",
      "rule": "citation precision: a citation is correct when it names the gold chunk; faithfulness: every citation on a claim names the gold chunk and its verbatim quote contains, or is contained in, the gold quote"
    },
    "questions": 30,
    "reranker": "fastembed:Xenova/ms-marco-MiniLM-L-6-v2",
    "reranker_fallback_reason": null,
    "run": {
      "ablated": true,
      "k": 3,
      "label": "ablate:vector-only-top3",
      "mode": "vector"
    },
    "unanswerable": 6
  },
  "questions": [
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 1,
      "error": null,
      "gold": {
        "chunk_id": 586,
        "chunk_index": 5,
        "doc_id": "attention-is-all-you-need",
        "page_no": 3,
        "quote": "The encoder is composed of a stack of N = 6 identical layers."
      },
      "gold_rank": 1,
      "id": "T1",
      "kind": "answerable",
      "latency_ms": 41.8,
      "question": "How many identical layers are in the Transformer encoder stack?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 1,
      "error": null,
      "gold": {
        "chunk_id": 589,
        "chunk_index": 8,
        "doc_id": "attention-is-all-you-need",
        "page_no": 5,
        "quote": "In this work we employ h = 8 parallel attention layers, or heads."
      },
      "gold_rank": 1,
      "id": "T2",
      "kind": "answerable",
      "latency_ms": 177.6,
      "question": "How many parallel attention heads does the Transformer use?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 1,
      "error": null,
      "gold": {
        "chunk_id": 591,
        "chunk_index": 10,
        "doc_id": "attention-is-all-you-need",
        "page_no": 5,
        "quote": "The dimensionality of input and output is dmodel = 512, and the inner-layer has dimensionality dff = 2048."
      },
      "gold_rank": 2,
      "id": "T3",
      "kind": "answerable",
      "latency_ms": 210.7,
      "question": "What is the dimensionality of the Transformer's inner feed-forward layer?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 0,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 596,
        "chunk_index": 15,
        "doc_id": "attention-is-all-you-need",
        "page_no": 7,
        "quote": "We trained the base models for a total of 100,000 steps or 12 hours."
      },
      "gold_rank": null,
      "id": "T4",
      "kind": "answerable",
      "latency_ms": 204.3,
      "question": "How long did training the Transformer base models take?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "retrieval",
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 597,
        "chunk_index": 16,
        "doc_id": "attention-is-all-you-need",
        "page_no": 7,
        "quote": "We used warmup_steps = 4000."
      },
      "gold_rank": 1,
      "id": "T5",
      "kind": "answerable",
      "latency_ms": 185.9,
      "question": "How many warmup steps did the Transformer learning-rate schedule use?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 0,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 309,
        "chunk_index": 13,
        "doc_id": "bert",
        "page_no": 4,
        "quote": "chooses 15% of the token positions at random for"
      },
      "gold_rank": null,
      "id": "B1",
      "kind": "answerable",
      "latency_ms": 189.0,
      "question": "What percentage of token positions does BERT's training data generator choose for prediction?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "retrieval",
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 1,
      "error": null,
      "gold": {
        "chunk_id": 309,
        "chunk_index": 13,
        "doc_id": "bert",
        "page_no": 4,
        "quote": "a random token 10% of the time (3)"
      },
      "gold_rank": 2,
      "id": "B2",
      "kind": "answerable",
      "latency_ms": 176.7,
      "question": "What proportion of chosen BERT tokens is replaced with a random token?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 306,
        "chunk_index": 10,
        "doc_id": "bert",
        "page_no": 3,
        "quote": "BERTLARGE (L=24, H=1024,"
      },
      "gold_rank": 1,
      "id": "B3",
      "kind": "answerable",
      "latency_ms": 193.5,
      "question": "How many layers and hidden units does BERT-Large have?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 313,
        "chunk_index": 17,
        "doc_id": "bert",
        "page_no": 5,
        "quote": "English Wikipedia (2,500M words)."
      },
      "gold_rank": 1,
      "id": "B4",
      "kind": "answerable",
      "latency_ms": 193.9,
      "question": "How many words from English Wikipedia were used in BERT's pre-training corpus?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 0,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 347,
        "chunk_index": 51,
        "doc_id": "bert",
        "page_no": 13,
        "quote": "Cloud TPUs in Pod con\ufb01guration (16 TPU chips total)."
      },
      "gold_rank": null,
      "id": "B5",
      "kind": "answerable",
      "latency_ms": 197.2,
      "question": "How many TPU chips in total did BERT-Base pre-training use?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "retrieval",
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 0,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 371,
        "chunk_index": 11,
        "doc_id": "sentence-bert",
        "page_no": 4,
        "quote": "we fine-tune SBERT with a 3-way softmax-"
      },
      "gold_rank": null,
      "id": "S1",
      "kind": "answerable",
      "latency_ms": 175.7,
      "question": "What objective function does Sentence-BERT use when fine-tuning on NLI data?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "retrieval",
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 0,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 371,
        "chunk_index": 11,
        "doc_id": "sentence-bert",
        "page_no": 4,
        "quote": "used a batch-size of 16, Adam optimizer with"
      },
      "gold_rank": null,
      "id": "S2",
      "kind": "answerable",
      "latency_ms": 170.4,
      "question": "What batch size did Sentence-BERT use for fine-tuning?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "retrieval",
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 363,
        "chunk_index": 3,
        "doc_id": "sentence-bert",
        "page_no": 2,
        "quote": "(~5 seconds with SBERT)"
      },
      "gold_rank": 1,
      "id": "S3",
      "kind": "answerable",
      "latency_ms": 173.5,
      "question": "How long does Sentence-BERT take to compute 10,000 sentence embeddings?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 0,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 371,
        "chunk_index": 11,
        "doc_id": "sentence-bert",
        "page_no": 4,
        "quote": "pooling strategy is MEAN."
      },
      "gold_rank": null,
      "id": "S4",
      "kind": "answerable",
      "latency_ms": 227.3,
      "question": "What is Sentence-BERT's default pooling strategy?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "retrieval",
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 409,
        "chunk_index": 8,
        "doc_id": "dense-passage-retrieval",
        "page_no": 3,
        "quote": "using the dot product of their vectors:"
      },
      "gold_rank": 1,
      "id": "D1",
      "kind": "answerable",
      "latency_ms": 182.6,
      "question": "How does DPR compute the similarity between a question and a passage?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 410,
        "chunk_index": 9,
        "doc_id": "dense-passage-retrieval",
        "page_no": 3,
        "quote": "the simpler inner product function and improve the"
      },
      "gold_rank": 2,
      "id": "D2",
      "kind": "answerable",
      "latency_ms": 176.0,
      "question": "Which similarity function does DPR choose for its retriever?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 418,
        "chunk_index": 17,
        "doc_id": "dense-passage-retrieval",
        "page_no": 5,
        "quote": "with a batch size of 128 and one additional BM25 negative passage per question."
      },
      "gold_rank": 1,
      "id": "D3",
      "kind": "answerable",
      "latency_ms": 180.7,
      "question": "What batch size and how many BM25 negative passages per question did DPR use in training?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 0,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 411,
        "chunk_index": 10,
        "doc_id": "dense-passage-retrieval",
        "page_no": 3,
        "quote": "token as the output, so d = 768."
      },
      "gold_rank": null,
      "id": "D4",
      "kind": "answerable",
      "latency_ms": 172.7,
      "question": "What is the output dimension of DPR's encoders?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "retrieval",
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 0,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 412,
        "chunk_index": 11,
        "doc_id": "dense-passage-retrieval",
        "page_no": 3,
        "quote": "train on B2 (qi, pj) question/passage pairs in each"
      },
      "gold_rank": null,
      "id": "D5",
      "kind": "answerable",
      "latency_ms": 179.1,
      "question": "What does DPR's in-batch negative scheme train on in each batch?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "retrieval",
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 455,
        "chunk_index": 1,
        "doc_id": "retrieval-augmented-generation",
        "page_no": 1,
        "quote": "the parametric memory is a pre-trained seq2seq model and the non-parametric memory is a dense vector index of Wikipedia, accessed with a pre-trained neural retriever."
      },
      "gold_rank": 1,
      "id": "R1",
      "kind": "answerable",
      "latency_ms": 202.4,
      "question": "What form do RAG's parametric and non-parametric memories take?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 1,
      "error": null,
      "gold": {
        "chunk_id": 461,
        "chunk_index": 7,
        "doc_id": "retrieval-augmented-generation",
        "page_no": 3,
        "quote": "The RAG-Sequence model uses the same retrieved document to generate the complete sequence."
      },
      "gold_rank": 3,
      "id": "R2",
      "kind": "answerable",
      "latency_ms": 175.2,
      "question": "How does the RAG-Sequence model use retrieved documents?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 462,
        "chunk_index": 8,
        "doc_id": "retrieval-augmented-generation",
        "page_no": 3,
        "quote": "In the RAG-Token model we can draw a different latent document for each target token and marginalize accordingly."
      },
      "gold_rank": 2,
      "id": "R3",
      "kind": "answerable",
      "latency_ms": 196.2,
      "question": "How does the RAG-Token model treat the retrieved documents?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 0,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 466,
        "chunk_index": 12,
        "doc_id": "retrieval-augmented-generation",
        "page_no": 4,
        "quote": "Each Wikipedia article is split into disjoint 100-word chunks, to make a total of 21M documents."
      },
      "gold_rank": null,
      "id": "R4",
      "kind": "answerable",
      "latency_ms": 177.7,
      "question": "How large is RAG's Wikipedia document index?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "retrieval",
      "top_rerank_score": null
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 3,
      "citations_correct": 1,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 465,
        "chunk_index": 11,
        "doc_id": "retrieval-augmented-generation",
        "page_no": 4,
        "quote": "only fine-tuning the query encoder BERTq and the BART generator."
      },
      "gold_rank": 2,
      "id": "R5",
      "kind": "answerable",
      "latency_ms": 176.4,
      "question": "Which RAG components are fine-tuned while the document encoder and index stay fixed?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": false,
      "answered": true,
      "citations": 3,
      "citations_correct": 0,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": null,
      "gold_rank": null,
      "id": "U1",
      "kind": "near_miss",
      "latency_ms": 197.0,
      "question": "What inference latency per sentence did the Transformer achieve on WMT 2014 English-to-German?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": null,
      "schema_failures": 0,
      "stage_failed": "generation",
      "top_rerank_score": null
    },
    {
      "answerable": false,
      "answered": true,
      "citations": 3,
      "citations_correct": 0,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": null,
      "gold_rank": null,
      "id": "U2",
      "kind": "near_miss",
      "latency_ms": 196.9,
      "question": "What learning rate did RAG use when fine-tuning its generator?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": null,
      "schema_failures": 0,
      "stage_failed": "generation",
      "top_rerank_score": null
    },
    {
      "answerable": false,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": null,
      "gold_rank": null,
      "id": "U3",
      "kind": "out_of_corpus",
      "latency_ms": 206.7,
      "question": "What is the capital city of Australia?",
      "refusal_probability": null,
      "refusal_reason": "insufficient_context",
      "refusal_threshold": 0.2552458157056597,
      "refused": true,
      "retrieved": null,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": null
    },
    {
      "answerable": false,
      "answered": true,
      "citations": 1,
      "citations_correct": 0,
      "claims": 1,
      "claims_faithful": 0,
      "error": null,
      "gold": null,
      "gold_rank": null,
      "id": "U4",
      "kind": "out_of_corpus",
      "latency_ms": 179.3,
      "question": "Who won the 2018 FIFA World Cup?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": null,
      "schema_failures": 0,
      "stage_failed": "generation",
      "top_rerank_score": null
    },
    {
      "answerable": false,
      "answered": true,
      "citations": 3,
      "citations_correct": 0,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": null,
      "gold_rank": null,
      "id": "U5",
      "kind": "under_specified",
      "latency_ms": 179.3,
      "question": "How many layers does it use?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": null,
      "schema_failures": 0,
      "stage_failed": "generation",
      "top_rerank_score": null
    },
    {
      "answerable": false,
      "answered": true,
      "citations": 3,
      "citations_correct": 0,
      "claims": 3,
      "claims_faithful": 0,
      "error": null,
      "gold": null,
      "gold_rank": null,
      "id": "U6",
      "kind": "ambiguous",
      "latency_ms": 192.8,
      "question": "Which model is best for question answering?",
      "refusal_probability": null,
      "refusal_reason": null,
      "refusal_threshold": 0.2552458157056597,
      "refused": false,
      "retrieved": null,
      "schema_failures": 0,
      "stage_failed": "generation",
      "top_rerank_score": null
    }
  ],
  "retrieval_ablation": [
    {
      "configuration": "vector-only top-8",
      "k": 8,
      "mode": "vector",
      "n": 24,
      "recall": {
        "n": 24,
        "rate": 0.7083333333333334,
        "successes": 17,
        "wilson_95": [
          0.5083230633550337,
          0.8508535180303913
        ]
      }
    },
    {
      "configuration": "+lexical/RRF top-8",
      "k": 8,
      "mode": "hybrid",
      "n": 24,
      "recall": {
        "n": 24,
        "rate": 0.7083333333333334,
        "successes": 17,
        "wilson_95": [
          0.5083230633550337,
          0.8508535180303913
        ]
      }
    },
    {
      "configuration": "+rerank top-8",
      "k": 8,
      "mode": "hybrid_rerank",
      "n": 24,
      "recall": {
        "n": 24,
        "rate": 0.75,
        "successes": 18,
        "wilson_95": [
          0.55100555994846,
          0.8800063377140499
        ]
      }
    }
  ]
}