{
  "answer_path": {
    "answer_rate_answerable": {
      "n": 24,
      "rate": 0.3333333333333333,
      "successes": 8,
      "wilson_95": [
        0.1797219032729775,
        0.5329368316186824
      ]
    },
    "citation_precision": {
      "n": 9,
      "rate": 0.7777777777777778,
      "successes": 7,
      "wilson_95": [
        0.45258896910698887,
        0.9367748928821533
      ]
    },
    "configuration": "production: hybrid_rerank top-8",
    "faithfulness": {
      "n": 9,
      "rate": 0.7777777777777778,
      "successes": 7,
      "wilson_95": [
        0.45258896910698887,
        0.9367748928821533
      ]
    },
    "kind": "rule-based, not a model judge",
    "unanswerable_refusal_rate": {
      "n": 6,
      "rate": 1.0,
      "successes": 6,
      "wilson_95": [
        0.6096657120978346,
        1.0
      ]
    }
  },
  "gates": [
    {
      "detail": "0.750 >= 0.750",
      "n": 24,
      "name": "retrieval_recall_at_8",
      "passed": true,
      "rate": 0.75,
      "successes": 18,
      "threshold": 0.75,
      "wilson_95": [
        0.55100555994846,
        0.8800063377140499
      ]
    },
    {
      "detail": "0.333 < 0.500",
      "n": 24,
      "name": "answer_rate_answerable",
      "passed": false,
      "rate": 0.3333333333333333,
      "successes": 8,
      "threshold": 0.5,
      "wilson_95": [
        0.1797219032729775,
        0.5329368316186824
      ]
    },
    {
      "detail": "1.000 >= 0.800",
      "n": 6,
      "name": "unanswerable_refusal_rate",
      "passed": true,
      "rate": 1.0,
      "successes": 6,
      "threshold": 0.8,
      "wilson_95": [
        0.6096657120978346,
        1.0
      ]
    },
    {
      "detail": "0.778 < 0.800",
      "n": 9,
      "name": "citation_precision",
      "passed": false,
      "rate": 0.7777777777777778,
      "successes": 7,
      "threshold": 0.8,
      "wilson_95": [
        0.45258896910698887,
        0.9367748928821533
      ]
    },
    {
      "detail": "0.778 < 0.800",
      "n": 9,
      "name": "faithfulness",
      "passed": false,
      "rate": 0.7777777777777778,
      "successes": 7,
      "threshold": 0.8,
      "wilson_95": [
        0.45258896910698887,
        0.9367748928821533
      ]
    }
  ],
  "generated_at": "2026-10-23T18:22:37.202794+00:00",
  "metrics": {
    "answer_rate_answerable": {
      "n": 24,
      "rate": 0.3333333333333333,
      "successes": 8,
      "wilson_95": [
        0.1797219032729775,
        0.5329368316186824
      ]
    },
    "citation_precision": {
      "kind": "rule-based, not a model judge",
      "n": 9,
      "rate": 0.7777777777777778,
      "successes": 7,
      "wilson_95": [
        0.45258896910698887,
        0.9367748928821533
      ]
    },
    "faithfulness": {
      "kind": "rule-based, not a model judge",
      "n": 9,
      "rate": 0.7777777777777778,
      "successes": 7,
      "wilson_95": [
        0.45258896910698887,
        0.9367748928821533
      ]
    },
    "retrieval_recall_at_8": {
      "n": 24,
      "rate": 0.75,
      "successes": 18,
      "wilson_95": [
        0.55100555994846,
        0.8800063377140499
      ]
    },
    "stage_failed_counts": {
      "fusion": 0,
      "generation": 0,
      "rerank": 1,
      "retrieval": 5,
      "schema": 11
    },
    "unanswerable_refusal_rate": {
      "n": 6,
      "rate": 1.0,
      "successes": 6,
      "wilson_95": [
        0.6096657120978346,
        1.0
      ]
    }
  },
  "passed": false,
  "provenance": {
    "answerable": 24,
    "calibration": {
      "fit_set_sha256": "f1a55cddcad2154aa67df9265f412d9e27b44e21fd42aac292158cd1cc580479",
      "provenance": "gold-human-v1: question-level Platt scaling over the top reranked score of the 30 hand-authored gold questions (24 answerable, 6 unanswerable); the threshold maximises balanced accuracy. Coefficients and threshold are refitted on all 30, so the reported in-sample balanced accuracy is optimistic; held_out records the id-parity split measured out of sample.",
      "threshold": 0.6360770752477083,
      "uncalibrated": false
    },
    "embedder": "fastembed:BAAI/bge-small-en-v1.5",
    "generator": {
      "name": "openai-compatible:openai-fast",
      "structured_output": "tool"
    },
    "gold_path": "eval\\gold.jsonl",
    "gold_provenance": "hand-authored from the 260 ingested chunks; every quote is checked against the live chunk text before measurement",
    "gold_sha256": "3df448b7f080853395d6c9a546c184e8381247998db6b39c8450198a7e911516",
    "judge": {
      "kind": "rule-based, not a model judge",
      "metrics": [
        "citation_precision",
        "faithfulness"
      ],
      "model_judge_used": false,
      "reason": "no judge model is reachable: DOCQA_JUDGE_BASE_URL is unset",
      "rule": "citation precision: a citation is correct when it names the gold chunk; faithfulness: every citation on a claim names the gold chunk and its verbatim quote contains, or is contained in, the gold quote"
    },
    "questions": 30,
    "reranker": "fastembed:Xenova/ms-marco-MiniLM-L-6-v2",
    "reranker_fallback_reason": null,
    "run": {
      "ablated": false,
      "blocked": null,
      "finished_at": "2026-10-23T18:22:37.202794+00:00",
      "force_generation": false,
      "gate_run": true,
      "k": 8,
      "label": "production",
      "mode": "hybrid_rerank",
      "model_calls": {
        "rate_limited": 0,
        "requests": 26,
        "retries": 0
      },
      "model_min_interval_s": 31.0,
      "questions_measured": 30,
      "rate_limit_failures": 0,
      "started_at": "2026-10-23T18:09:38.622758+00:00",
      "wall_clock_min": 12.98,
      "wall_clock_s": 778.6
    },
    "unanswerable": 6
  },
  "questions": [
    {
      "answerable": true,
      "answered": true,
      "citations": 1,
      "citations_correct": 1,
      "claims": 1,
      "claims_faithful": 1,
      "error": null,
      "gold": {
        "chunk_id": 586,
        "chunk_index": 5,
        "doc_id": "attention-is-all-you-need",
        "page_no": 3,
        "quote": "The encoder is composed of a stack of N = 6 identical layers."
      },
      "gold_rank": 1,
      "id": "T1",
      "kind": "answerable",
      "latency_ms": 485.6,
      "question": "How many identical layers are in the Transformer encoder stack?",
      "refusal_probability": 0.9538837214564011,
      "refusal_reason": null,
      "refusal_threshold": 0.6360770752477083,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": 6.967045783996582
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 1,
      "citations_correct": 1,
      "claims": 1,
      "claims_faithful": 1,
      "error": null,
      "gold": {
        "chunk_id": 589,
        "chunk_index": 8,
        "doc_id": "attention-is-all-you-need",
        "page_no": 5,
        "quote": "In this work we employ h = 8 parallel attention layers, or heads."
      },
      "gold_rank": 1,
      "id": "T2",
      "kind": "answerable",
      "latency_ms": 28886.4,
      "question": "How many parallel attention heads does the Transformer use?",
      "refusal_probability": 0.8329435212305938,
      "refusal_reason": null,
      "refusal_threshold": 0.6360770752477083,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": 2.752397298812866
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 1,
      "citations_correct": 1,
      "claims": 1,
      "claims_faithful": 1,
      "error": null,
      "gold": {
        "chunk_id": 591,
        "chunk_index": 10,
        "doc_id": "attention-is-all-you-need",
        "page_no": 5,
        "quote": "The dimensionality of input and output is dmodel = 512, and the inner-layer has dimensionality dff = 2048."
      },
      "gold_rank": 1,
      "id": "T3",
      "kind": "answerable",
      "latency_ms": 28627.0,
      "question": "What is the dimensionality of the Transformer's inner feed-forward layer?",
      "refusal_probability": 0.7859231090505028,
      "refusal_reason": null,
      "refusal_threshold": 0.6360770752477083,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": 1.8455941677093506
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 1,
      "citations_correct": 1,
      "claims": 1,
      "claims_faithful": 1,
      "error": null,
      "gold": {
        "chunk_id": 596,
        "chunk_index": 15,
        "doc_id": "attention-is-all-you-need",
        "page_no": 7,
        "quote": "We trained the base models for a total of 100,000 steps or 12 hours."
      },
      "gold_rank": 6,
      "id": "T4",
      "kind": "answerable",
      "latency_ms": 28788.8,
      "question": "How long did training the Transformer base models take?",
      "refusal_probability": 0.9128731254281406,
      "refusal_reason": null,
      "refusal_threshold": 0.6360770752477083,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": 4.952225685119629
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 1,
      "citations_correct": 1,
      "claims": 1,
      "claims_faithful": 1,
      "error": null,
      "gold": {
        "chunk_id": 597,
        "chunk_index": 16,
        "doc_id": "attention-is-all-you-need",
        "page_no": 7,
        "quote": "We used warmup_steps = 4000."
      },
      "gold_rank": 1,
      "id": "T5",
      "kind": "answerable",
      "latency_ms": 29001.6,
      "question": "How many warmup steps did the Transformer learning-rate schedule use?",
      "refusal_probability": 0.8562725161087571,
      "refusal_reason": null,
      "refusal_threshold": 0.6360770752477083,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": 3.2798001766204834
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 309,
        "chunk_index": 13,
        "doc_id": "bert",
        "page_no": 4,
        "quote": "chooses 15% of the token positions at random for"
      },
      "gold_rank": 3,
      "id": "B1",
      "kind": "answerable",
      "latency_ms": 28553.6,
      "question": "What percentage of token positions does BERT's training data generator choose for prediction?",
      "refusal_probability": 0.9042447233626156,
      "refusal_reason": "validation_failed",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": "schema",
      "top_rerank_score": 4.644357681274414
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 309,
        "chunk_index": 13,
        "doc_id": "bert",
        "page_no": 4,
        "quote": "a random token 10% of the time (3)"
      },
      "gold_rank": 3,
      "id": "B2",
      "kind": "answerable",
      "latency_ms": 29104.8,
      "question": "What proportion of chosen BERT tokens is replaced with a random token?",
      "refusal_probability": 0.8306006850794441,
      "refusal_reason": "validation_failed",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": "schema",
      "top_rerank_score": 2.7027974128723145
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 306,
        "chunk_index": 10,
        "doc_id": "bert",
        "page_no": 3,
        "quote": "BERTLARGE (L=24, H=1024,"
      },
      "gold_rank": 2,
      "id": "B3",
      "kind": "answerable",
      "latency_ms": 29135.5,
      "question": "How many layers and hidden units does BERT-Large have?",
      "refusal_probability": 0.8439095261412042,
      "refusal_reason": "validation_failed",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": "schema",
      "top_rerank_score": 2.992274761199951
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 313,
        "chunk_index": 17,
        "doc_id": "bert",
        "page_no": 5,
        "quote": "English Wikipedia (2,500M words)."
      },
      "gold_rank": 1,
      "id": "B4",
      "kind": "answerable",
      "latency_ms": 28658.4,
      "question": "How many words from English Wikipedia were used in BERT's pre-training corpus?",
      "refusal_probability": 0.9155284205343981,
      "refusal_reason": "validation_failed",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": "schema",
      "top_rerank_score": 5.052515029907227
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 347,
        "chunk_index": 51,
        "doc_id": "bert",
        "page_no": 13,
        "quote": "Cloud TPUs in Pod con\ufb01guration (16 TPU chips total)."
      },
      "gold_rank": 2,
      "id": "B5",
      "kind": "answerable",
      "latency_ms": 28841.5,
      "question": "How many TPU chips in total did BERT-Base pre-training use?",
      "refusal_probability": 0.8422175336440862,
      "refusal_reason": "validation_failed",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": "schema",
      "top_rerank_score": 2.9543910026550293
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": "response contained no tool call",
      "gold": {
        "chunk_id": 371,
        "chunk_index": 11,
        "doc_id": "sentence-bert",
        "page_no": 4,
        "quote": "we fine-tune SBERT with a 3-way softmax-"
      },
      "gold_rank": null,
      "id": "S1",
      "kind": "answerable",
      "latency_ms": 29075.4,
      "question": "What objective function does Sentence-BERT use when fine-tuning on NLI data?",
      "refusal_probability": null,
      "refusal_reason": "model_call_error",
      "refusal_threshold": null,
      "refused": false,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "retrieval",
      "top_rerank_score": 6.23803186416626
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 371,
        "chunk_index": 11,
        "doc_id": "sentence-bert",
        "page_no": 4,
        "quote": "used a batch-size of 16, Adam optimizer with"
      },
      "gold_rank": null,
      "id": "S2",
      "kind": "answerable",
      "latency_ms": 28597.7,
      "question": "What batch size did Sentence-BERT use for fine-tuning?",
      "refusal_probability": 0.8470145265505572,
      "refusal_reason": "insufficient_context",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "retrieval",
      "top_rerank_score": 3.062675952911377
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 363,
        "chunk_index": 3,
        "doc_id": "sentence-bert",
        "page_no": 2,
        "quote": "(~5 seconds with SBERT)"
      },
      "gold_rank": 1,
      "id": "S3",
      "kind": "answerable",
      "latency_ms": 28926.6,
      "question": "How long does Sentence-BERT take to compute 10,000 sentence embeddings?",
      "refusal_probability": 0.9636742249455325,
      "refusal_reason": "validation_failed",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": "schema",
      "top_rerank_score": 7.704224586486816
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 371,
        "chunk_index": 11,
        "doc_id": "sentence-bert",
        "page_no": 4,
        "quote": "pooling strategy is MEAN."
      },
      "gold_rank": null,
      "id": "S4",
      "kind": "answerable",
      "latency_ms": 29046.9,
      "question": "What is Sentence-BERT's default pooling strategy?",
      "refusal_probability": 0.8701284478358733,
      "refusal_reason": "validation_failed",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "retrieval",
      "top_rerank_score": 3.6276540756225586
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 409,
        "chunk_index": 8,
        "doc_id": "dense-passage-retrieval",
        "page_no": 3,
        "quote": "using the dot product of their vectors:"
      },
      "gold_rank": 1,
      "id": "D1",
      "kind": "answerable",
      "latency_ms": 29014.6,
      "question": "How does DPR compute the similarity between a question and a passage?",
      "refusal_probability": 0.9432454356681055,
      "refusal_reason": "validation_failed",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": "schema",
      "top_rerank_score": 6.3189287185668945
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 410,
        "chunk_index": 9,
        "doc_id": "dense-passage-retrieval",
        "page_no": 3,
        "quote": "the simpler inner product function and improve the"
      },
      "gold_rank": 4,
      "id": "D2",
      "kind": "answerable",
      "latency_ms": 29088.5,
      "question": "Which similarity function does DPR choose for its retriever?",
      "refusal_probability": 0.8875194561697443,
      "refusal_reason": "validation_failed",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": "schema",
      "top_rerank_score": 4.112160682678223
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 418,
        "chunk_index": 17,
        "doc_id": "dense-passage-retrieval",
        "page_no": 5,
        "quote": "with a batch size of 128 and one additional BM25 negative passage per question."
      },
      "gold_rank": 1,
      "id": "D3",
      "kind": "answerable",
      "latency_ms": 28007.3,
      "question": "What batch size and how many BM25 negative passages per question did DPR use in training?",
      "refusal_probability": 0.9625142347605607,
      "refusal_reason": "validation_failed",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": "schema",
      "top_rerank_score": 7.607539176940918
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 411,
        "chunk_index": 10,
        "doc_id": "dense-passage-retrieval",
        "page_no": 3,
        "quote": "token as the output, so d = 768."
      },
      "gold_rank": null,
      "id": "D4",
      "kind": "answerable",
      "latency_ms": 28793.8,
      "question": "What is the output dimension of DPR's encoders?",
      "refusal_probability": 0.8269057340314497,
      "refusal_reason": "insufficient_context",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "retrieval",
      "top_rerank_score": 2.6256697177886963
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 412,
        "chunk_index": 11,
        "doc_id": "dense-passage-retrieval",
        "page_no": 3,
        "quote": "train on B2 (qi, pj) question/passage pairs in each"
      },
      "gold_rank": null,
      "id": "D5",
      "kind": "answerable",
      "latency_ms": 29090.8,
      "question": "What does DPR's in-batch negative scheme train on in each batch?",
      "refusal_probability": 0.9073814789553193,
      "refusal_reason": "validation_failed",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "retrieval",
      "top_rerank_score": 4.753281593322754
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 1,
      "citations_correct": 1,
      "claims": 1,
      "claims_faithful": 1,
      "error": null,
      "gold": {
        "chunk_id": 455,
        "chunk_index": 1,
        "doc_id": "retrieval-augmented-generation",
        "page_no": 1,
        "quote": "the parametric memory is a pre-trained seq2seq model and the non-parametric memory is a dense vector index of Wikipedia, accessed with a pre-trained neural retriever."
      },
      "gold_rank": 1,
      "id": "R1",
      "kind": "answerable",
      "latency_ms": 29034.1,
      "question": "What form do RAG's parametric and non-parametric memories take?",
      "refusal_probability": 0.9162603481864152,
      "refusal_reason": null,
      "refusal_threshold": 0.6360770752477083,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": 5.080662250518799
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 461,
        "chunk_index": 7,
        "doc_id": "retrieval-augmented-generation",
        "page_no": 3,
        "quote": "The RAG-Sequence model uses the same retrieved document to generate the complete sequence."
      },
      "gold_rank": 1,
      "id": "R2",
      "kind": "answerable",
      "latency_ms": 28988.5,
      "question": "How does the RAG-Sequence model use retrieved documents?",
      "refusal_probability": 0.9579558424586418,
      "refusal_reason": "validation_failed",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": "schema",
      "top_rerank_score": 7.253520488739014
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 1,
      "citations_correct": 1,
      "claims": 1,
      "claims_faithful": 1,
      "error": null,
      "gold": {
        "chunk_id": 462,
        "chunk_index": 8,
        "doc_id": "retrieval-augmented-generation",
        "page_no": 3,
        "quote": "In the RAG-Token model we can draw a different latent document for each target token and marginalize accordingly."
      },
      "gold_rank": 2,
      "id": "R3",
      "kind": "answerable",
      "latency_ms": 28663.4,
      "question": "How does the RAG-Token model treat the retrieved documents?",
      "refusal_probability": 0.9581275625442802,
      "refusal_reason": null,
      "refusal_threshold": 0.6360770752477083,
      "refused": false,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": 7.266175270080566
    },
    {
      "answerable": true,
      "answered": true,
      "citations": 2,
      "citations_correct": 0,
      "claims": 2,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 466,
        "chunk_index": 12,
        "doc_id": "retrieval-augmented-generation",
        "page_no": 4,
        "quote": "Each Wikipedia article is split into disjoint 100-word chunks, to make a total of 21M documents."
      },
      "gold_rank": null,
      "id": "R4",
      "kind": "answerable",
      "latency_ms": 29180.2,
      "question": "How large is RAG's Wikipedia document index?",
      "refusal_probability": 0.7601262463296258,
      "refusal_reason": null,
      "refusal_threshold": 0.6360770752477083,
      "refused": false,
      "retrieved": false,
      "schema_failures": 0,
      "stage_failed": "rerank",
      "top_rerank_score": 1.4096797704696655
    },
    {
      "answerable": true,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": {
        "chunk_id": 465,
        "chunk_index": 11,
        "doc_id": "retrieval-augmented-generation",
        "page_no": 4,
        "quote": "only fine-tuning the query encoder BERTq and the BART generator."
      },
      "gold_rank": 1,
      "id": "R5",
      "kind": "answerable",
      "latency_ms": 29050.6,
      "question": "Which RAG components are fine-tuned while the document encoder and index stay fixed?",
      "refusal_probability": 0.8005271345282986,
      "refusal_reason": "validation_failed",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": true,
      "schema_failures": 0,
      "stage_failed": "schema",
      "top_rerank_score": 2.109445571899414
    },
    {
      "answerable": false,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": null,
      "gold_rank": null,
      "id": "U1",
      "kind": "near_miss",
      "latency_ms": 28784.1,
      "question": "What inference latency per sentence did the Transformer achieve on WMT 2014 English-to-German?",
      "refusal_probability": 0.890858368927551,
      "refusal_reason": "insufficient_context",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": null,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": 4.212551116943359
    },
    {
      "answerable": false,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": null,
      "gold_rank": null,
      "id": "U2",
      "kind": "near_miss",
      "latency_ms": 1.0,
      "question": "What learning rate did RAG use when fine-tuning its generator?",
      "refusal_probability": 0.4415435564827254,
      "refusal_reason": "low_score",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": null,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": -2.7028563022613525
    },
    {
      "answerable": false,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": null,
      "gold_rank": null,
      "id": "U3",
      "kind": "out_of_corpus",
      "latency_ms": 0.0,
      "question": "What is the capital city of Australia?",
      "refusal_probability": 0.04389737969144231,
      "refusal_reason": "low_score",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": null,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": -11.134010314941406
    },
    {
      "answerable": false,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": null,
      "gold_rank": null,
      "id": "U4",
      "kind": "out_of_corpus",
      "latency_ms": 0.0,
      "question": "Who won the 2018 FIFA World Cup?",
      "refusal_probability": 0.04823923742279572,
      "refusal_reason": "low_score",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": null,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": -10.841124534606934
    },
    {
      "answerable": false,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": null,
      "gold_rank": null,
      "id": "U5",
      "kind": "under_specified",
      "latency_ms": 0.0,
      "question": "How many layers does it use?",
      "refusal_probability": 0.5120279041657908,
      "refusal_reason": "low_score",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": null,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": -1.8644516468048096
    },
    {
      "answerable": false,
      "answered": false,
      "citations": 0,
      "citations_correct": 0,
      "claims": 0,
      "claims_faithful": 0,
      "error": null,
      "gold": null,
      "gold_rank": null,
      "id": "U6",
      "kind": "ambiguous",
      "latency_ms": 19385.7,
      "question": "Which model is best for question answering?",
      "refusal_probability": 0.8418902960930116,
      "refusal_reason": "validation_failed",
      "refusal_threshold": 0.6360770752477083,
      "refused": true,
      "retrieved": null,
      "schema_failures": 0,
      "stage_failed": null,
      "top_rerank_score": 2.9471023082733154
    }
  ],
  "retrieval_ablation": [
    {
      "configuration": "vector-only top-8",
      "k": 8,
      "mode": "vector",
      "n": 24,
      "recall": {
        "n": 24,
        "rate": 0.7083333333333334,
        "successes": 17,
        "wilson_95": [
          0.5083230633550337,
          0.8508535180303913
        ]
      }
    },
    {
      "configuration": "+lexical/RRF top-8",
      "k": 8,
      "mode": "hybrid",
      "n": 24,
      "recall": {
        "n": 24,
        "rate": 0.7083333333333334,
        "successes": 17,
        "wilson_95": [
          0.5083230633550337,
          0.8508535180303913
        ]
      }
    },
    {
      "configuration": "+rerank top-8",
      "k": 8,
      "mode": "hybrid_rerank",
      "n": 24,
      "recall": {
        "n": 24,
        "rate": 0.75,
        "successes": 18,
        "wilson_95": [
          0.55100555994846,
          0.8800063377140499
        ]
      }
    }
  ]
}