rohitg00--ai-engineering-from-scratch
wehub-resource-sync
103 行
3.9 KiB
JSON
103 行
3.9 KiB
JSON
{
|
|
"lesson": "28-long-context-evaluation",
|
|
"title": "Long-Context Evaluation — NIAH, RULER, LongBench, MRCR",
|
|
"questions": [
|
|
{
|
|
"stage": "pre",
|
|
"question": "What does the original NIAH benchmark measure?",
|
|
"options": [
|
|
"Tokenizer fertility",
|
|
"Whether the model can retrieve a planted fact at controlled depths across a long context",
|
|
"Embedding cosine drift",
|
|
"Multi-hop reasoning only"
|
|
],
|
|
"correct": 1,
|
|
"explanation": "NIAH = needle in a haystack: plant a fact, ask the model to retrieve it, sweep depth and length."
|
|
},
|
|
{
|
|
"stage": "pre",
|
|
"question": "Why is the advertised context window often very different from the usable context?",
|
|
"options": [
|
|
"Attention degrades with length and task; spec-sheet maximums rarely hold under multi-hop or reasoning loads",
|
|
"Tokenizers truncate",
|
|
"Embeddings overflow",
|
|
"Beam search slows"
|
|
],
|
|
"correct": 0,
|
|
"explanation": "Effective context for reasoning is usually 25-50% of the advertised max."
|
|
},
|
|
{
|
|
"stage": "check",
|
|
"question": "What does RULER add over NIAH?",
|
|
"options": [
|
|
"Translation tasks",
|
|
"Faster inference",
|
|
"Per-token logprobs",
|
|
"Thirteen task types across retrieval, multi-hop tracing, aggregation, and QA at multiple context lengths"
|
|
],
|
|
"correct": 3,
|
|
"explanation": "RULER expands NIAH into a multi-task long-context benchmark catching models that saturate NIAH but fail elsewhere."
|
|
},
|
|
{
|
|
"stage": "check",
|
|
"question": "What is the 'lost in the middle' effect?",
|
|
"options": [
|
|
"Models under-attend to content placed in the middle of long inputs; depth=0.5 often performs worse than depth=0 or 1",
|
|
"Models reorder tokens",
|
|
"Models forget the first token",
|
|
"Models lose punctuation"
|
|
],
|
|
"correct": 0,
|
|
"explanation": "Mid-context content is least attended; sweeping depth exposes the U-shaped accuracy curve."
|
|
},
|
|
{
|
|
"stage": "check",
|
|
"question": "Why must NIAH-only evaluation be supplemented with multi-hop tests?",
|
|
"options": [
|
|
"Multi-hop is faster",
|
|
"NIAH lacks ground truth",
|
|
"NIAH cannot run on long context",
|
|
"Frontier models can ace single-needle retrieval but still fail multi-hop variable-tracing or aggregation tasks"
|
|
],
|
|
"correct": 3,
|
|
"explanation": "Retrieval pass does not imply reasoning pass; multi-hop benchmarks expose the real ceiling."
|
|
},
|
|
{
|
|
"stage": "post",
|
|
"question": "What does NoLiMa stress?",
|
|
"options": [
|
|
"Streaming output",
|
|
"Latency",
|
|
"Tokenization",
|
|
"Needles that share no literal tokens with the query, so retrieval requires a semantic reasoning step"
|
|
],
|
|
"correct": 3,
|
|
"explanation": "NoLiMa removes lexical overlap so the model must reason rather than match keywords."
|
|
},
|
|
{
|
|
"stage": "post",
|
|
"question": "What two numbers should a long-context spec sheet report?",
|
|
"options": [
|
|
"Effective retrieval length (e.g. 90% NIAH pass) and effective reasoning length (e.g. 70% multi-hop pass)",
|
|
"GPU memory and latency only",
|
|
"Only the advertised max",
|
|
"Tokens per second only"
|
|
],
|
|
"correct": 0,
|
|
"explanation": "Distinguishing retrieval-effective from reasoning-effective length is essential for real-world claims."
|
|
},
|
|
{
|
|
"stage": "post",
|
|
"question": "Why measure time-to-first-token at long context lengths?",
|
|
"options": [
|
|
"1M-token prefills can take tens of seconds; accuracy alone hides product-impacting latency",
|
|
"Tokenization is slow",
|
|
"Required by RAG",
|
|
"Beam search depends on it"
|
|
],
|
|
"correct": 0,
|
|
"explanation": "Long prompts have huge prefill costs; latency must be tracked alongside accuracy."
|
|
}
|
|
]
|
|
}
|