rohitg00--ai-engineering-from-scratch
wehub-resource-sync
79 行
3.6 KiB
JSON
79 行
3.6 KiB
JSON
{
|
|
"lesson": "69-end-to-end-rag-system",
|
|
"title": "End-to-End RAG System",
|
|
"questions": [
|
|
{
|
|
"stage": "pre",
|
|
"question": "Why is the integration test the only honest grade of a six-component RAG pipeline?",
|
|
"options": [
|
|
"The chunker is the only component that matters",
|
|
"Each stage's isolated metric does not predict the system's metric because each stage interacts with the next; the only valid grade is the whole pipeline run end to end against the same qrels",
|
|
"Production servers cannot run individual stage tests",
|
|
"The components run on different machines"
|
|
],
|
|
"correct": 1,
|
|
"explanation": "Isolated wins do not compose; the pipeline is graded on the joint output."
|
|
},
|
|
{
|
|
"stage": "pre",
|
|
"question": "What is the role of refuse-on-low-confidence in the answer generator?",
|
|
"options": [
|
|
"It logs every refusal to disk",
|
|
"It returns 'I do not know' when the reranker top-1 score sits below a corpus-specific threshold, preventing hallucinated answers when the retrieved context does not address the query",
|
|
"It blocks queries longer than 100 tokens",
|
|
"It triggers a re-retrieval with a different chunker"
|
|
],
|
|
"correct": 1,
|
|
"explanation": "A consistently low rank-1 score is the model telling you nothing in the corpus answers this query; refuse rather than hallucinate."
|
|
},
|
|
{
|
|
"stage": "check",
|
|
"question": "Why are the rerank training triples kept disjoint from the eval queries?",
|
|
"options": [
|
|
"To save GPU memory",
|
|
"To prevent train-test leakage; if the eval queries overlap the rerank training set the eval looks magical and the regression test does not catch real drift",
|
|
"Because the cross-encoder requires distinct vocabularies",
|
|
"Because RRF cannot fuse overlapping lists"
|
|
],
|
|
"correct": 1,
|
|
"explanation": "Strict separation is the basic hygiene; otherwise the rerank metric does not predict production behavior."
|
|
},
|
|
{
|
|
"stage": "check",
|
|
"question": "What is the purpose of the self-terminating demo's exit code?",
|
|
"options": [
|
|
"Cosmetic",
|
|
"It encodes pass/fail against the metric thresholds; a CI smoke test can plug the demo into the pipeline and gate the merge on a non-zero exit",
|
|
"It is required by Python",
|
|
"The reranker uses it as a training signal"
|
|
],
|
|
"correct": 1,
|
|
"explanation": "Exit code 0 means thresholds met; non-zero means regression. That is the contract a CI gate consumes."
|
|
},
|
|
{
|
|
"stage": "check",
|
|
"question": "What does a citation anchor look like in this pipeline?",
|
|
"options": [
|
|
"[doc_id:chunk_index]",
|
|
"Markdown link with the source URL",
|
|
"JSON-RPC envelope",
|
|
"The literal embedding vector"
|
|
],
|
|
"correct": 0,
|
|
"explanation": "The generator emits the (doc_id, chunk_index) tuple as an anchor so a downstream UI or auditor can map every claim back to its source chunk."
|
|
},
|
|
{
|
|
"stage": "post",
|
|
"question": "Which failure mode is invisible to the mock generator and must be tested with a real LLM before shipping?",
|
|
"options": [
|
|
"Latency",
|
|
"Hallucination: the mock generator only emits text from retrieved chunks and cannot invent content; a real LLM can ignore the chunks and generate ungrounded answers",
|
|
"Tokenization",
|
|
"Vector index size"
|
|
],
|
|
"correct": 1,
|
|
"explanation": "The mock cannot test the generator's safety against hallucinated content; only a real model swap-in can."
|
|
}
|
|
]
|
|
}
|