karpathy--llm.c
Andrej Karpathy
125 行
3.5 KiB
Plaintext
125 行
3.5 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Simple visualizer for log files written by the training loop"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import matplotlib.pyplot as plt\n",
|
|
"%matplotlib inline"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def parse_logfile(logfile):\n",
|
|
" streams = {} # stream:str -> (steps[], values[])\n",
|
|
" with open(logfile, \"r\") as f:\n",
|
|
" for line in f:\n",
|
|
" parts = line.split()\n",
|
|
" assert len(parts) == 2\n",
|
|
" step = int(parts[0].split(\":\")[1])\n",
|
|
" stream = parts[1].split(\":\")[0]\n",
|
|
" val = float(parts[1].split(\":\")[1])\n",
|
|
" if not stream in streams:\n",
|
|
" streams[stream] = ([], [])\n",
|
|
" xs, ys = streams[stream]\n",
|
|
" xs.append(step)\n",
|
|
" ys.append(val)\n",
|
|
" return streams\n",
|
|
"\n",
|
|
"# parse_logfile(\"../log124M/main.log\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"sz = \"124M\"\n",
|
|
"loss_baseline = {\n",
|
|
" \"124M\": 3.424958,\n",
|
|
" \"350M\": 3.083089,\n",
|
|
" \"774M\": 3.000580,\n",
|
|
" \"1558M\": 2.831273,\n",
|
|
"}[sz]\n",
|
|
"hella_baseline = {\n",
|
|
" \"124M\": 0.294463,\n",
|
|
" \"350M\": 0.375224,\n",
|
|
" \"774M\": 0.431986,\n",
|
|
" \"1558M\": 0.488946,\n",
|
|
"}[sz]\n",
|
|
"\n",
|
|
"# assumes each model run is stored in this way\n",
|
|
"logfile = f\"../log{sz}/main.log\"\n",
|
|
"streams = parse_logfile(logfile)\n",
|
|
"\n",
|
|
"plt.figure(figsize=(16, 6))\n",
|
|
"\n",
|
|
"# Panel 1: losses: both train and val\n",
|
|
"plt.subplot(121)\n",
|
|
"xs, ys = streams[\"trl\"] # training loss\n",
|
|
"plt.plot(xs, ys, label=f'llm.c ({sz}) train loss')\n",
|
|
"print(\"Min Train Loss:\", min(ys))\n",
|
|
"xs, ys = streams[\"tel\"] # validation loss\n",
|
|
"plt.plot(xs, ys, label=f'llm.c ({sz}) val loss')\n",
|
|
"# horizontal line at GPT-2 baseline\n",
|
|
"if loss_baseline is not None:\n",
|
|
" plt.axhline(y=loss_baseline, color='r', linestyle='--', label=f\"OpenAI GPT-2 ({sz}) checkpoint val loss\")\n",
|
|
"plt.xlabel(\"steps\")\n",
|
|
"plt.ylabel(\"loss\")\n",
|
|
"plt.yscale('log')\n",
|
|
"plt.legend()\n",
|
|
"plt.title(\"Loss\")\n",
|
|
"print(\"Min Validation Loss:\", min(ys))\n",
|
|
"\n",
|
|
"# Panel 2: HellaSwag eval\n",
|
|
"plt.subplot(122)\n",
|
|
"xs, ys = streams[\"eval\"] # HellaSwag eval\n",
|
|
"plt.plot(xs, ys, label=f\"llm.c ({sz})\")\n",
|
|
"# horizontal line at GPT-2 baseline\n",
|
|
"if hella_baseline:\n",
|
|
" plt.axhline(y=hella_baseline, color='r', linestyle='--', label=f\"OpenAI GPT-2 ({sz}) checkpoint\")\n",
|
|
"plt.xlabel(\"steps\")\n",
|
|
"plt.ylabel(\"accuracy\")\n",
|
|
"plt.legend()\n",
|
|
"plt.title(\"HellaSwag eval\")\n",
|
|
"print(\"Max Hellaswag eval:\", max(ys))\n"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "pytorch3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.14"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|