{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# K-fold cross validation - Regression Model\n", "Based on the [Ludwig regression example](https://ludwig-ai.github.io/ludwig-docs/examples/#simple-regression-fuel-efficiency-prediction) \n", "\n", "[Data set](https://archive.ics.uci.edu/ml/datasets/auto+mpg)\n", "\n", "This example demonstrates teh following:\n", "\n", "- Download a data set and create a pandas dataframe\n", "- Create a training and hold-out test data sets\n", "- Create a Ludwig config data structure from the pandas dataframe\n", "- Run a 5-fold cross validation analysis with the training data\n", "- Use Ludwig APIs to train and assess model performance on hold-out test data set" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import logging\n", "import os\n", "import os.path\n", "import shutil\n", "import tempfile\n", "\n", "import matplotlib.pyplot as plt\n", "import pandas as pd\n", "import requests\n", "import seaborn as sns\n", "from sklearn.model_selection import train_test_split\n", "\n", "from ludwig.api import kfold_cross_validate, LudwigModel" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Contstants" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "DATA_SET_URL = \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\"\n", "DATA_SET = \"auto_mpg.data\"\n", "RESULTS_DIR = \"results\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Clean out previous results" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "if os.path.isfile(DATA_SET):\n", " os.remove(DATA_SET)\n", "\n", "shutil.rmtree(RESULTS_DIR, ignore_errors=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Retrieve data from UCI Machine Learning Repository" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Download required data" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "r = requests.get(DATA_SET_URL)\n", "if r.status_code == 200:\n", " with open(DATA_SET, \"w\") as f:\n", " f.write(r.content.decode(\"utf-8\"))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Create Pandas DataFrame from downloaded data" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(398, 8)" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "raw_df = pd.read_csv(DATA_SET, header=None, na_values=\"?\", comment=\"\\t\", sep=\" \", skipinitialspace=True)\n", "\n", "\n", "raw_df.columns = [\"MPG\", \"Cylinders\", \"Displacement\", \"Horsepower\", \"Weight\", \"Acceleration\", \"ModelYear\", \"Origin\"]\n", "raw_df.shape" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
MPGCylindersDisplacementHorsepowerWeightAccelerationModelYearOrigin
018.08307.0130.03504.012.0701
115.08350.0165.03693.011.5701
218.08318.0150.03436.011.0701
316.08304.0150.03433.012.0701
417.08302.0140.03449.010.5701
\n", "
" ], "text/plain": [ " MPG Cylinders Displacement Horsepower Weight Acceleration ModelYear \\\n", "0 18.0 8 307.0 130.0 3504.0 12.0 70 \n", "1 15.0 8 350.0 165.0 3693.0 11.5 70 \n", "2 18.0 8 318.0 150.0 3436.0 11.0 70 \n", "3 16.0 8 304.0 150.0 3433.0 12.0 70 \n", "4 17.0 8 302.0 140.0 3449.0 10.5 70 \n", "\n", " Origin \n", "0 1 \n", "1 1 \n", "2 1 \n", "3 1 \n", "4 1 " ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "raw_df.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Create train/test split" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(318, 8)\n", "(80, 8)\n" ] } ], "source": [ "train_df, test_df = train_test_split(raw_df, train_size=0.8, random_state=17)\n", "print(train_df.shape)\n", "print(test_df.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Setup Ludwig config" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "num_features = [\"Cylinders\", \"Displacement\", \"Horsepower\", \"Weight\", \"Acceleration\", \"ModelYear\"]\n", "cat_features = [\"Origin\"]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Create Ludwig input_features" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "input_features = []\n", "# setup input features for number variables\n", "for p in num_features:\n", " a_feature = {\n", " \"name\": p,\n", " \"type\": \"number\",\n", " \"preprocessing\": {\"missing_value_strategy\": \"fill_with_mean\", \"normalization\": \"zscore\"},\n", " }\n", " input_features.append(a_feature)\n", "\n", "# setkup input features for categorical variables\n", "for p in cat_features:\n", " a_feature = {\"name\": p, \"type\": \"category\"}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Create Ludwig output features" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "output_features = [{\"name\": \"MPG\", \"type\": \"number\", \"num_fc_layers\": 2, \"fc_size\": 64}]" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'input_features': [{'name': 'Cylinders',\n", " 'type': 'number',\n", " 'preprocessing': {'missing_value_strategy': 'fill_with_mean',\n", " 'normalization': 'zscore'}},\n", " {'name': 'Displacement',\n", " 'type': 'number',\n", " 'preprocessing': {'missing_value_strategy': 'fill_with_mean',\n", " 'normalization': 'zscore'}},\n", " {'name': 'Horsepower',\n", " 'type': 'number',\n", " 'preprocessing': {'missing_value_strategy': 'fill_with_mean',\n", " 'normalization': 'zscore'}},\n", " {'name': 'Weight',\n", " 'type': 'number',\n", " 'preprocessing': {'missing_value_strategy': 'fill_with_mean',\n", " 'normalization': 'zscore'}},\n", " {'name': 'Acceleration',\n", " 'type': 'number',\n", " 'preprocessing': {'missing_value_strategy': 'fill_with_mean',\n", " 'normalization': 'zscore'}},\n", " {'name': 'ModelYear',\n", " 'type': 'number',\n", " 'preprocessing': {'missing_value_strategy': 'fill_with_mean',\n", " 'normalization': 'zscore'}}],\n", " 'output_features': [{'name': 'MPG',\n", " 'type': 'number',\n", " 'num_fc_layers': 2,\n", " 'fc_size': 64}],\n", " 'training': {'epochs': 100, 'batch_size': 32}}" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "config = {\n", " \"input_features\": input_features,\n", " \"output_features\": output_features,\n", " \"trainer\": {\"epochs\": 100, \"batch_size\": 32},\n", "}\n", "config" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Perform K-fold Cross Validation analysis" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "starting 5-fold cross validation\n", "training on fold 1\n", "CPU times: user 40.7 s, sys: 5.38 s, total: 46 s\n", "Wall time: 40.6 s\n" ] } ], "source": [ "%%time\n", "with tempfile.TemporaryDirectory() as tmpdir:\n", " data_csv_fp = os.path.join(tmpdir, \"train.csv\")\n", " train_df.to_csv(data_csv_fp, index=False)\n", "\n", " (kfold_cv_stats, kfold_split_indices) = kfold_cross_validate(\n", " num_folds=5, config=config, dataset=data_csv_fp, output_directory=tmpdir, logging_level=logging.ERROR\n", " )" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'loss_mean': 8.111681,\n", " 'loss_std': 2.4598064,\n", " 'error_mean': 0.0380627,\n", " 'error_std': 0.5965346,\n", " 'mean_squared_error_mean': 8.111682,\n", " 'mean_squared_error_std': 2.4598064,\n", " 'mean_absolute_error_mean': 2.0598435,\n", " 'mean_absolute_error_std': 0.2779836,\n", " 'r2_mean': 0.8666786,\n", " 'r2_std': 0.03552912}" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "kfold_cv_stats[\"overall\"][\"MPG\"]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Train model and assess model performance" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "model = LudwigModel(config=config, logging_level=logging.ERROR)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 8.34 s, sys: 1.78 s, total: 10.1 s\n", "Wall time: 15 s\n" ] } ], "source": [ "%%time\n", "training_stats = model.train(\n", " training_set=train_df,\n", " output_directory=RESULTS_DIR,\n", ")" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/project/ludwig/data/preprocessing.py:1045: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " computed_fill_value,\n" ] } ], "source": [ "test_stats, mpg_hat_df, _ = model.evaluate(dataset=test_df, collect_predictions=True, collect_overall_stats=True)" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'MPG': {'loss': 8.303831,\n", " 'error': -0.45136052,\n", " 'mean_squared_error': 8.303831,\n", " 'mean_absolute_error': 2.2274728,\n", " 'r2': 0.8558148},\n", " 'combined': {'loss': 8.303831}}" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_stats" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.6/dist-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.\n", " FutureWarning\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQ8AAAEKCAYAAAAM4tCNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO2deXyV5Zn3v1cCkSVAIAkQNlnCBFFWEVCsgzqtYq1axe21o2Ndal9nxplWW+lMO52ZzttW/LR2fK0Oii86dbdYLVQZi4CliIgoO1ES1kDIQkLIRrbr/eM853hycnK25OzX9/PJ55znfrbrwHN+576v+7quW1QVwzCMcMmItwGGYSQnJh6GYUSEiYdhGBFh4mEYRkSYeBiGEREmHoZhRESfaF5cRA4Cp4F2oE1V54jIMOAVYDxwELhJVWuiaYdhGL1PLHoel6rqTFWd42w/DKxV1cnAWmfbMIwkIx7DlmuB55z3zwHXxcEGwzB6iEQzwlREDgA1gAL/parLRKRWVXOc/QLUuLd9zr0XuBdg4MCB50+ZMiVqdhpGutLWrpRW1XP66GdVqpofzrlR9XkAF6tqmYgMB94VkX3eO1VVRcSveqnqMmAZwJw5c3Tr1q1RNtUw0ouKumZueXozLaea2fvviw6Fe35Uhy2qWua8VgBvAHOBEyJSAOC8VkTTBsMwuuIWjvJTzay4c25E14iaeIjIQBEZ5H4PfAXYBbwF3OEcdgfwZrRsMAyjK77CMXfCsIiuE81hywjgDZdbgz7Ai6r6joh8BLwqIncBh4CbomiDYRhe9JZwQBTFQ1VLgRl+2quBy6N1X8Mw/NObwgEWYWoYaUFvCweYeBhGyhMN4QATD8NIaaIlHGDiYRgpSzSFA0w8DCMlibZwgImHYaQcsRAOMPEwjJQiVsIBJh6GkTLEUjjAxMMwUoJYCweYeBhG0hMP4QATD8NIauIlHGDiYRhJSzyFA0w8DCMpibdwgImHYSQdiSAcYOJhGElFoggHmHgYRtKQSMIBJh6GkRQkmnCAiYdhJDyJKBxg4mEYCU2iCgeYeBhGwpLIwgEmHoaRkCS6cICJh2EkHMkgHGDiYRgJRbIIB5h4GEbCkEzCASYehpEQJJtwgImHYcSdZBQOMPEwjLiSrMIBJh6GETeSWTjAxMMw4kKyCweYeBhGzEkF4QATD8OIKakiHGDiYRgxI5WEA0w8DCMmpJpwgImHYUSdVBQOMPEwjKiSqsIBMRAPEckUkU9EZJWzPUFEPhSR/SLyiohkRdsGw4gHqSwcEJuexwPAXq/tnwO/VNVCoAa4KwY2GEZMSXXhgCiLh4iMAb4KPONsC3AZ8LpzyHPAddG0wTBiTToIB0S/5/EY8D2gw9nOBWpVtc3ZPgqM9neiiNwrIltFZGtlZWWUzTSM3iFdhAOiKB4icjVQoaofR3K+qi5T1TmqOic/P7+XrTOM3iedhAOgTxSvvQC4RkSuAvoBg4FfATki0sfpfYwByqJog2HEhHQTDohiz0NVl6jqGFUdD9wCvKeqtwHrgMXOYXcAb0bLBsOIBekoHBCfOI/vA98Rkf24fCDL42CDYfSItnaXGy9dhQOiO2zxoKrrgfXO+1Jgbizuaxi9TWllPW/vKufP+6uYMTaHVduPUd3QknbCATESD8NIBUor67nxqQ+obmgBYFNJNQCP3Twj7YQDLDzdMELm7V3lHuHwpqy2OQ7WxB8TDyMpcPsY4nWvtvYO/ry/yu/xm0qqaI+hfYmCDVuMhMbbx7CgMI9F541kYn52zO/VJzODGWNz+OjgSTo6YPqYIeRmZ3GgqpGLJuWRmZl+v8MmHkbC4s/H8OzGA7x234W9LiDB7lVR18zvtx9DFc4pGMRFhblMLRhMcflpFhalZxBj+smlkTT48zFUN7Tw9q7ymN6roq6ZG57cxNGaJto6lF3H6nhiXQk/enM3BTn9uX35Fkor63vdpkTHxMNISHriYwjXPxLoXuuLK7hl2WbK67o6RasbWthfUU/OgKyoCFqiY+JhJCR9MjNYUJjnd193PobSynqeWLef25/dwhPr9ofcGwh0r5LKBsrrmikaMcjv/n3ldYzPG5CWTlMTDyNhWXTeSHIHdq4VlTswi0XnjexyrNtnsXRNMZtKqlm6ppgbn/ogZAHxd68MgcaWNlbcOZdF0wr8njdl5GAOpqnTNL0+rZFUTMzP5rX7LuShK4pYUJjLQ1cUdess7al/xPteF4wfyrCBWWT1yeD5b85j7oRh3QpZ4fBsahtb/ApaqiOqGm8bgjJnzhzdunVrvM0w4kh7e0e3v+xt7R3c/uwWT8SnNwsKc3n+zrkh9woq6po9Pg7fkHP3VO6mkipmjRvK1IJBHD/VzKVFw6M2fRwrRORjVZ0Tzjk2VWskBYG+/G6fhT/xCGc44Uly8yMc4Oqd3H9pIfddMpHMzIyAgpYOpO8nN1KKcPwj/mZjwsmOdQtGOgsHWM/DSBHcPgv3sOKiSV2jUbuLIE3ntPqeYD4PI+XwHk60tXfQJzOjSwQpuHomT31jNt9fuTPthcN8Hkba4y0Wm0urKattYnROf8YNG0CHzw9ldUML3/rNNppb29NaOCLFxMNICbyHJAuL8pkxJocDVQ3sPlZHc2sHWZkZ/PPVU/nuq9s7nVfb2MKLd88z4YiAgOIhIv8ZwjXqVPWfe8kewwgb3yHJ12aM4n+/sK1TklvuwCwevXEGk/KzKfEKHLv5grHMn+Q/utQITLCex7XAj4Ic8zBg4mHEDe8AscLh2eyvqPcbMPbhgWpmjB3iEY/B/fpwz5cmxtxe99Aq2QkmHr9U1ecCHSAiQ3vRHsMIC9+ktvG5A9h7vM7vsR8fquHIySYyxNXjuOdLE2Ma3BXL2iSxIKB4qOpjwS4QyjGGES18A8QOVDWysCjfb8DY/op6zrR18OLd82I+VIllbZJYEbDvJCLnisg1Xtu/FJFnnb/Z0TfPMILjHSBWUlnPlJGD/Ca5NTmzKvHwccSyNkmsCDbw+hngXejgCmA1roWbgvlCDCMm+CbQjR3anyWLpnDXxRO4YPxQhvTvS5+MDB65YVpcZlVStf5pMPEoUNVNXtt1qvpbVf1vwFzURkwJVOTHnXfy/J1z+fhwLQ++voM/7jnB7mN11DW10tLewZGa+FQ5j6Q2STIQzOpOFVBUdb7X5vDeN8cwuhJOkR8Fz6/8oZONNLa04w4Ni+evfDi5N8lCsNmWYyIyT1U/9G4UkfnAseiZZRguQnU0uqc/3VXOe5ph29uEknuTbAQTj+8Dr4jICmCb03Y+rgWqb46iXYYBfOFonJSfzYS8ARyoaqTEmfK8/9LCLtOfc8cPZdWOrr9rifAr75vSn+wEm6rdIiLzgL8F/sZp3g3MV9UTUbbNSHPOtLaz40gtP71+Gvsr6tl7vI6FRfnc/aUJrC+uoKTiNDf91+ZOvZIMgaw+GTx28wzKapsT8lc+FYQDQsttGQXsAF5S1b1RtscwPEltTa1t3H3JRO7774+7hJovu/183vz0WJfpzw6Fr88azXWzxgCkzK98IhIszuNHwKvADcBqEbknJlYZacvBqnq2HaqhpLKBtXsr2VBcyYNXFDF0QF/PMdUNLRyubmTroRoAzh42gAFZmYiz//DJRo9j1IQjegTredwMzFTVRhHJBd4Bno6+WUa6crSmiZ++va9LT+N7V05hycqdnuM2lVYxb0Iuf/kX+fx6fQnNre1cNa2Aiyfn0djSZqIRA4L9C59R1UYAVa0O4XjDiJi29g427q/yG4m5v6KeSV4+i8H9spgyMpufv7OPU02tdCis3nmcR9cUM39ibqxNT0uC9TwmishbznsBJnlto6rX+D/NMCJjx9FTftvdiyuVVNaTOzCLSfkD+d5vd9LhUwivuqGF9cWVnDtqSAysTW9CScn35tFoGWIYfTIzmD8x12+MxqyxQzlxuomHrihi+pgh3P/CNurPtPm9zqaSKnOUxoBgU7UbIr2wiPQD3gfOcu7zuqr+i4hMAF4GcoGPgb9W1Zbur2SkAqHWsLh6egHPbTrYpdbo9bNHc/awAVQ3tHDL05tpbe/gulmjWbmtrMs1kjnkO5kIVklsR6D9qjo9wO4zwGWqWi8ifYGNIvI28B1cdUJeFpGngLuAJ8O020gSwq1hESgS07vK+XPfnEdedhYbiiu7CE28g8HShYDV00XkU1zpAi8CvweavPer6qGQbiIyANgIfBtXVu5IVW0TkQuBH6vqFYHOt+rpyUl3FctDrWHhXQW9u+URymoa2fBZJat3Ho9qMFiqVP/qjl6vnq6qM0VkCnArLgHZ47z+j6r6H3B2NigT19CkEHgCKAFqvc49Cozu5tx7gXsBxo0bF9KHMRKLQDUs7r+0MOj5gYTDt0fzb9ecy6Th/ley7wmpVv2rNwlr3RYRuRmXCPxcVZeGcV4O8AbwQ2CFqhY67WOBt1X1vEDnW88j+eit9WO9hWP5HXO4cFJej3s0oRKr+yQCkfQ8gv7vichoEfmuiGwEvgH8I2H6KFS1FlcBoQuBHBFx93jGAF09XkbS0xs1LNzCcay2iWtnjuLx9/bz4oeHWLXjeEyqcqVi9a/eJFh4+gZcvo6+wJ24smlXA1kiErAkk4jkOz0ORKQ/8GVgLy4RWewcdgfwZk8+gJG49KSGhbdwZGVm8NKWI2wqqea9fRVsLu3am4HerdeRqtW/epNgcR5n43KYfgvH/+AgTnuguvUFwHOO3yMDeFVVV4nIHuBlEfkJ8AmwPFLjjcTAnzOxrb0j4hoW3kOVr88azUtbjnj2BSpw3JtTtL6FlaN1n2QmmMN0fKQXVtUdwCw/7aXA3EivayQO/pyJGQKrd3ZuC6eGha+P4/H39nfaX1JZz91fmkDuwKyoT9EuOm8kz248YFPB3RBsqnakqgYc4IVyTE8xh2ni0Z0zccmiKTz4+o5ObaE6GP3Nqjyxbj9L1xR3KgZ0suEMj986i+1HT0W9XodbIBOxLkhvEo2Frv8ABFtiIZRjjBSirb2DzaXVfp2Je8tPd1rSMdSp2e7iOL46bSQjBp3F3vLTnmJA54wcxJih/bl4cn7Uw9BTrfpXbxJMPGaIiP/lt1wIEGi/kUJ4D1OmjxnCT6+fxiPv7KOmsdVzjHcCm5tguSbdCQe4ivv4S9F/7b4LgdjV6zDh6Eown0dmrAwxEhO3M9RfIWJ/dTamjBzM+uLKTtcI5GAMJBzQ80AzI3qYnBp+8V7uIFBshXedjdyBWZwzclCnXkcgB2Mw4bDp0sQmlBqmRprh28sYkJVJY0u732P3ldex6LwR9M8a7ZlteeiKoqAOxmDCATZdmuiYeBhd8B0qBIutuO+SiSh4Yj2CORhDEQ43Nl2auIQkHiIyCTiqqmdEZCEwHXjeCTs3Ugh/Q4VAsRULi/J56v3SLolj3QUAhCMckJqLJaUKISXGOan5c4DxuKZm3wTOVdWromqdg8V5xBZ3bIU3Qwf07RJbsbAon79/8RNKqho8x+UOzOJXt8zk1+tLumShhiscvnin6Bu9S1QS4xw6nDT6rwOPq+pDuMLPjRTEX05Khgijcvp7FpO+/9JC1hdXdhIOcDlR1xVXcqLuDEvXFHPjUx9QWlkfknAEWsgabLo00QjV59EqIrfiSmT7mtPWN8DxRhITbKiQmZkRcCbEO9ajuqGF1z4+yprd5d0Kh9XMSE5CHbZMBe4DPlDVl5w6pDep6s+jbSDYsKW3iKQaVqChgr/hDcBdF09w9UqcKdv+fTMQkW6FI11qZiQyURu2qOoeVf17VX3J2T4QK+Eweo53zMYT6/ZT6hWHEQy3cPgbUnSXcl84PLtTrEdbh3Y7VLGaGclLqLMtC4Af40rR74OTkq+qgVLyjTjh3cPwFxn67MYDIf+yBxpS+A5v5k3IZdSQfvzHHzovabx08fRufRyBgsAsnySxCdXnsRxXBbGPAf/RQkbc8f2iXzOjoEfh3aEIj2/iWGllPbfMHceKPx+grUNZuni6Z9FpXywILLkJVTxOqerbUbXE6BH+vuifnzhNxekzfo8P5Zc9HOFxXyf7rD6s2V2OiPDC3fOCTsdaEFjyEqp4rBORpcBKXOuxAKCq26JilRE2/r7oO8vqWDRtZES/7OEMKdzDpEjiOCwILHkJVTzmOa/e3lgFLutdc4xI6O6LXlJZz9SCQRFV3QplSOE9TJoxNodV249R3dASdgCY1cxITkISD1W9NNqGGJET6It+/FRzxL/sgYYU/oZJAI/dPCPsyFE3JhzJRaizLUOAfwEucZo2AP+mqv6XNDdiTndf9EuLhgOu8PKZY3IYOqBrbF938R8T87N55VvzWbP7RBfheWLd/i7DJICy2uZe/FRGIhPqsOVZYBdwk7P918D/A66PhlFG+HTnO8gQuOFJ/0FYQLfTsN5DkqunF7D0humMGjoAsClWw0Wo4jFJVW/w2v5XJ1nOSCD8+Q789RCqG1pYteM4Gz6r5ONDNUDnaVjAb9Uw9xRtn8wMZozNsSnWNCfU/+UmEbnYveEEjTUFON6II96zIN31ED48UE2OzxCmuqGFzaXVQaM+K+qaWbX9WJdr2hRrehFqz+PbuBZwGoIruvQk8DfRMsroHQI5UqeNHsK7eyq6tB+rbWLbYf9lWjaVVHH9zFHc9uwWqhtaeOzmGZTVNtsUa5oS6mzLp7gqqQ92tq1iepLQnSP14sI8ntpQ2uX4UTn96Z/Vx6/gTB+Tw23PbukSx2E+jvQkoHiIyDdU9Tci8h2fdgBU9RdRtM3oBQI5Uv3Ff8yfmAvQRXCG9u/bbRyHP+GIJIPXSC6C9TwGOq+D/OwLnstvJATdBWEFiv/w3jd9TOAAMN9EPKvNkR6EWs9jgar+OVhbtLB6HtElUM2O4zWNfocq0FUouitLaLU5Ep9oliF8PMQ2IwkJVOU8kHDc+NQHLF1TzKaSapauKeb25Vu4+5LOVRqsNkfqEszncSFwEZDv4/cYDNhqcilMpCu5uReBCme5SSM5Cfa/mQVk4xKZQV5/dcDi6Jpm9JRgBYW7oycrubnrl3pjgWOpSbC1ajcAG0RkhaoeipFNRoR4ryvrz2kZygxIT1dym1owmPf2fbFWrQWOpS6hBok9IyI3uhd5EpGhwMuqekX0TDNCxVss5k90lQJ85k+l1DS2svd4HSMGncWqHcfZXFodcAakN1Zyu27WaHIGZFngWBoQ6mzLJ6o6K1hbtLDZlu7jJrqrPv7gFUUsWbmTn14/jUfXFAetTh5uIZ+DVfVsPVjD3vLT7CuvY8rIwZwzchBzxg9lfF62LdCUZER10ScRGed1o7OxOI+YEKzyeSDH5WVFw9lfUR+0OnkkFcBW7yznwdd3sL64kn59M1lfXMmDr+9g9U7XdU04Up9Q/4f/CdgoIv8tIr8B3geWBDpBRMaKyDoR2SMiu0XkAad9mIi8KyKfO69De/YRUhd/06HuFdgguONyxrgh7D3uP5NgU0kV7e0dEQmH931LKutZu7fCM7vivq6R+oS6bss7wGzgFeBl4HxVXRPktDbgu6o6FZgP3O8sHvUwsFZVJwNrnW3DD8GyW92OS39MGTmYTw+f4pyCwX73XzQpj+qGFo9wLL9jTlgVwK6e7n+1UZtZSR8C/i+LyBTndTYwDjjm/I1z2rpFVY+7CySr6mlgLzAauBZ4zjnsOeC6nnyAVCVYwR33r3t3Cy9dXJhHS3s7Fxfm+d0/d/xQbnl6M8dqm7h25igef29/0AWhvIdQVfUtPLp4eqfKZDazkl4EdJiKyNOqeo+IrPOzW1U1pALIIjIe11DnPOCwquY47QLUuLd9zrkXuBdg3Lhx5x86lH4zxd0t5/jQFUWdlj5wz7ZsKqli2ughnJ07kFe3HmHYwCxONrRw05yxHKpuYGfZKaaNHsK5owbzyJpiKk+fISszg7rmNs+1ugsn784x+6tbZvLkhhKbWUlyInGYhjTb0hNEJBtXzdP/UNWVIlLrLRYiUqOqAf0e6TrbEu46ru3tHby45TA/fHN3l30PLyriSHUTG/dXcehkIxkCi88fw6tbj3Y51lecILCQWfRo8hOJeAQLTw9Yo1RVVwY5vy/wW+AFr2NPiEiBqh4XkQKga0UaAwh/TRPFFaTlL9V+SP8sfrbliy//rHE5HK3xXwzOO5zcHaUarGapkX4ECxL7mvM6HFeOy3vO9qXAJlyLQPnFGZIsB/b61P14C7gD+Jnz+mb4ZqcPoaxp4rt+yhO3zWbj51V8cqSGaaNzmDZ6MD94Y6fn+OyzMvnmggnsOV7HppJqJuVnMyFvAAeqGimprOeiSXkcO9XEW9uPewogz5+YazVLjU4EC0+/E0BE/geYqqrHne0CYEWQay/AVWV9p1ex5B/gEo1XReQu4BBfVGQ3AhBIOHyLFb/60RF+ePVU9pbXUVx+mlNNLZxq+sKv8U9fncqP3tzND756Do8uns7e8tPsPV7HwqJ8vv2XEzlvzBCue2JTp2s+unh6RItHGalLqOHpY93C4XAC1+xLt6jqRlz1Tv1xeYj3NYLQ3XTuzrJTHHR6EvmDzvLsKxye7Qkc6+hQfvb2vi5V0h+/dVaXa/7HH/by+K2z2H70lIWeG0Do4rFWRNYALznbNwN/jI5JRqgECxL72oyRPP/BYU43t3qKFTe1tLHtcC2Fw7PZV37ar/C8/3lll7T6msZWntxQwvN3zjUHqQGEXgD5b0Xk63yxYtwyVX0jemYZoRAou3Vi3kB+s/kwjS1tPP/NL1arb2/v4Kn3S/nkcE230ac7y04xPm9AJ/EA828YnQnnSdgGrFbVfwTWiIi/uqZGjFlYlN8lCGxo/768s+sEVfUtZGVmkJf9xf7MzAwWnTeSmsbWbqNP503IpbaxtVOb+TcMX0Jdq/YeXAFbw4BJuCJFn8J8F1Ej1Orjm0urefCKIg5UNbD72CnGDRvAH/dUUFl/BoC65jbe3lXeKW5jYn42//BXk2lrV373SVkXJ+jV0wu4enpBRItjG+lDqD6P+4G5wIcAqvq5iAyPmlVpTDjVx9vaO1i7t4JNJdX89fxxTM7P5oUth2lt7xz451sGsKymkSfXl9DU2s5Pvn4enxyuZfexU0wZOZjzz87h7GEDyMzMCDpFbKQ3oYrHGVVtca/XIiJ9sJT8XsfftKt7/Vh/AuLt83j/sypqm1q7CAd09VVs+KyScwoGs3zjAb79m21Mys9mfN4A1hdXMmxgVqdjTTiM7gj1ydggIj8A+ovIl4HXgN9Hz6z0JFgWrT8WnTeSof37cuhkI00t7Qzu1/n3wNdX0dbewaodxykcnu3xlbjT6msbW7jy3BG9+ImMVCbUnsf3gbuBncC3gD8Az0TLqHQkWBat7/DB7RPJPqsP2f36UN/SxgN/Vch5o3P4oKSKnWWnmDdhGFdPH9Wp1+LurTzyzj6+d+UU9lfUeyqBXVqUz6Th5gc3QiOoeIhIJrBbVacAT0ffpPQk0LSr97DDNxTdvZLb0sXT+fdVe6luaPEMQzZ8VsXsca6cQ28BcdcfXbJyp+fYT4/Uctu8gHF/htGJoOKhqu0iUiwi41T1cCyMSle6KyrsHnb484kAngAwd3tJZb0nRmNdcSWfHqll6eLpHgHxTbibPW6ozaYYYRPqsGUosFtEtgCetQRV9ZqoWJWmBMuiXbXjONUNLZw9bACV9WdoamlHgcMnGzl8stHvNfeV15EzoK/f6VqbTTF6Qqji8cOoWmF46O5L3dbewe6yUyxZNIVfry+hubWdq6YVcPHkPNYXV3DD7DG8/nFZl+tNGTmY9cWVNLe2+xUKEw4jUoLV8+gH3AcU4nKWLlfVtkDnGL2Dvy/1VdNG8o+vbqfDmY1dvdO1FssvbppBzoAsv1mvhcOzWb7xANfPHm1CYfQqwXoezwGtwJ+ARcBU4IFoG5VuhBJNerKhhR//fo9HONxUN7TwyZFapo8ewhO3zWZzSTUfHTrJlJGDKRyezSPv7LPQciMqBBOPqao6DUBElgNbom9S+hBqNGlFXTM3L9vMqaZWP1eBjw6e5NqZo1j85AecnTuAv7tsMvvK61i14xh3XjSeq2eMMmeo0esEEw/P06qqbe4IU6PnhBpN+mFpFd9+4RNONbVy3azRrNzW1a9x0aQ8PiipprqhheqGFu5c8ZFnCjZv0FkmHEZUCDYIniEidc7faWC6+72I+M/nNkLCPXPiTXVDC6t2uGouldU08vbO49z69IecbGihvUO5YPwwv8soXHHuCM95btxRo6t3HrdFmIyoEKwMYWasDEkn2to72FzaNRgM4MMD1fxxzyB+9+kxNu6v6uTjcEeFltU28cnhmk5TuaEEmBlGb2JPVZyYPmaI3/Zpo4fwh13lvLevgjofH0dNYytLVu7kRF0T910yibOH9ffs627xJ3OUGtEi1DgPoxfpk5nBxYV5vLb1aJep1aIRg1i+8QBt7cpV0wpYvfN4l/OHD+rHm9vLGNI/i+bWDjIk/GUaDKOnmHjEiTFD+7Nk0RT2lp/2JKaNHtKP//P2Pk9a/cKifDaXVncRmDFDB/DEuhLP9q9umcn4vGyLGjViiolHDwm14pcv4/Oy6VDo1zeDqQWDaG5t5/+uK6Hy9BnPMRkZ0klgZo7NYczQATzyzj7PMdUNLWzcX8WFE3M9gmHCYcQCE48ICafily/egnPoZBPr9lVQWtVAY0sbo4b049ipZgqHZ7PnWB3LNx5gUn4218wo4EhNo6fH4c3OslO9+tkMIxRMPCIg3Ipf3ue5BWf+xFxGDenHsvdLPAsyDe7Xh6fvmMP64krPEgngmnZ9a/txFhbl+73uBeOHWW/DiDkmHhEQqOKX7wLRbvwJztD+fcnKzARc4lHX3MbavRX8/eWTPUskuKdfSyrruftLE/zmr0wZOYj29g4TECOm2NMWJsEqfnUXkOVPcGqaWqlpbGHUkC+mXD88UO0RAt/p10fe2ceSRVN44PJCFhTmcu8lE3nwiiLKaptMOIyYYz2PMAm14pc3gQRnSsEgRgzux7FTrhXrp4/J8ezznX6dNyEXgD99XsXQgVmeuqOv3Xdhb3w0wwgLE48ICFbxy5dAgjNvQi7riys91/jK1BGdBMh3+rW0sp4Tp5IjfPAAAApVSURBVM+wqaSK62ePtlgOI26IauKvoDBnzhzdunVrvM3ohNv5GWpAVmllPTf8ehM1XlGjuQOzePTGGTyzsdSTQn+6uZV7L5kU9P7m4zB6ExH5WFXnhHOO9TwiJNyALO8q50UjB3FxYR5n5w7kP9/7nGEDs1hfXMnyjQdYUJjLXQsmBL2mCYcRb0w8ekgoX+KKumZueXoz1Q0tvHD3fM4fl8MrW4+wZOXOLsdaIpuRLNhTGmXcwlF+qpkVd85l7gRXTMb8ibmWyGYkNdbziCL+hMONJbIZyY45TKNEIOHwxZyfRryJxGEatSdWRJ4VkQoR2eXVNkxE3hWRz53XodG6fzwJRzjAnJ9GchLNp3YFcKVP28PAWlWdDKx1tlOKcIXDMJKVqImHqr4PnPRpvhbXcg44r9dF6/7xwITDSCdi3V8eoaru0ljlwIjuDhSRe0Vkq4hsraysjI11PcCEw0g34jbYVpentltvraouU9U5qjonP99/KnqiYMJhpCOxFo8TIlIA4LxWxPj+vY4Jh5GuxFo83gLucN7fAbwZ4/v3KiYcRjoTzanal4APgCIROSoidwE/A74sIp8Df+VsJyUmHEa6E7UIU1W9tZtdl0frnrHChMMwLLclbEw4DMOFiUcYmHAYxheYeISICYdhdMbEIwRMOAyjKyYeQTDhMAz/mHgEwITDMLrHxKMbTDgMIzAmHn4w4TCM4Jh4+GDCYRihYeLhhQmHYYSOiYeDCYdhhIeJByYchhEJaS8eJhyGERlpLR4mHIYROWkrHiYchtEz0lI8TDgMo+eknXiYcBhG75BW4mHCYRi9R9qIhwmHYfQuaSEeJhyG0fukvHiYcBhGdEhp8TDhMIzokbLiYcJhGNElJcXDhMMwok/KiYcJh2HEhpQSDxMOw4gdKSMeJhyGEVtSQjxMOAwj9iS9eJhwGEZ8SGrxMOEwjPiRtOJhwmEY8SUpxcOEwzDiT9KJhwmHYSQGSSUeJhyGkTgkjXiYcBhGYhEX8RCRK0WkWET2i8jDwY5va1cTDsNIMGIuHiKSCTwBLAKmAreKyNRA55RW1ZtwGEaCEY+ex1xgv6qWqmoL8DJwbaATWtvVhMMwEow+cbjnaOCI1/ZRYJ7vQSJyL3Cvs3lm3sTcXTGwrTfIA6ribUQYJJO9yWQrJJe9ReGeEA/xCAlVXQYsAxCRrao6J84mhUQy2QrJZW8y2QrJZa+IbA33nHgMW8qAsV7bY5w2wzCSiHiIx0fAZBGZICJZwC3AW3GwwzCMHhDzYYuqtonI3wJrgEzgWVXdHeS0ZdG3rNdIJlshuexNJlshuewN21ZR1WgYYhhGipM0EaaGYSQWJh6GYUREQotHuGHssUZEnhWRChHZ5dU2TETeFZHPndeh8bTRjYiMFZF1IrJHRHaLyANOe6La209EtojIdsfef3XaJ4jIh84z8YrjdE8IRCRTRD4RkVXOdiLbelBEdorIp+5p2nCfhYQVj0jC2OPACuBKn7aHgbWqOhlY62wnAm3Ad1V1KjAfuN/590xUe88Al6nqDGAmcKWIzAd+DvxSVQuBGuCuONroywPAXq/tRLYV4FJVnekVixLes6CqCfkHXAis8dpeAiyJt11+7BwP7PLaLgYKnPcFQHG8bezG7jeBLyeDvcAAYBuuSOQqoI+/ZyTONo5xvnCXAasASVRbHXsOAnk+bWE9Cwnb88B/GPvoONkSDiNU9bjzvhwYEU9j/CEi44FZwIcksL3OMOBToAJ4FygBalW1zTkkkZ6Jx4DvAR3Odi6JayuAAv8jIh87qSAQ5rOQsOHpqYCqqogk1Fy4iGQDvwX+QVXrRMSzL9HsVdV2YKaI5ABvAFPibJJfRORqoEJVPxaRhfG2J0QuVtUyERkOvCsi+7x3hvIsJHLPI1nD2E+ISAGA81oRZ3s8iEhfXMLxgqqudJoT1l43qloLrMPV9c8REfePXqI8EwuAa0TkIK4s8cuAX5GYtgKgqmXOawUuYZ5LmM9CIotHsoaxvwXc4by/A5dvIe6Iq4uxHNirqr/w2pWo9uY7PQ5EpD8u/8xeXCKy2DksIexV1SWqOkZVx+N6Tt9T1dtIQFsBRGSgiAxyvwe+Auwi3Gch3o6bIE6dq4DPcI11/yne9vix7yXgONCKa0x7F66x7lrgc+CPwLB42+nYejGuce4O4FPn76oEtnc68Ilj7y7gR077RGALsB94DTgr3rb62L0QWJXItjp2bXf+dru/W+E+CxaebhhGRCTysMUwjATGxMMwjIgw8TAMIyJMPAzDiAgTD8MwIsLEwzCMiDDxSHBEJNdJm/5URMpFpMxru8cp3iLyLyLyU5+2mSKyN8A5PxaRB3t67wDXd6eLz3G214vIYfGKpReR34lIvfN+vIg0Of8me0TkKRHJcPZNFpFVIlLi5HGsE5FLnH03O+nyq6L1WVIZE48ER1Wr1ZU2PRN4CleK90znr8Ur/DlSXgJu9mm7xWmPJ5eqqvdyALW4wsBxIk8LfI4vcf6NpuMq4XCdiPQDVgPLVHWSqp4P/B2uIClU9RXg7uh+jNTFxCMJEZEVzq/rh8Ajvj0BEdnlZM4iIt9wiup8KiL/5dRJ8aCqnwE1IuK98NZNwEsico+IfOQU5PmtiAzwY8t6rx5CnpPf4c6IXeqcv0NEvuW0F4jI+449u0TkSyF+7JdxiRrA9cBKfwepK4t1E1AI3AZ8oKpvee3fpaorQrynEQATj+RlDHCRqn6nuwNE5BxcvYoFzq9yO64vlC8v4XwxnYI7J1X1c2Clql6groI8ewmvmM1dwClVvQC4ALhHRCYA/wtXXYuZwAxcYfKhsBa4xBG/W4BX/B3kCNzlwE7gXFx1QIwoYCn5yctr6kpZD8TlwPnAR467oD/+MyVfATaJyHfpPGQ5T0R+AuQA2biWywiVrwDTRcSdGDYEmIwr4fFZJ8P3d6oaqni0Axsd+/qr6kHvcgLAJKf2hwJvqurbIvJl7wNE5A3Hhs9U9fowPovhBxOP5KXB630bnXuR/ZxXAZ5T1SWBLqSqR0TkAPCXwA24Ut/BVWbxOlXdLiJ/gyvpyxfve/fzahfg71S1i+A4DsuvAitE5Beq+nwg+7x4GVf6+I/97HP7PLzZDVzi3lDVrztDrEdDvJ8RABu2pAYHgdkAIjIbmOC0rwUWOwVf3AVuz+7mGi8BvwRKVfWo0zYIOO70EvwNd9z3Pt95v9irfQ3wbedcROQvnFTws4ETqvo08Izb7hD5E/BTQnfmvggsEJFrvNq6+G2MyLCeR2rwW+B2EdmNq7TgZwCqukdE/hlXubkMXKUD7gcO+bnGa8B/4pqNcPND53qVzusgP+c9CrwqrlJ2q73an8FV33WbM8VaCVyHq/fykIi0AvXA7aF+SHWlgIfca1DVJnFV+fqFiDwGnABOAz8J9RpG91hKvpFwODM2c1S1Kgb3Wgg8qKpXR/teqYYNW4xEpBJY654CjhYicjPwa1zLIhhhYj0PwzAiwnoehmFEhImHYRgRYeJhGEZEmHgYhhER/x89paLFxrVANQAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "a = plt.axes(aspect=\"equal\")\n", "sns.scatterplot(test_df[\"MPG\"].values, mpg_hat_df[\"MPG_predictions\"].values, s=50)\n", "plt.xlabel(\"True Values [MPG]\")\n", "plt.ylabel(\"Predictions [MPG]\")\n", "lims = [0, 50]\n", "plt.xlim(lims)\n", "plt.ylim(lims)\n", "_ = plt.plot(lims, lims)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Compare K-fold Cross Validation metrics against hold-out test metrics" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Hold-out Test Metrics" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'loss': 8.303831,\n", " 'error': -0.45136052,\n", " 'mean_squared_error': 8.303831,\n", " 'mean_absolute_error': 2.2274728,\n", " 'r2': 0.8558148}" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_stats[\"MPG\"]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### K-fold Cross Validation Metrics" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'loss_mean': 8.111681,\n", " 'loss_std': 2.4598064,\n", " 'error_mean': 0.0380627,\n", " 'error_std': 0.5965346,\n", " 'mean_squared_error_mean': 8.111682,\n", " 'mean_squared_error_std': 2.4598064,\n", " 'mean_absolute_error_mean': 2.0598435,\n", " 'mean_absolute_error_std': 0.2779836,\n", " 'r2_mean': 0.8666786,\n", " 'r2_std': 0.03552912}" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "kfold_cv_stats[\"overall\"][\"MPG\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.9" } }, "nbformat": 4, "nbformat_minor": 4 }