rasbt--deeplearning-models
rasbt
1009 行
69 KiB
Plaintext
1009 行
69 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "b1f28f63",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"torch : 1.10.1\n",
|
|
"pytorch_lightning: 1.6.0.dev0\n",
|
|
"torchmetrics : 0.6.2\n",
|
|
"matplotlib : 3.3.4\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"%load_ext watermark\n",
|
|
"%watermark -p torch,pytorch_lightning,torchmetrics,matplotlib"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "cf389dd3",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%load_ext pycodestyle_magic\n",
|
|
"%flake8_on --ignore W291,W293,E703"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "4db1eed3",
|
|
"metadata": {},
|
|
"source": [
|
|
"<a href=\"https://pytorch.org\"><img src=\"https://raw.githubusercontent.com/pytorch/pytorch/master/docs/source/_static/img/pytorch-logo-dark.svg\" width=\"90\"/></a> <a href=\"https://www.pytorchlightning.ai\"><img src=\"https://raw.githubusercontent.com/PyTorchLightning/pytorch-lightning/master/docs/source/_static/images/logo.svg\" width=\"150\"/></a>\n",
|
|
"\n",
|
|
"# Model Zoo -- Multilayer Perceptron"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "145dd47a",
|
|
"metadata": {},
|
|
"source": [
|
|
"## General settings and hyperparameters"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "c5bd0f3f",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Here, we specify some general hyperparameter values and general settings\n",
|
|
"- Note that for small datatsets, it is not necessary and better not to use multiple workers as it can sometimes cause issues with too many open files in PyTorch. So, if you have problems with the data loader later, try setting `NUM_WORKERS = 0` instead."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"id": "21d90d58",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"BATCH_SIZE = 256\n",
|
|
"NUM_EPOCHS = 10\n",
|
|
"LEARNING_RATE = 0.005\n",
|
|
"NUM_WORKERS = 4"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "693df2f2",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Implementing a Neural Network using PyTorch Lightning's `LightningModule`"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "592e3bb8",
|
|
"metadata": {},
|
|
"source": [
|
|
"- In this section, we set up the main model architecture using the `LightningModule` from PyTorch Lightning.\n",
|
|
"- We start with defining our neural network model in pure PyTorch, and then we use it in the `LightningModule` to get all the extra benefits that PyTorch Lightning provides."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "e6f66399",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import torch\n",
|
|
"import torch.nn.functional as F\n",
|
|
"\n",
|
|
"\n",
|
|
"# Regular PyTorch Module\n",
|
|
"class PyTorchMLP(torch.nn.Module):\n",
|
|
" def __init__(self, input_size, hidden_units, num_classes):\n",
|
|
" super().__init__()\n",
|
|
"\n",
|
|
" # Initialize MLP layers\n",
|
|
" all_layers = []\n",
|
|
" for hidden_unit in hidden_units:\n",
|
|
" layer = torch.nn.Linear(input_size, hidden_unit, bias=False)\n",
|
|
" all_layers.append(layer)\n",
|
|
" all_layers.append(torch.nn.ReLU())\n",
|
|
" input_size = hidden_unit\n",
|
|
"\n",
|
|
" output_layer = torch.nn.Linear(\n",
|
|
" in_features=hidden_units[-1],\n",
|
|
" out_features=num_classes)\n",
|
|
"\n",
|
|
" all_layers.append(output_layer)\n",
|
|
" self.layers = torch.nn.Sequential(*all_layers)\n",
|
|
"\n",
|
|
" def forward(self, x):\n",
|
|
" x = torch.flatten(x, start_dim=1) # to make it work for image inputs\n",
|
|
" x = self.layers(x)\n",
|
|
" return x # x are the model's logits"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"id": "b4c872ad",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pytorch_lightning as pl\n",
|
|
"import torchmetrics\n",
|
|
"\n",
|
|
"\n",
|
|
"# LightningModule that receives a PyTorch model as input\n",
|
|
"class LightningMLP(pl.LightningModule):\n",
|
|
" def __init__(self, model, learning_rate):\n",
|
|
" super().__init__()\n",
|
|
"\n",
|
|
" self.learning_rate = learning_rate\n",
|
|
" # The inherited PyTorch module\n",
|
|
" self.model = model\n",
|
|
"\n",
|
|
" # Save settings and hyperparameters to the log directory\n",
|
|
" # but skip the model parameters\n",
|
|
" self.save_hyperparameters(ignore=['model'])\n",
|
|
"\n",
|
|
" # Set up attributes for computing the accuracy\n",
|
|
" self.train_acc = torchmetrics.Accuracy()\n",
|
|
" self.valid_acc = torchmetrics.Accuracy()\n",
|
|
" self.test_acc = torchmetrics.Accuracy()\n",
|
|
" \n",
|
|
" # Defining the forward method is only necessary \n",
|
|
" # if you want to use a Trainer's .predict() method (optional)\n",
|
|
" def forward(self, x):\n",
|
|
" return self.model(x)\n",
|
|
" \n",
|
|
" # A common forward step to compute the loss and labels\n",
|
|
" # this is used for training, validation, and testing below\n",
|
|
" def _shared_step(self, batch):\n",
|
|
" features, true_labels = batch\n",
|
|
" logits = self(features)\n",
|
|
" loss = torch.nn.functional.cross_entropy(logits, true_labels)\n",
|
|
" predicted_labels = torch.argmax(logits, dim=1)\n",
|
|
"\n",
|
|
" return loss, true_labels, predicted_labels\n",
|
|
"\n",
|
|
" def training_step(self, batch, batch_idx):\n",
|
|
" loss, true_labels, predicted_labels = self._shared_step(batch)\n",
|
|
" self.log(\"train_loss\", loss)\n",
|
|
" self.train_acc(predicted_labels, true_labels)\n",
|
|
" self.log(\"train_acc\", self.train_acc, on_epoch=True, on_step=False)\n",
|
|
" return loss # this is passed to the optimzer for training\n",
|
|
"\n",
|
|
" def validation_step(self, batch, batch_idx):\n",
|
|
" loss, true_labels, predicted_labels = self._shared_step(batch)\n",
|
|
" self.log(\"valid_loss\", loss)\n",
|
|
" self.valid_acc(predicted_labels, true_labels)\n",
|
|
" self.log(\"valid_acc\", self.valid_acc,\n",
|
|
" on_epoch=True, on_step=False, prog_bar=True)\n",
|
|
"\n",
|
|
" def test_step(self, batch, batch_idx):\n",
|
|
" loss, true_labels, predicted_labels = self._shared_step(batch)\n",
|
|
" self.test_acc(predicted_labels, true_labels)\n",
|
|
" self.log(\"test_acc\", self.test_acc, on_epoch=True, on_step=False)\n",
|
|
"\n",
|
|
" def configure_optimizers(self):\n",
|
|
" optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)\n",
|
|
" return optimizer"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "1e2e4afa",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Setting up the dataset"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "f87004a2",
|
|
"metadata": {},
|
|
"source": [
|
|
"- In this section, we are going to set up our dataset."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "44e07f8b",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Inspecting the dataset"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"id": "25b41881",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import torch\n",
|
|
"\n",
|
|
"from torchvision import datasets\n",
|
|
"from torchvision import transforms\n",
|
|
"from torch.utils.data import DataLoader\n",
|
|
"\n",
|
|
"\n",
|
|
"train_dataset = datasets.MNIST(root='./data', \n",
|
|
" train=True, \n",
|
|
" transform=transforms.ToTensor(),\n",
|
|
" download=True)\n",
|
|
"\n",
|
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
|
" batch_size=BATCH_SIZE, \n",
|
|
" num_workers=NUM_WORKERS,\n",
|
|
" drop_last=True,\n",
|
|
" shuffle=True)\n",
|
|
"\n",
|
|
"test_dataset = datasets.MNIST(root='./data', \n",
|
|
" train=False,\n",
|
|
" transform=transforms.ToTensor())\n",
|
|
"\n",
|
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
|
" batch_size=BATCH_SIZE,\n",
|
|
" num_workers=NUM_WORKERS,\n",
|
|
" drop_last=False,\n",
|
|
" shuffle=False)\n",
|
|
"\n",
|
|
"# Checking the dataset\n",
|
|
"all_train_labels = []\n",
|
|
"all_test_labels = []\n",
|
|
"\n",
|
|
"for images, labels in train_loader: \n",
|
|
" all_train_labels.append(labels)\n",
|
|
"all_train_labels = torch.cat(all_train_labels)\n",
|
|
" \n",
|
|
"for images, labels in test_loader: \n",
|
|
" all_test_labels.append(labels)\n",
|
|
"all_test_labels = torch.cat(all_test_labels)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"id": "d32adc45",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Training labels: tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])\n",
|
|
"Training label distribution: tensor([5913, 6735, 5950, 6123, 5831, 5411, 5909, 6255, 5841, 5936])\n",
|
|
"\n",
|
|
"Test labels: tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])\n",
|
|
"Test label distribution: tensor([ 980, 1135, 1032, 1010, 982, 892, 958, 1028, 974, 1009])\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print('Training labels:', torch.unique(all_train_labels))\n",
|
|
"print('Training label distribution:', torch.bincount(all_train_labels))\n",
|
|
"\n",
|
|
"print('\\nTest labels:', torch.unique(all_test_labels))\n",
|
|
"print('Test label distribution:', torch.bincount(all_test_labels))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b439bc07",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Performance baseline"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "d35dfcad",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Especially for imbalanced datasets, it's quite useful to compute a performance baseline.\n",
|
|
"- In classification contexts, a useful baseline is to compute the accuracy for a scenario where the model always predicts the majority class -- you want your model to be better than that!"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"id": "c44b0d3b",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Baseline ACC: 11.35%\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"majority_prediction = torch.argmax(torch.bincount(all_test_labels))\n",
|
|
"baseline_acc = torch.mean((all_test_labels == majority_prediction).float())\n",
|
|
"print(f'Baseline ACC: {baseline_acc*100:.2f}%')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "30a6d015",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Setting up a `DataModule`"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "e634c797",
|
|
"metadata": {},
|
|
"source": [
|
|
"- There are three main ways we can prepare the dataset for Lightning. We can\n",
|
|
" 1. make the dataset part of the model;\n",
|
|
" 2. set up the data loaders as usual and feed them to the fit method of a Lightning Trainer -- the Trainer is introduced in the next subsection;\n",
|
|
" 3. create a LightningDataModule.\n",
|
|
"- Here, we are going to use approach 3, which is the most organized approach. The `LightningDataModule` consists of several self-explanatory methods as we can see below:\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"id": "80ea49d1",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import os\n",
|
|
"\n",
|
|
"from torch.utils.data.dataset import random_split\n",
|
|
"from torch.utils.data import DataLoader\n",
|
|
"\n",
|
|
"\n",
|
|
"class DataModule(pl.LightningDataModule):\n",
|
|
" def __init__(self, data_path='./'):\n",
|
|
" super().__init__()\n",
|
|
" self.data_path = data_path\n",
|
|
" \n",
|
|
" def prepare_data(self):\n",
|
|
" datasets.MNIST(root=self.data_path,\n",
|
|
" download=True)\n",
|
|
" return\n",
|
|
"\n",
|
|
" def setup(self, stage=None):\n",
|
|
" # Note transforms.ToTensor() scales input images\n",
|
|
" # to 0-1 range\n",
|
|
" train = datasets.MNIST(root=self.data_path, \n",
|
|
" train=True, \n",
|
|
" transform=transforms.ToTensor(),\n",
|
|
" download=False)\n",
|
|
"\n",
|
|
" self.test = datasets.MNIST(root=self.data_path, \n",
|
|
" train=False, \n",
|
|
" transform=transforms.ToTensor(),\n",
|
|
" download=False)\n",
|
|
"\n",
|
|
" self.train, self.valid = random_split(train, lengths=[55000, 5000])\n",
|
|
"\n",
|
|
" def train_dataloader(self):\n",
|
|
" train_loader = DataLoader(dataset=self.train, \n",
|
|
" batch_size=BATCH_SIZE, \n",
|
|
" drop_last=True,\n",
|
|
" shuffle=True,\n",
|
|
" num_workers=NUM_WORKERS)\n",
|
|
" return train_loader\n",
|
|
"\n",
|
|
" def val_dataloader(self):\n",
|
|
" valid_loader = DataLoader(dataset=self.valid, \n",
|
|
" batch_size=BATCH_SIZE, \n",
|
|
" drop_last=False,\n",
|
|
" shuffle=False,\n",
|
|
" num_workers=NUM_WORKERS)\n",
|
|
" return valid_loader\n",
|
|
"\n",
|
|
" def test_dataloader(self):\n",
|
|
" test_loader = DataLoader(dataset=self.test, \n",
|
|
" batch_size=BATCH_SIZE, \n",
|
|
" drop_last=False,\n",
|
|
" shuffle=False,\n",
|
|
" num_workers=NUM_WORKERS)\n",
|
|
" return test_loader"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "ce2bc142",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Note that the `prepare_data` method is usually used for steps that only need to be executed once, for example, downloading the dataset; the `setup` method defines the the dataset loading -- if you run your code in a distributed setting, this will be called on each node / GPU. \n",
|
|
"- Next, lets initialize the `DataModule`; we use a random seed for reproducibility (so that the data set is shuffled the same way when we re-execute this code):"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"id": "bbc153b6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"torch.manual_seed(1) \n",
|
|
"data_module = DataModule(data_path='./data')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "8de0b488",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Training the model using the PyTorch Lightning Trainer class"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "c64332b4",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Next, we initialize our model.\n",
|
|
"- Also, we define a call back so that we can obtain the model with the best validation set performance after training.\n",
|
|
"- PyTorch Lightning offers [many advanced logging services](https://pytorch-lightning.readthedocs.io/en/latest/extensions/logging.html) like Weights & Biases. Here, we will keep things simple and use the `CSVLogger`:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"id": "4e6b0957",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pytorch_lightning.callbacks import ModelCheckpoint\n",
|
|
"from pytorch_lightning.loggers import CSVLogger\n",
|
|
"\n",
|
|
"\n",
|
|
"pytorch_model = PyTorchMLP(\n",
|
|
" input_size=28*28,\n",
|
|
" hidden_units=(128, 256),\n",
|
|
" num_classes=10\n",
|
|
")\n",
|
|
"\n",
|
|
"\n",
|
|
"lightning_model = LightningMLP(\n",
|
|
" pytorch_model, learning_rate=LEARNING_RATE)\n",
|
|
"\n",
|
|
"callbacks = [ModelCheckpoint(\n",
|
|
" save_top_k=1, mode='max', monitor=\"valid_acc\")] # save top 1 model \n",
|
|
"logger = CSVLogger(save_dir=\"logs/\", name=\"my-mlp\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "3e64b7a0",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Now it's time to train our model:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"id": "6895471d",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/home/jovyan/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/callback_connector.py:90: LightningDeprecationWarning: Setting `Trainer(progress_bar_refresh_rate=50)` is deprecated in v1.5 and will be removed in v1.7. Please pass `pytorch_lightning.callbacks.progress.TQDMProgressBar` with `refresh_rate` directly to the Trainer's `callbacks` argument instead. Or, to disable the progress bar pass `enable_progress_bar = False` to the Trainer.\n",
|
|
" rank_zero_deprecation(\n",
|
|
"GPU available: True, used: True\n",
|
|
"TPU available: False, using: 0 TPU cores\n",
|
|
"IPU available: False, using: 0 IPUs\n",
|
|
"LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
|
|
"\n",
|
|
" | Name | Type | Params\n",
|
|
"-----------------------------------------\n",
|
|
"0 | model | PyTorchMLP | 135 K \n",
|
|
"1 | train_acc | Accuracy | 0 \n",
|
|
"2 | valid_acc | Accuracy | 0 \n",
|
|
"3 | test_acc | Accuracy | 0 \n",
|
|
"-----------------------------------------\n",
|
|
"135 K Trainable params\n",
|
|
"0 Non-trainable params\n",
|
|
"135 K Total params\n",
|
|
"0.543 Total estimated model params size (MB)\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validation sanity check: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "3da60c1a4742427c90bce139f57661c6",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Training: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Training took 2.76 min in total.\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import time\n",
|
|
"\n",
|
|
"\n",
|
|
"trainer = pl.Trainer(\n",
|
|
" max_epochs=NUM_EPOCHS,\n",
|
|
" callbacks=callbacks,\n",
|
|
" progress_bar_refresh_rate=50, # recommended for notebooks\n",
|
|
" accelerator=\"auto\", # Uses GPUs or TPUs if available\n",
|
|
" devices=\"auto\", # Uses all available GPUs/TPUs if applicable\n",
|
|
" logger=logger,\n",
|
|
" deterministic=True,\n",
|
|
" log_every_n_steps=100)\n",
|
|
"\n",
|
|
"start_time = time.time()\n",
|
|
"trainer.fit(model=lightning_model, datamodule=data_module)\n",
|
|
"\n",
|
|
"runtime = (time.time() - start_time)/60\n",
|
|
"print(f\"Training took {runtime:.2f} min in total.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "efdefb29",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Evaluating the model"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "0f01c048",
|
|
"metadata": {},
|
|
"source": [
|
|
"- After training, let's plot our training ACC and validation ACC using pandas, which, in turn, uses matplotlib for plotting (you may want to consider a [more advanced logger](https://pytorch-lightning.readthedocs.io/en/latest/extensions/logging.html) that does that for you):"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"id": "a6c9f738",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"<AxesSubplot:xlabel='Epoch', ylabel='ACC'>"
|
|
]
|
|
},
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
},
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEICAYAAABF82P+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA5lUlEQVR4nO3deXhU1f3H8feZyWRfSSY7kABJWMMWNlkSRGURxBW1ikWlikvdqlVbbau1rf1pbWtFFBesW1FRUGRT0QDKjkJYEyAGCAkkYQkJEMhyfn/cAQIMkISZ3GTyfT3PPJm5M3fmmyvOJ+fcc89RWmuEEEKIM1nMLkAIIUTTJAEhhBDCKQkIIYQQTklACCGEcEoCQgghhFMSEEIIIZxya0AopUYopbKVUtuUUk84ef4WpVSW47ZUKdW91nN5Sqn1Sqm1SqnV7qxTCCHE2ZS7roNQSlmBHOByIB9YBdystd5U6zWXAJu11geUUiOBP2mt+zmeywPStNYldf3MiIgInZCQ0KB6Dx8+TEBAQIP29TRyLE4nx+N0cjxO8YRjsWbNmhKttd3Zc15u/Ny+wDatdS6AUmo6MBY4GRBa66W1Xr8ciL+YD0xISGD16oY1NjIzM8nIyLiYj/cYcixOJ8fjdHI8TvGEY6GU2nGu59zZxRQH7Kr1ON+x7VzuBObVeqyBr5RSa5RSd7mhPiGEEOfhzhaEcrLNaX+WUmooRkAMqrV5oNa6QCkVCXytlNqitV7sZN+7gLsAoqKiyMzMbFCx5eXlDd7X08ixOJ0cj9PJ8TjF04+FOwMiH2hd63E8UHDmi5RSqcCbwEit9b4T27XWBY6fRUqpmRhdVmcFhNZ6KjAVIC0tTTe0uecJTUVXkWNxOjkep5PjcYqnHwt3BsQqIEkplQjsBm4CflH7BUqpNsBnwHitdU6t7QGARWtd5rh/BfCsG2sVQjRBlZWV5OfnU1FRYXYpToWEhLB582azy6gTX19f4uPjsdlsdd7HbQGhta5SSt0PLACswNta641KqUmO518D/gCEA68qpQCqtNZpQBQw07HNC/hQaz3fXbUKIZqm/Px8goKCSEhIwPF90KSUlZURFBRkdhkXpLVm37595Ofnk5iYWOf93NmCQGs9F5h7xrbXat2fCEx0sl8u0P3M7UKIlqWioqLJhkNzopQiPDyc4uLieu0nV1ILIZo0CQfXaMhxbPEBUVFZzeuLtrOxpNrsUoQQoklp8QHhbbXwxpJcluyuNLsUIYRoUlp8QFgsiiFJdjaUVFNdI8uvCiFOOXjwIK+++mq99xs1ahQHDx6s934TJkxgxowZ9d7PXVp8QACkp9gpr4T1u0vNLkUI0YScKyCqq8/fJT137lxCQ0PdVFXjcesopuZicJIdBWRmF9GjdajZ5QghnHhm9kY2FRxy6Xt2jg3mj2O6nPP5J554gu3bt9OjRw9sNhuBgYHExMSwdu1aNm3axM0330xhYSEVFRU8+OCD3HWXMSvQiXnhysvLGTlyJIMGDWLp0qXExcXx+eef4+fnd8HaFi5cyKOPPkpVVRV9+vRhypQp+Pj48MQTT/DFF1/g5eXFFVdcwYsvvsgnn3zCM888g9VqJSQkhMWLz7qmuEGkBQG0CvAmMcTCopz6DQETQni2559/nvbt27N27VpeeOEFVq5cyV/+8hc2bTLmHJ08eTJr1qxh9erVvPzyy+zbt++s99i6dSv33XcfGzduJDQ0lE8//fSCn1tRUcGECRP46KOPWL9+PVVVVUyZMoX9+/czc+ZMNm7cSFZWFk899RQAzz77LAsWLGDdunV88cUXLvv9pQXh0C3Cyhe5Bzlw+DhhAd5mlyOEOMP5/tJvLH379j3tQrPXXnuNuXONS7127drF1q1bCQ8PP22fxMREevToAUDv3r3Jy8u74OdkZ2eTmJhIcnIyAL/85S+ZPHky999/P76+vkycOJErr7yS0aNHAzBw4EAmTJjAuHHjuPbaa13wmxqkBeGQareiNSzZVuflJ4QQLUzttR8yMzPJzMxk2bJlrFu3jp49ezqdEsTHx+fkfavVSlVV1QU/51zr9Hh5ebFy5Uquu+46Zs2axYgRIwAjqJ577jl27dpFjx49nLZkGkICwiExxEKYv43M7CKzSxFCNBFBQUGUlZU5fa60tJTQ0FD8/f3ZsmULy5cvd9nnduzYkby8PLZt2wbAe++9R3p6OuXl5ZSWljJq1Cj+9a9/sXbtWgC2b99Ov379ePbZZ4mIiGDXrl3nefe6ky4mB4tSDE6yszinhJoajcUiV28K0dKFh4czcOBAunbtip+fH1FRUSefGzFiBK+88gqpqamkpKTQv39/l32ur68v06ZN44Ybbjh5knrSpEns37+fsWPHUlFRgdaaf/7znwA89thjbN26Fa01w4YNo3t318xUJAFRS0aKnS/WFbCp8BBd40LMLkcI0QR8+OGHTrf7+Pjw2WefOZ2s78R5hoiICDZs2HBy+6OPPnrez3rnnXdO3h82bBg//fTTac/HxMSwcuXKs/b77LPPzvu+DSVdTLUMTjKWZZVuJiGEkIA4jT3Ih25xITLcVQjhVvfddx89evQ47TZt2jSzyzqLdDGdIT3ZzpRF2yk9WkmIX90X1hBCiLqaPHmy2SXUibQgzpCRYqe6RvODDHcVQrRwEhBn6NE6lGBfLzkPIYRo8SQgzuBltTA4yc6inOJzXqwihBAtgQSEE+kpdvYeOsaWPc4vkBFCiJZAAsKJ9OQTw11lNJMQou4CAwMBKCgo4Prrr3f6moyMDFavXn3O90hISKCkpGmcA5WAcCIq2JdOMcEsypHzEEKI+ouNjW1SC/80lAxzPYeMFDtvLM6lrKKSIF8Z7iqE6eY9AXvWu/Y9o7vByOfP+fTjjz9O27ZtuffeewH405/+hFKKxYsXc+DAAY4dO8Zf//pXxo4de9p+eXl5jB49mg0bNnD06FFuv/12Nm3aRKdOnTh69Gidy3vppZd4++23AZg4cSIPPfQQhw8fZty4ceTn51NdXc3TTz/NjTfe6HSdiIslAXEO6cl2pmRu54dt+xjRNdrscoQQJrjpppt46KGHTgbExx9/zPz583n44YcJDg4mLy+Pyy67jKuuugqlnM/fNmXKFPz9/cnKyiIrK4tevXrV6bPXrFnDtGnTWLFiBVpr+vXrR3p6Orm5ucTGxjJnzhzAmDTwxDoRW7ZsQSnVoOVOnZGAOIfebcMI9PFiUU6xBIQQTcF5/tJ3l549e1JUVERBQQHFxcWEhYURExPDww8/fHLVtt27d7N3716io51/TyxevJgHHngAgNTUVFJTU+v02d9//z3XXHPNySnGr732WpYsWcKIESN49NFHefzxxxk9ejSDBw+mqqrK6ToRF0vOQZyDzWphYIdwFmUXyXBXIVqw66+/nhkzZvDRRx9x00038cEHH1BcXMyaNWv44YcfiIqKcroORG3nal2cz7m+d5KTk1mzZg3dunXjySef5Nlnnz3nOhEXSwLiPDJSIikorWBbUbnZpQghTHLTTTcxffp0ZsyYwfXXX09paSmRkZHYbDYWL17Mjh07zrv/kCFD+OCDDwDYsGEDWVlZdfrcIUOGMGvWLI4cOcLhw4eZOXMmgwcPpqCgAH9/f2699VYeffRRfvzxx3OuE3GxpIvpPGoPd02KOntKXyGE5+vSpQtlZWXExcURExPDLbfcwpgxY0hLS6NLly507NjxvPvfc8893H777aSmptKjRw/69u1bp8/t1asXEyZMOPn6iRMn0rNnTxYsWMBjjz2GxWLBZrMxZcoUysrKnK4TcbEkIM4jNtSP5KhAMnOK+NWQdmaXI4Qwyfr1p0ZPRUREsGzZMgDKyspOWw+ivNzobUhISDi5DoSfnx/Tp0+v82fVXrP6kUce4ZFHHjnt+eHDhzN8+PCz9nO2TsTFki6mC8hIiWTVzwc4fOzC68gKIYQnkYC4gPRkO8era1i23TWLgAshBEC/fv3OWhOidkulKZAupgtISwjD39vKopxiLuscdeEdhBAupbVu0Cigpm7FihWN+nkNGY0pLYgL8PGyckn7CDJzZLirEI3N19eXffv2yf97F0lrzb59+/D19a3XftKCqIP0FDvfbN5Lbslh2tsDzS5HiBYjPj6e/Px8ioub5sSZFRUV9f7SNYuvry/x8fH12kcCog4yHMNdF2UXS0AI0YhsNhuJiYlml3FOmZmZ9OzZ0+wy3MatXUxKqRFKqWyl1Dal1BNOnr9FKZXluC1VSnWv676NqXUrf9rZA8jMaZp/xQghhDu4LSCUUlZgMjAS6AzcrJTqfMbLfgbStdapwJ+BqfXYt1FlJEeyIncfFZXVZpYhhBCNxp0tiL7ANq11rtb6ODAdOG1OXK31Uq31AcfD5UB8XfdtbBkpdo5V1bAsV4a7CiFaBncGRBywq9bjfMe2c7kTmNfAfd2ub2IrfG0WFskqc0KIFsKdJ6mdDVx2OlZNKTUUIyAGNWDfu4C7AKKiosjMzKx3oWBcIn+hfZNDFfPW7iAj2LNDoi7HoiWR43E6OR6nePqxcGdA5AOtaz2OBwrOfJFSKhV4Exiptd5Xn30BtNZTcZy7SEtL0xkZGQ0qNjMzkwvtm2f7mT/N3kRitz60DQ9o0Oc0B3U5Fi2JHI/TyfE4xdOPhTu7mFYBSUqpRKWUN3AT8EXtFyil2gCfAeO11jn12dcMGSmRACyS0UxCiBbAbQGhta4C7gcWAJuBj7XWG5VSk5RSkxwv+wMQDryqlFqrlFp9vn3dVWtdJUQE0Dbcn0w5DyGEaAHceqGc1nouMPeMba/Vuj8RmFjXfZuCjGQ7H63eRUVlNb42q9nlCCGE28hcTPWUkRJJRWUNq/L2m12KEEK4lQREPfVvF463l0W6mYQQHk8Cop78vK30S2wlJ6qFEB5PAqIB0pPtbCsqJ//AEbNLEUIIt5GAaIATw12lm0kI4ckkIBqgvT2A+DA/6WYSQng0CYgGUEqRnmxn6bYSjlfVmF2OEEK4hQREA2WkRHL4eDWrd8hwVyGEZ5KAaKAB7cOxWZXM7iqE8FgSEA0U6ONFn4RWcqJaCOGxJCAuQkaKney9ZRSWHjW7FCGEcDkJiIuQnuyY3VVaEUIIDyQBcRGSowKJCfGV4a5CCI8kAXERTgx3/X5rCZXVMtxVCOFZJCAuUkaKnbJjVfy444DZpQghhEtJQFykSzpE4GVR0s0khPA4EhAXKdjXRq+2YTLcVQjhcSQgXCAjxc6mwkMUHaowuxQhhHAZCQgXSE+2A0g3kxDCo0hAuEDnmGAig3zIlIAQQngQCQgXqD3ctUqGuwohPIQEhIukp9gpPVrJuvyDZpcihBAuIQHhIoM72LEomXZDCOE5JCBcJMTfRs82YXIeQgjhMSQgXCgj2U5Wfikl5cfMLkUIIS6aBIQLpacYw12XbJVWhBCi+ZOAcKGusSFEBHrLVdVCCI8gAeFCFotiSJKdxTnFVNdos8sRQoiLIgHhYukpdg4cqWT97lKzSxFCiIsiAeFig5PsKBnuKoTwABIQLtYqwJvU+FAyc4rMLkUIIS6KBIQbZCTbWbvrIAcOHze7FCGEaDAJCDfISLGjNSzZVmJ2KUII0WASEG6QGh9KmL+NzGzpZhLCk6maKrNLcCsJCDewWhSDk+wszimhRoa7CuGZlrzEoO9vgY0zza7EbdwaEEqpEUqpbKXUNqXUE06e76iUWqaUOqaUevSM5/KUUuuVUmuVUqvdWac7pCfbKSk/xqbCQ2aXIoRwte3fwcJn0UrBJxPgh5dBe94fg24LCKWUFZgMjAQ6AzcrpTqf8bL9wAPAi+d4m6Fa6x5a6zR31ekuQxyrzEk3kxAepnQ3fHon2FNY3n8qdL4avn4a5vwGqj2ry8mdLYi+wDatda7W+jgwHRhb+wVa6yKt9Sqg0o11mMIe5EO3uBBZhlQIT1J13GgxVFbAuPeosgXD9dNg4IOw+i2YfjMcKze7SpfxcuN7xwG7aj3OB/rVY38NfKWU0sDrWuupzl6klLoLuAsgKiqKzMzMBhVbXl7e4H3PJcHnOHN+rmTO198RYFMufW93csexaM7keJyuJR+P9tvepHX+SjZ2fpTijQXGsVi8GGyXEpt0jKStUyn/z2DWd3uK4z7hZpd70dwZEM6+EevTSTdQa12glIoEvlZKbdFaLz7rDY3gmAqQlpamMzIyGlRsZmYmDd33XAIT9jP7tWWo6I5kdItx6Xu7kzuORXMmx+N0LfZ4bJwJmbOh7910GfU0cOaxyICcYQR9MoFLNv4BbvkEos7sVW9e3NnFlA+0rvU4Hiio685a6wLHzyJgJkaXVbPSo3Uowb5ech5CiOauZCt8fj/E94Ernjv365KvgDvmQU0VvD3cOJndjLkzIFYBSUqpRKWUN3AT8EVddlRKBSilgk7cB64ANritUjfxsloYnGRnUU4x2gNHOAjRIhw/DB+NBy8fuOEd8PI+/+tjusOvFkJIPHxwPfz0QaOU6Q5uCwitdRVwP7AA2Ax8rLXeqJSapJSaBKCUilZK5QOPAE8ppfKVUsFAFPC9UmodsBKYo7We765a3Sk9xc7eQ8fYsqfM7FKEEPWlNXz5MBRvgWvfML706yIkHu6YDwmD4PN74bu/NsthsO48B4HWei4w94xtr9W6vwej6+lMh4Du7qytsaSfHO5aTKeYYJOrEULUy5ppkPURZDwJHYbVb1/fELhlBsx+CBb9HQ7sgKv+c+EWSBMiV1K7WVSwL51iglkks7sK0bzs/hHmPQ7th8GQ3zbsPaw2GPsKDH0KsqbD+9fC0QOurdONJCAAtn6DV6X7xi6nJ9tZnXeAsgqPu9xDCM90ZD98/EsIiDS6liwX8VWpFKQ/BtdMhZ3L4a3hRmuiGZCAOLIfPr6NtNUPwLaFbvmIjBQ7VTWaH7btc8v7CyFcqKYGZk6CskIY9y4EuOh6hu43wviZUL4H3rzMaKE0cRIQ/q1gwmyqrX5G82/2Q3DMtSeUe7cNI9DHS66qFqI5+P4l2LoARvwN4nu79r0TB8OdX4PNF965ErbMvfA+JpKAAIjrzZre/4RLfg1r3oEpl8DPS1z29jarhYEdwlmUXSTDXYVoynIz4bu/QNfroM9E93yGPQUmLgR7R5j+C1jxuns+xwXqFBCO6xIsjvvJSqmrlFI295bWuGqs3sYFMLfPA2WF/442TlAdP+KS989IiaSgtIJtRZ4zT4sQHuVQAcy4E8KTYMzLxrkDdwmMhAlzIGUUzPstzP8d1FS77/MaqK4tiMWAr1IqDlgI3A68466iTNV2ANzzA/S9C1a8Bq8Ngp0rLvptaw93FUI0MdWV8MntUHkUbnwPfALd/5ne/sZn9ZsEyyfDx7e57A9SV6lrQCit9RHgWuA/WutrMKbw9kzeATDqBbjtC+MfzrQR8PUfjBkcGyg21I/kqEAyZbirEE3PN3+CXcvhqpeNLqDGYrHCyL/DiOdhyxz47xgobzp/RNY5IJRSA4BbgDmObW69yK5JaJdutCZ6jocf/g1T06Hgpwa/XXqynVU/H+DwMc+aM16IZm3T57DsFejzK+h2vTk19L8Hbnwf9m6EN4cZcz81AXUNiIeAJ4GZjuky2gHNexaquvINNv6quGUGVJTCG8Pg278Y88LXU0ZKJMera1i2XYa7CtEklGyDWfdBXG8Y/hdza+k02jgvUXnEGAab94O59VDHgNBaL9JaX6W1/rvjZHWJ1voBN9fWtCRdDvcug243wOL/gzcvhT31mz8wLSEMf2+rDHcVoik4fsTo97fa4Ib/GpPxmS2+N0z8BgLs8N7VkPWJqeXUdRTTh0qpYMfMqpuAbKXUY+4trQnyC4NrX4ebPoSyPTA1Axa/UOdlBn28rFzSPpzMHBnuKoSptDaWCC3aBNe9AaGtL7xPYwlLgDu/gvi+8NlEWPyiaRP91bWLqbPW+hBwNcbke22A8e4qqsnreCXcu8JoEn77HLx1ORRn12nX9JRIdu0/Sm7JYTcXKYQ4px//C+s+hPTHocNlZldzNv9WMP4z6DYOvv0zzH7AGDDTyOoaEDbHdQ9XA59rrSup3+pwnicg3Jgb/vppcCAPXhsMS/9zwbHMGY7hrotkuKsQ5ihYC3N/C+0vhfQGTsLXGLx84NqpMOQx+PFd+HAcVBxq1BLqGhCvA3lAALBYKdUWY0pu0fVauHe58VfIV0/BtFGwb/s5X966lT/t7AFkynkIIRrf0QPGeYeACLj2TWOYaVOmFFz6FFz1Cvy8GKaNhNLdjfbxdT1J/bLWOk5rPUobdgBD3Vxb8xEUBTd9ANe8DkWbjYvrVkw1Jv1yIiM5khW5+6iobHpXTgrhsWpqYOY9xhXTN/zXdZPwNYZe4+EXHxuzwL45DAqzGuVj63qSOkQp9ZJSarXj9g+M1oQ4QSnofpMx0qnNAJj3GLw31um0vukpdo5V1bAsV4a7CtFofvgX5MwzhrO27mN2NfXXYRjcuQCUxWhJbP3G7R9Z1y6mt4EyYJzjdgiY5q6imrWQOLj1Uxjzb2M63ymXwJr/njYKoV9iK3xtFjkPIURj+XmxcbK3y7XGNDrNVVQXY6K/VonGOYk177j14+oaEO211n/UWuc6bs8A7dxZWLOmFPSeAPcshdiexgiED24wmraAr81K/3bhcj2EEI3hUCHMuAPCOxgXvbpzEr7GEBxjTCra/lKY/SB888w5u7MvVl0D4qhSatCJB0qpgcBRt1TkScLaGvM5jXwB8r6HV/vDuumgNRnJdn4uOcyOfTLcVQi3qa40wuH4YWPxH58gsytyDZ8guHk69L7dWL/is4lQdczlH1PXgJgETFZK5Sml8oBXgLtdXo0nslig313GnE72jjDzbph+C5e2Nv6KkVaEEG608BnYudSYvjuyk9nVuJbVC0b/Ey57xhidpVy/vE9dRzGt01p3B1KBVK11T+BSl1fjycLbG83Cy/8M276hzf+G8suQn2T6byHcZfNs49qkPhMh9Qazq3EPpWDQQ8ZccVbXL9FTr8jRWh9yXFEN8IjLq/F0FisMfADuXgxhCTxz7AWuy32ailKZAlwIl9q3HWbdC7G9YPhfza7G/dx0PcfFtEma+ZkeE0V2hDu/Znu3h7lcrcQyZUCTX5tWiGaj8ih8/EvjS3NcE5mEr5m6mIBo2VNtXCyrF7Fjnub66r+yX4XB9Jth5iTIX2PKnCtCeIw5j8LeDXDtGxDaxuxqmrXzLvqjlCrDeRAowM8tFbUgft5WQhJ7MuHg88wfsgKW/APW/Q+8/Iz56dv0g9b9jYt6/MLMLleIpu/Hd2Ht+zDkt8YU/eKinDcgtNYeMias6UpPtvPcnBLyez5MfJ+JxoiLnSuM5Q+//xfoakAZIzBa94M2/Y2fYQnNfzy3EK5UmGW0HtplQMYTZlfjETx/2dAmLiMlkufmbCYzu5hb+7eFLtcYNzDGbu9eYwTGzmWw4VNY47iAPTDKERb9jZZGdKpbRjEI0SwcPQgfjwf/cLjuraY/CV8zIQFhsvb2AOJC/ViU4wiI2rwDIHGIcQNjKvGizUbrYucK2LncWE8XwOZvdEu17mfMBdW6D/iGNO4vI4QZtDZGLJXmG0PJAyLMrshjSECYTClFRoqdWT/t5nhVDd5e5xk3YLFCdFfj1meise1QgREUuxyB8f0/Qb+I0S3V+dR5jDb9ILStdEsJz/PDvyF7Dox4Hlr3NbsajyIB0QRkpETywYqdrN6xn0va1/Ovn+BYY02Krtcaj4+Vw+7Vp85jZH0Cq982nguKOf08RnSqcTWmEM1V3vew8FmjW7bfJLOr8Tjy7dAEDGgfjs2qWJRdXP+AOJNPoHGSrl2G8bim2lh3d+fyUy2NTbOM52z+EJ92qoURL91S4hzK9sCmL2DrAlL37YP9yUZ/v38r8I9w3A83unf8w41Rd+4+D1C2x5hnqVU7uOo/0jp2AwmIJiDQx4s+Ca3IzC7myVEuni/GYoXobsat76+MbaW7T53H2LUclrwIugZQENWFDl5toWscRCS5thbRvBwqhM1fwMZZxiAJNEQk41WljT82juyD4+Xn2FkZIVE7NJyGSSvH4wjjnFtdv+Srq4xwOFYG42d5ziR8TYwERBORnmznb/O2UFh6lJgQN19iEhIHIddB1+uMx8fKIH/1yfMYMXlfw+R5xtz5Qx71vEnOxLkdKjBaCptmGSGABnsnY9ho56shsiM/ZmaSkZFhvL6yAo7uh8MlRmDUvtXetj8X8lcZ92uqnH+21ccRHOGnQuREoJy5be0HsOMHuGYqRHVunGPTArk1IJRSI4B/A1bgTa3182c83xFj4aFewO+11i/WdV9PM6xTJH+bt4U/fbGRl2/uiY9XIw7T8wmC9kONG7D8q1kMVD/BqjdhwwzodJWxcHpMauPVJBqPs1CI7AwZT0KXq8Gecu59bb5gizXOhdWF1lBR6giO/XCk5IxA2e94XAIHdxr3K0qdv1faHdD9xnr+sqI+3BYQSikrMBm4HMgHVimlvtBab6r1sv3AA8DVDdjXo3SIDOIPozvz7JebmPjf1bw+vjf+3uY08Cq9QyHjGRj4ICx/FVa8bnQ1pIwygiKulyl1CRcq3X2q+2jXcmNbZBcY+jujpWBPds/nKgV+ocYtvH3d9qmuPD04juwzgqbjaPfUKE5y5zdQX2Cb1joXQCk1HRgLnPyS11oXAUVKqSvru68numNQIoG+XjzxaRbj31rJ2xP6EOJn4sVv/q3g0qdgwP2wciosmwxvDIUOl0P6b2VIYXNTutu4bmbTLKM7ERyh8Hv3hsLFstogKMq4iUblzoCIA3bVepwP9GuEfZu1cWmtCfTx4sHpP3Hz1OW8e2dfIgJNno3SL9QIhH6TjG6nZa/AW5dDYrqxPWHQBd9CmKQ03wiFjbMgf6WxLaorDH3K6D6SgQjiPNwZEM6GI9R1Btg676uUugu4CyAqKorMzMw6fsTpysvLG7yvq/kDD/Tw5j8/HWL0Swt5rI8v4X6uXy3qXM5/LHph6f0qsQULaLPzM7zfuZKDIV3Y0XYcB8K6e9xQQ6/KMnTpHpZ8c5Rqr+YxP6VPRTH24qXYi38g5FA2AOUBiRQl3kKxfSBH/eOM/5s27AZ21/v9m9L/K2bz9GPhzoDIB1rXehwPFLh6X631VGAqQFpamj45uqKeMmuPzGgCMoD+ffZzx7RVvLQO3rszjXb2wEb57LodixFQ+Vf48V1Cv/8XoVl/NK6jODGLZnMNCq2hZCvkzIPs+Ub/vHYsCO8TYpyMDY41Fo4PjjPuB8We2u4XZs7vfnDXqe6j/FXGtqhukPYUdL6GwIgOBALtXPBRTe3/FTN5+rFwZ0CsApKUUokYf6bcBPyiEfb1GH0SWvG/u/pz29srGff6Mt67sx+dYoLNLusUmx/0uxt6T4Cf3jdmn/3wBojpYXQ9pYxqHkFRXWmM88+ebwTD/lxje1Q3GPwbNhdV0ik+1Bjtc+K2dyOU7+Wshq2XryMs4owr10/cD651P8DumovIDu481X20e7WxLbobXPq0cWVxXU8CC3EObgsIrXWVUup+YAHGUNW3tdYblVKTHM+/ppSKBlYDwUCNUuohoLPW+pCzfd1Va1PWNS6Ej+8ewK1vruDG15cx7fa+9G7bxNaG8PKBPndCr9tg3XTjwrvpvzD6uoc8Cp3GgqXxusjq5Mh+2LbQCISt38CxUrB6GxMj9r8XkkdAqNGI3ZuZSadBGWe/R3WlcTVvWSEc2n16gBwqMFofhwqh5owFoJS1VnjUbomccd/L++zPPLjTCIRNs4yZfsGYMmXYH4wTzRIKwoXcOo5Saz0XmHvGttdq3d+D0X1Up31bqg6RgXwyaQDj31rB+LdWMHV8GoOSmuCMlVYb9BoP3W82rp9Y/CJ8MgHsHWHwo8Z8UWZOw1yyFbLnQc58Y7y/rjb+mu80BlJGQLuhxlQldWW1GSES2vrcr6mpMYZmngyO3Y5Acdzfuwm2fg2VR87eN8B+qgsrKBr2ZJ0KhZjuMOyP0HmshIJwG7mSuplo3cqfjycNYPybK7njnVX85xc9Gd4l2uyynLN6QfeboNsNxl+6i16AzyZC5t+MFkW3Gxpn7YrqKqPrKGe+EQz7txvbI7vAoIcgeaQxRbo7WzcWCwRGGrfYHs5fc+LisdNaIrXuH9xptEZC2xih0OVqY/4hIdxMAqIZiQzy5aO7+zNh2iru/eBHXrg+lWt7OW2ANQ0WqzGdR+drYMuXsPj/YNY9kPk8DH4Euv/CeTfKxTh6wOg6yp4H2742vnit3pAw2BimmzKi6a1TXPviMZnWRDQhEhDNTKi/N+9P7Mdd767mkY/XUX6sitsGJJhd1vlZLND5KqMrJ2c+LPo/mP2g0bIY9BD0HG9M2dBQ+7af6jrasdToOvKPMK60TR5hTCEik7kJUW8SEM1QoI8Xb0/ow/0f/sQfPt9IWUUV92a0RzX1EUNKQcpI40t7+0IjKOY+Ckv+YUzr0euX4O1/4feprjKuBD4xFHXfVmN7ZGfjfVJOdB3JspNCXAwJiGbK12Zlyq29eOyTdbywIJtDFZU8MaJj0w8JMIKiw2XQfhj8vBgWvwDznzCC4pJfQ9qdZ58sriiFbd8YgbD1K6g4CBabcRV3319B8nAISzDjtxHCY0lANGM2q4WXxvUg0NeL1xflUlZRxZ/HdsVqaQYhAUZQtEs3bjuWGi2Kr/9gXE8x4F7jOorcRUZLYcdSY5pov1anWiHtLwXfJnRdiBAeRgKimbNYFH8e25UgXxtTMrdTXlHFP8Z1x2ZtYtcdXEjbS+C2WbBrldGi+PY54wbGMNkB9xvBEN9Huo6EaCQSEB5AKcXjIzoS5OvF/83PpvxYFa/e0gtfWzP8Im3dB275GAp+gsJ1xoVrMqRTCFM0sz8zxfncm9GB567uynfZRUyYtpLyY+dYuas5iO1pTOEh4SCEaSQgPMyt/dvyrxt7sCrvALe8sZwDh4+bXZIQopmSgPBAY3vE8fqtvdm8p4wbpy5j76EKs0sSQjRDEhAe6rLOUbxzex92HzjKDa8tY9d+J3P9CCHEeUhAeLBL2kfw/sR+lB6t5PrXlrJ1b5nZJQkhmhEJCA/Xs00YH989gBoN415fxvr8UrNLEkI0ExIQLUBKdBCf3D0Af28vbn5jOSty95ldkhCiGZCAaCESIgKYcc8AooJ9uO3tlXyXXWR2SUKIJk4CogWJCfHj47sHkBQVyK/+u5ovs+q6RLgQoiWSgGhhwgN9+PBX/enZJpQH/vcTH63aaXZJQogmSgKiBQr2tfHuHf0YnGTn8U/X8+aSXLNLEkI0QRIQLZSft5U3bkvjym4xPDdnMy99nYPW2uyyhBBNiEzW14J5e1l4+eaeBPhYeXnhVg4dreQPozubXZYQoomQgGjhrBbF369LJcjXxlvf/0z5sSpGhktLQgghASEwpgt/6spOBPva+Oc3OWyPtOLdupjUuFBC/G1mlyeEMIkEhACMkHjwsiSCfL14bs4mxr+1EoC24f6kxofSPT6E1PhQusYF4+8t/2yEaAnk/3RxmjsGJRJ1NI+QhG5k7T5I1q5S1uTtZ/Y645oJi4KkyCBS40Mct1A6xgTh49UMFycSQpyXBIQ4S4BNMSgpgkFJESe3FZcdIyv/IOvyS8nKP8jCLUV8siYfAG+rhY4xQScDIzU+hKTIoOazNrYQwikJCFEn9iAfhnWKYlinKAC01uw+eJSs/FLW5RstjVk/FfD+cuPCOz+bla5xwScDo3t8KG3D/VFKQkOI5kICQjSIUor4MH/iw/wZ1S0GgJoaTW7JYbLyD5LlaGm8v3wHx6pqAAj29ToZGKnxoXRvHUJ0sK+EhhBNlASEcBmLRdEhMpAOkYFc2ysegMrqGnL2lp0MjHW7Snl9cS7VNcZQWnuQD93jQ+gWF0pqa6Ol0SrA28xfQwjhIAEh3MpmtdAlNoQusSHc3LcNABWV1WwqPETWroMnu6gWbinixIXc8WF+dI8PpX+7VtyQ1hpfm5wAF8IMEhCi0fnarPRqE0avNmEnt5VVVLJ+dynr80tPhsac9YW8tiiX31yRzNU94rDISW8hGpUEhGgSgnxtXNI+gkvanxo5tXR7CX+bu4VHPl7HG0t+5smRHRmSbDexSiFaFpmsTzRZl7SP4PP7BvLyzT0pP1bJbW+vZPxbK9iwW5ZNFaIxSECIJs1iUVzVPZZvHknn6dGdWb+7lDGvfM/DH60l/8ARs8sTwqO5NSCUUiOUUtlKqW1KqSecPK+UUi87ns9SSvWq9VyeUmq9UmqtUmq1O+sUTZ+Pl5U7ByWy6LGhTEpvz9z1hVz64iL+OnczpUcqzS5PCI/ktoBQSlmBycBIoDNws1LqzLmkRwJJjttdwJQznh+qte6htU5zV52ieQnxs/H4iI5892gGV/WI5Y0luQx54TveWJxLRWW12eUJ4VHc2YLoC2zTWudqrY8D04GxZ7xmLPCuNiwHQpVSMW6sSXiI2FA/XryhO3MfGEzPNqH8Ze5mhv1jETN/yqemxvOmK9day4JOotG5cxRTHLCr1uN8oF8dXhMHFAIa+EoppYHXtdZT3ViraKY6xQTzzu19+WFbCX+bt5mHP1rHm0t+5smRnU6bS6o5qqisZnFOMfM27OGbzXsJ8vFiZLcYRnaNplebMBn2K9zOnQHh7F/vmX8Cne81A7XWBUqpSOBrpdQWrfXisz5EqbswuqeIiooiMzOzQcWWl5c3eF9P01yPxSNdNSvCfZiRU8atb62ga7iVcSk22gRf3IV2jXk8Kqo0WSXVrN5Txbriao5VQ4ANeti9OFxZyX9/+Jm3vv+ZUB9F7ygrfaK9SA6zYGnE6Uqa678Pd/D0Y+HOgMgHWtd6HA8U1PU1WusTP4uUUjMxuqzOCghHy2IqQFpams7IyGhQsZmZmTR0X0/TnI/FpcAjVdW8t2wH//l2G39cVsE1PeP4zRUpxIX6Neg93X08DlVU8u3mIuZtKCQzu5hjVTWEB3hzbe84RnaNZkD7cGxWoze4rKKSb7cUMXe98dqFOyuICPRmeJdoRnWLoV9iK7ys7h2c2Jz/fbiapx8LdwbEKiBJKZUI7AZuAn5xxmu+AO5XSk3H6H4q1VoXKqUCAIvWusxx/wrgWTfWKjyIj5eViYPbcUPv1ry6aBvTfsjjy6xCbh+YwL0ZHQjxM3+VvINHjvPVpr3M37CH77eWcLy6hsggH27q05oRXWPom9jK6XTpQb42xvaIY2yPOA4fq+K77CLmrd/DZz/u5oMVOwnzt3FF52hGdovmkvYReHvJSHbRcG4LCK11lVLqfmABYAXe1lpvVEpNcjz/GjAXGAVsA44Atzt2jwJmOmb59AI+1FrPd1etwjOF+Nt4cmQnbhuQwD++ymbq4lw+WrWL+4d2YPyAto2+yFFJ+TG+2riXeRsKWbZ9H1U1mrhQP24b0JaR3aLp2bp+5xUCfLwYnRrL6NRYjh6vZlFOMfM2FDJnfSEfrd5FsK8Xl3eOZmTXaAYlRcicVqLe3DrVhtZ6LkYI1N72Wq37GrjPyX65QHd31iZajrhQP14a14OJg9rx/PwtPDdnM+8szeOx4SmMSY1168nevYcqmL9hD/M2FLLy5/3UaEgI92fi4HaM6hZNt7gQl0x37udtZUTXaEZ0jaaisprvt5Ywd0MhX2/aw6c/5hPo48WwTpGM7BpDRopdwkLUiczFJFqMzrHBvHtHX5ZsLeZvc7fw4PS1vLEklydHdmJgB9eNeMo/cMQRCntYs+MAAEmRgdw/tAMju8XQMTrIrWtg+NqsXNY5iss6R3G8qoal20uYt34PCzbt4fO1Bfh7WxnaMZJRjrAI8JGvAeGc/MsQLc7gJDsDfx3B5+t28+KCHG55cwXpyXaeGNmRTjHBDXrPvJLDzHO0FLLyjbmiOsUE85vLkxnZLZoOkUGu/BXqzNvLQkZKJBkpkTxX3ZUVufuZu6GQBRv2MCerEB8vCxkpdkZ1i+HSjpEE+Zp/fkY0HRIQokWyWBTX9IxnZNcYx4inrYx6eQnX9YrnkcuTia3DiKete8scobCHzYWHAOgeH8ITIzsyoks0CREB7v416sVmtZxca/zPY7uyKm8/89YXMm/DHhZs3Iu31cKQ5AhGdo3hsk5RhPhLWLR0EhCiRfO1WfnVkHbckBbPq5nbeeeHPGavK+COQYlMSm9/2ognrTWbC8uYt8H4Ut1WVA5AWtswnrqyEyO6RhMf5m/Wr1IvVouif7tw+rcL549juvDjzgPMXW+0gL7ZXITNqrikfQSjukVzeedoWeWvhZKAEAII9ffmd6M6cduAtvzjqxymZG7nfyt38utLk+BgNcvnbWHehkJ27DuCRUHfxFbcNqALw7tEExXsa3b5F8ViUaQltCItoRVPXdmJdfkHmbdhD3PXF/L4p+v53cwNDGgXzshu0VzROdrsckUjkoAQopb4MH/+eWMP7hyUyPPztvDnLzcB4GXJZUD7cCalt+fyzlFEBPqYXKl7WCyKnm3C6NkmjCdHdmRjwSHmri9k7vpCfj9zA0/P2kBKmIUCv52M6CotC08nASGEE13jQnh/Yj+Wbi8hc8Va7r16CKH+LevLUClF17gQusaF8NjwFLbsKWPu+kI+Xr6d381cz9Ofb2BQhwhGp8ZwRZfoJnEBonAtCQghzuOS9hEc3+XV4sLhTEopOsUE0ykmmF62AuzJvfgyq5DZ6wp4bEYWv5+5gSHJdsZ0N05wy9BZzyD/FYUQ9VK7ZfH4iBTW7jrIl1mFfJlVwDeb9+Jrs3Bpx0jGpMYytGOkXJTXjElACCEaTKlT5yx+P6oTq3ccYPa6AuZtKGTu+j0EeBsX7Y1JjWVwckSjT28iLo4EhBDCJSwWRd/EVvRNbMUfx3Rmxc/7mb2ugPkbjSu4g3y9GN4lmjHdY7mk1gy1oumSgBBCuJyX1cLADhEM7BDBs2O78sO2EmZnFTB/wx5mrMknzN/GyG4xjE6NoV9iuNOZa4X5JCCEEG7l7WVhaMdIhnaMpKLSmHX2y6xCZv64mw9X7MQe5MOV3WIY0z2m3jPaCveSgBBCNBpfm5XhXaIZ3iWaI8er+HZLEbPXFfDhyp28szSP2BBfrkyNYUz3WJfNdCsaTgJCCGEKf+9T61mUVVTyzea9zF5XyDtL83hjyc+0DfdndGoMo1Nj3T4DrnBOAkIIYbogXxvX9Iznmp7xlB6pZMHGPczOKuC1RblM/m47HSIDT4ZFh8hAs8ttMSQghBBNSoi/jXF9WjOuT2tKyo8xb8MevlxXwL8XbuVf32ylU0wwo1NjGNYpkpQoaVm4kwSEEKLJigj0YXz/tozv35a9hyqY47gg74UF2bywIJvoYF/Sk+1kpNgZmBRBsKxn4VISEEKIZiEq2Jc7BiVyx6BECkuPsjinmMzsYuY61uC2WhS924SRnmInPdlOl9hgaV1cJAkIIUSzExPix4192nBjnzZUVtfw086DLMopIjO7+GTrwh7kw5Ako3UxOCmixc+n1RASEEKIZs1mtZy8gvux4R0pKqtgcU4JmdlFfLN5L5/+mI9FQY/WoWSkRJKebKdbXIhcb1EHEhBCCI8SGeTL9b3jub53PNU1mrW7DrIop5hF2UX885scXvo6h1YB3gxJiiAjJZLBSRGEe+j6HhdLAkII4bGsFkXvtmH0bhvGI5cns6/8GEu2Gq2LxVtLmLW2AKUgNS6EdEfrokfrUJn6w0ECQgjRYoQH+nB1zziu7hlHTY1m/e5SFuUUk5ldxCvfbuXlhVsJ9bcxOMk40T0kOYLIoOa9pOzFkIAQQrRIFouie+tQurcO5YFhSRw8cpwlW0uM7qicYmavKwCgS2wwGSl20pMj6dUmFK8WNAutBIQQQgCh/t6M6R7LmO6x1NRoNhUecpy7KD55RXeQrxeDkyJITzYC41y01lTXaCqrNcerajhebdwqT9yvqqHy5E/N8epqjlfpWttqv06fte3Ee53YFuDtxQs3dHf5MZGAEEKIM1gsp1bNu29oBw5VVPKDo3VhXHuxB4AIP4Xfym+prPXlfiIMtHZtTVaLwmZV2KwWfLws2KwWvB0/7W46yS4BIYQQFxDsa6xfMbJbDFprcvaWk5ldxLc/bSU2ptXJL25vLwvetb64T/60qrO3OV57aps664u/dhiYceJcAkIIIepBKUVKdJBx07vIyOhhdklu03LOtgghhKgXCQghhBBOSUAIIYRwSgJCCCGEUxIQQgghnJKAEEII4ZQEhBBCCKckIIQQQjiltKuvBzeRUqoY2NHA3SOAEheW05zJsTidHI/TyfE4xROORVuttd3ZEx4VEBdDKbVaa51mdh1NgRyL08nxOJ0cj1M8/VhIF5MQQginJCCEEEI4JQFxylSzC2hC5FicTo7H6eR4nOLRx0LOQQghhHBKWhBCCCGcavEBoZQaoZTKVkptU0o9YXY9ZlJKtVZKfaeU2qyU2qiUetDsmsymlLIqpX5SSn1pdi1mU0qFKqVmKKW2OP6NDDC7JjMppR52/H+yQSn1P6WUr9k1uVqLDgillBWYDIwEOgM3K6U6m1uVqaqA32itOwH9gfta+PEAeBDYbHYRTcS/gfla645Ad1rwcVFKxQEPAGla666AFbjJ3Kpcr0UHBNAX2Ka1ztVaHwemA2NNrsk0WutCrfWPjvtlGF8AceZWZR6lVDxwJfCm2bWYTSkVDAwB3gLQWh/XWh80tSjzeQF+SikvwB8oMLkel2vpAREH7Kr1OJ8W/IVYm1IqAegJrDC5FDP9C/gtUGNyHU1BO6AYmObocntTKRVgdlFm0VrvBl4EdgKFQKnW+itzq3K9lh4QzlYBb/HDupRSgcCnwENa60Nm12MGpdRooEhrvcbsWpoIL6AXMEVr3RM4DLTYc3ZKqTCM3oZEIBYIUErdam5VrtfSAyIfaF3rcTwe2EysD6WUDSMcPtBaf2Z2PSYaCFyllMrD6Hq8VCn1vrklmSofyNdan2hRzsAIjJbqMuBnrXWx1roS+Ay4xOSaXK6lB8QqIEkplaiU8sY4yfSFyTWZRimlMPqYN2utXzK7HjNprZ/UWsdrrRMw/l18q7X2uL8Q60prvQfYpZRKcWwaBmwysSSz7QT6K6X8Hf/fDMMDT9p7mV2AmbTWVUqp+4EFGKMQ3tZabzS5LDMNBMYD65VSax3bfqe1nmteSaIJ+TXwgeOPqVzgdpPrMY3WeoVSagbwI8bov5/wwKuq5UpqIYQQTrX0LiYhhBDnIAEhhBDCKQkIIYQQTklACCGEcEoCQgghhFMSEELUg1KqWim1ttbNZVcTK6USlFIbXPV+QlysFn0dhBANcFRr3cPsIoRoDNKCEMIFlFJ5Sqm/K6VWOm4dHNvbKqUWKqWyHD/bOLZHKaVmKqXWOW4npmmwKqXecKwz8JVSys+0X0q0eBIQQtSP3xldTDfWeu6Q1rov8ArGTLA47r+rtU4FPgBedmx/GVikte6OMafRiSv4k4DJWusuwEHgOrf+NkKch1xJLUQ9KKXKtdaBTrbnAZdqrXMdEx7u0VqHK6VKgBitdaVje6HWOkIpVQzEa62P1XqPBOBrrXWS4/HjgE1r/Vwj/GpCnEVaEEK4jj7H/XO9xpljte5XI+cJhYkkIIRwnRtr/VzmuL+UU0tR3gJ877i/ELgHTq57HdxYRQpRV/LXiRD141drplsw1mg+MdTVRym1AuMPr5sd2x4A3lZKPYaxItuJGVAfBKYqpe7EaCncg7EymRBNhpyDEMIFHOcg0rTWJWbXIoSrSBeTEEIIp6QFIYQQwilpQQghhHBKAkIIIYRTEhBCCCGckoAQQgjhlASEEEIIpyQghBBCOPX/R9iaR7f4khUAAAAASUVORK5CYII=\n",
|
|
"text/plain": [
|
|
"<Figure size 432x288 with 1 Axes>"
|
|
]
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA4UElEQVR4nO3deXxV5Z348c83OyErAQIkkLBE9j2CFbVBqsWV1lqLbbVSlbF1ZWbaOnbm13bazjhTO1PasVKq1tFq1bHS2hbFNUYUlYR9F4FAAoQlO2S9+f7+ODdwCTdAQk7u9n2/XveVe855zjnfJ4H7vc95znkeUVWMMcaYjqICHYAxxpjgZAnCGGOMX5YgjDHG+GUJwhhjjF+WIIwxxvgVE+gAelL//v01Nze3W/seO3aMvn379mxAQc7qHP4irb5gde6qkpKSI6o6wN+2sEoQubm5FBcXd2vfwsJCCgoKejagIGd1Dn+RVl+wOneViJR2ts0uMRljjPHLEoQxxhi/LEEYY4zxK6z6IPxpaWmhrKyMxsbGM5ZLTU1l69atvRRVcDjXOickJJCdnU1sbGwvRGWMCRZhnyDKyspITk4mNzcXEem0XF1dHcnJyb0YWeCdS51VlaNHj1JWVsbw4cN7KTJjTDAI+0tMjY2NZGRknDE5mM6JCBkZGWdtgRljwk/YJwjAksN5st+fMZEp7C8xGWOMm441tbJ+XzUby2uIj4liUGofBqcmMDg1gYykeKKjQvcLlqsJQkTmAouBaOBxVX24w/Z04ElgJNAIfFNVN3m33Q/cCQjwW1X9hZuxGmPM2agqe44eZ01pFWv2VrFmbzXbD9bS1sm0OjFRQmZKApkp8QxO7cMgb+I4+bMPA5PjiY0Ozos5riUIEYkGHgWuAMqA1SLyiqpu8Sn2ELBOVb8oImO85eeIyASc5DADaAZeE5G/qeonbsXrlurqap577jm+/e1vd2m/q6++mueee460tDR3AjPGnNWxplbWl1Wzdm81a0qrWLuvmspjzQAkxccwZWga98wexdScdKZkp9GmyoGaRg7WNHKgtpGDNQ0nlrceqOXtbYdoaPGccg4RGJAUz+DUBDJTTiYO30SSmZJAQmx0r9ffzRbEDGCnqu4CEJHngXmAb4IYB/w7gKpuE5FcEckExgIfqupx777vAl8E/tPFeF1RXV3Nr3/969MShMfjITq68z/48uXL3Q7NGONDVSk9etzbMqhiTWk123xaByMH9OXyMQOZNiydaTlp5A1M9nv5KCMpnglZqZ2eo7ax1UkgNQ3enycTyp6jx1i16yh1ja2n7duvbxyDUpykMSg1gcEp7QmkD/vr23r0d9HOzQSRBezzWS4DZnYosx64AVgpIjOAHCAb2AT8VEQygAbgaqB7gyz5+NFfNrNlf63fbWf7wO7MuCEp/OC68Z1uf/DBB/n000+ZMmUKsbGxJCUlMXjwYNatW8eWLVv4whe+wL59+2hsbOT+++9n4cKFwMlxperr67nqqqu45JJL+OCDD8jKyuLPf/4zffr08Xu+3/72tyxdupTm5mZGjRrFM888Q2JiIhUVFdx1113s2rULgMcee4yJEyfy9NNP88gjjyAiTJo0iWeeeabLvwNjQpFv62Ct93JRZ62DqUPTSEuMO+9zigipfWJJ7RPL6EGd32Je3+QkkYra9gRysiVyoKaRdT4tGYDkWPjqtecd3unxujUntYh8Gfi8qt7hXb4FmKGq9/qUScHpo5gKbATGAHeo6noRuR24G6jHaXU0qOoiP+dZCCwEyMzMnP7888+fsj01NZVRo0YB8B+vf8q2inq/8apqt+7WGZOZxPeuHNnp9tLSUm666SY++ugj3nvvPb785S/z4Ycf0j7qbGVlJf369aOhoYGCggKWL19ORkYGEyZM4N1336W+vp4pU6bw7rvvMmnSJL7xjW9w1VVXMX/+fL/nO3r0KBkZGQD867/+KwMHDuSuu+7itttu48ILL+Tuu+/G4/FQX1/Pvn37uPXWW3njjTfIyMg4EYs/O3fupKampsu/n2BTX19PUlJSoMPoNZFWX/BfZ1Xl0HFlZ7WHT6vb2FndRll924nWwaC+wqi0aEalRTEyLZqsJCEqyO/ea/Yo1U1KZaNSXd/ARcO693eePXt2iarm+9vmZguiDBjqs5wN7PctoKq1wAIAcT6dd3tfqOoTwBPebf/mPd5pVHUpsBQgPz9fO45ouHXr1hMPg/3kS1M6DdatB+WSkpKIiooiOTmZxMREZsyYwcSJE09s//nPf86yZcsAKC8v5+DBgyce6mv/Rz58+HBmzZoFwMyZM6moqOg01jVr1nDLLbdQXV1NfX09n//850lOTqaoqIjnnnuO+Ph4ANLS0vjDH/7ATTfddCJZnan+CQkJTJ069bx/H4EWaSN9Bqq+7V88A3GLdGFhITMuvoT1+2pYs7eKtXurWLu3mqPeb9x946KZMiydeRemM21YOlOGppHe9/xbB4Hk1t/ZzQSxGsgTkeFAOTAf+KpvARFJA46rajNwB1DkTRqIyEBVPSQiw3AuQ33GxVh7je+Y7YWFhbz55pusWrWKxMRECgoK/D6Q1v6hDhAdHU1DQ0Onx7/tttv405/+xOTJk3nqqacoLCzstGx3W03G+LO/uoH3PjlM0Y4jvP/pEaqPtyACUSJEi5x4HyUQFSUn3kdHCdK+Xrzro3zen7Led/nU44j3POWHGyh//XU83ubBiP59KRg9kGk5aUwbls4Fmf77DszpXEsQqtoqIvcAK3Buc31SVTeLyF3e7UtwOqOfFhEPzmWk230O8UdvH0QLcLeqVrkVq5uSk5Opq6vzu62mpob09HQSExPZtm0bH3744Xmfr66ujsGDB9PS0sKzzz5LVlYWAHPmzOGxxx7jgQcewOPxcOzYMQoKCvj617/OokWLznqJyZiOGpo9fLT7KEU7jlD0yWF2HnIu32amxPO5sZlkpfVBVWlT8KjSpooqeNpOvm9T9S7jLat42nzee8uoKm1tznHaj9m+r+9xPG1KSpxwff4IpuWkMXVoesi3DgLJ1ecgVHU5sLzDuiU+71cBeZ3se6mbsfWWjIwMZs2axYQJE+jTpw+ZmZknts2dO5clS5YwadIkRo8ezUUXXXTe5/vxj3/MzJkzycnJYeLEiSeS0+LFi1m4cCFPPPEE0dHRPPbYY0yYMIHvf//7fPaznyU6OpqpU6fy1FNPnXcMJjypKtsr6ija4bQSPt5TSXNrG/ExUcwY3o/5Fw7l0rwBXJCZFNCWqXO5ZXTAzh9OXOukDoT8/HztOKPc1q1bGTt27Fn3tcH6zuxcf4/BzvoguuZofRMrdx6haMcR3vvkMIfqmgC4IDOJy/IGcOkFA5g5vF9A7tHvTKT9jeG8Z5QLSCe1MSbENLe2sXZvFUXevoRN+2tQhbTEWC4Z1d+bFPozONX/bdYmvFiCCFF3330377///inr7r//fhYsWBCgiEyoKj16jKIdh3l3xxFWfXqEY80eoqOEacPSWPS5C7jsggFMzEq1jt0IZAkiRD366KOBDsGEqLrGFlZ9evREK2Fv5XEAstP7MG9qFpflDeDiURmkJNgEUZHOEoQxYa6tTdm0v+ZE5/KavVW0timJcdF8ZkQGt18ynMsuGEBuRqLd9mxOYQnCmABr9bTR7GmjpVVp8nho8SjNrW20eNpobnW2+S63eNpoam07a7lmTxtbdzeyqOgNqo63ADB+SAp3XjaCy/IGMD0nnbiY4BxF1AQHSxDGuKT6eDOvb6ngtU0H2Vt5vNMP886Giu6umCghLiaK2OgoYrSN2WMHcdkFA7gkrz/9k+LPfgBjvCxBGNODKo818/rmgyzfdJAPdh6htU3JSuvD5KGpxEVHnfjgjouJOm355PqTH/Bx0VHExkQR7/0ZF+1v/1PLR/l0Jju3P04J3C/EhDRLEEEmKSmJ+vp69u/fz3333cdLL710WpmCggIeeeQR8vP93rpsetnR+iZWbK5g+cYDrNp1FE+bMqxfIrdfOpxrJg5mYlaqXds3IckSRJAaMmSI3+RggsPhuiZe23yQVzce4MNdR2lTyM1I5O8uG8HVEwczfkiKJQUT8iIrQbz6IBzc6HdTH08rRHfj1zFoIlz1cKebv/e975GTk3NiwqAf/vCHiAhFRUVUVVXR0tLCT37yE+bNm3fKfnv27OHaa69l06ZNNDQ0sGDBArZs2cLYsWPPOFgfwLe+9S1Wr15NQ0MDN954Iz/60Y8AWL16Nffffz/Hjh0jPj6eP/3pTyQmJvK9732PFStWICLceeed3HvvvWc8fqQ6VNvIa5sP8rcNB/h4TyWqzkBw3y4YxdUTBzN2cLIlBRNWIitBBMD8+fN54IEHTiSIF198kddee41FixaRkpLCkSNHuOiii7j++us7/XB57LHHSExMZMOGDWzYsIFp06ad8Zw//elP6devHx6Phzlz5rBhwwbGjBnDV77yFV544QUuvPBCamtr8Xg8LF26lN27d7N27VpiYmKorKzs8d9BKDtY08irmw7w6saDrC51ksKogUnce3keV08cxOhMSwomfEVWgjjDN/0Gl8Zimjp1KocOHWL//v0cPnyY9PR0Bg8ezKJFiygqKiIqKory8nIqKioYNGiQ32MUFRVx3333ATBp0iQmTZp0xnO++OKLLF26lNbWVg4cOMCWLVsQEQYPHsyFF14IQEpKCnV1dbz55pvcddddxMQ4/xRsNFdn2OpXNx1k+cYDlJQ6gwiPzkzm/jl5XDNxMHmZkTVml4lckZUgAuTGG2/kpZde4uDBg8yfP59nn32Ww4cPU1JSQmxsLLm5uX7ngfB1rt9Sd+/ezSOPPMLq1atJT0/ntttuo7GxsdO5H2xOCEdZ1XFe3XiQ5ZsOsHZvNQBjBiXzD1dcwFUTBzNqYGTNymYMWILoFfPnz+fOO+/kyJEjvPvuu7z44osMHDiQ2NhY3nnnHUpLS8+4/2WXXcazzz7L7Nmz2bRpExs2bOi0bG1tLX379iU1NZWKigpeffVVCgoKGDNmDPv372f16tVceOGF1NXV0draypVXXsmSJUsoKCg4cYkpUloR+yqPs3zjAZZvPMD6Mmc61fFDUvjO50dz1YRBjBhgScFENksQvWD8+PHU1dWRlZXF4MGD+drXvsZ1111Hfn4+U6ZMYcyYMWfc/1vf+hYLFixg0qRJTJkyhRkzZnRadvLkyUydOpXx48czYsSIE1OVxsXF8cILL3DvvffS0NBAnz59WLZsGXfccQc7duxg0qRJxMbGcuedd3LPPff0aP2DyaHjbTxW+CnLNx5gY7mTFCZmpfK9uWO4asIgcvv3PcsRjIkcrs4HISJzgcU4M8o9rqoPd9ieDjwJjAQagW+q6ibvtkU405AqsBFYoKpnvA5j80F0TaTMB9HWpry2+SC/LtzJpvJaACYPTePqCYO4euJghvZLDHCE7rG5ESJDyM0HISLRwKPAFUAZsFpEXlHVLT7FHgLWqeoXRWSMt/wcEckC7gPGqWqDiLyIM6f1U27Fa8KPqvLujsM88vp2NpXXMmpgEvNHx3HPF2aRnR6+ScGYnuLmJaYZwE5V3QUgIs8D83Dmnm43Dvh3AFXdJiK5ItI+J2cM0EdEWoBEYL+LsYakmTNn0tTUdMq6Z555hokTJwYoouCxek8lP3ttOx/vqWRovz78102TmTcli/eK3rXkYMw5cjNBZAH7fJbLgJkdyqwHbgBWisgMIAfIVtUSEXkE2As0AK+r6uvdDSRc79T56KOPeuU8oTQt7abyGn7++nbe2X6YAcnx/HjeeL5y4TAbtdSYbnCtD0JEvgx8XlXv8C7fAsxQ1Xt9yqTg9FFMxelnGIPT77AX+CPwFaAa+D/gJVX9vZ/zLAQWAmRmZk5//vnnT9melJREZmYmqalnHg/H4/EQHR088+r2hnOps6pSU1NDRUUF9fX1vRRZ1x081sbLnzTz8UEPfWPhmuGxzMmJJT761L95fX09SUmRc3dSpNUXrM5dNXv27IDMSV0GDPVZzqbDZSJVrQUWAIjz6b3b+/o8sFtVD3u3vQxcDJyWIFR1KbAUnE7qjh01LS0tlJWVUV5efsZgGxsbSUhIOPfahYFzrXNCQgKTJ08mNjb4Zhgrr27gl29+wktryoiPieK+y0dx+6UjSO3jP9ZI68CMtPqC1bknuZkgVgN5IjIcKMfpZP6qbwERSQOOq2ozTsuhSFVrRWQvcJGIJOJcYpoDnHp70jmKjY1l+PDhZy1XWFjI1KlTu3OKkBXKdT5S38Sj7+zk2Q/3AvCNz+Ty7dkjbb4DY3qQawlCVVtF5B5gBc5trk+q6mYRucu7fQkwFnhaRDw4nde3e7d9JCIvAWuAVmAt3laCiWw1DS08/t4unli5m6bWNr48PZt75+SRldYn0KEZE3ZcfVBOVZcDyzusW+LzfhWQ18m+PwB+4GZ8JnQ0NHt46oM9LHn3U2oaWrh20mAWXXEBI+1pZ2NcY09Sm6DW3NrG86v38qu3d3K4ronLxwzkH668gPFDUgMdmjFhzxKECUqeNuVPa8v57zd3UFbVwIzh/Xjsa9PIz42McaKMCQaWIExQUVVWbD7Iz1/fwSeH6pmQlcJPvziRy/L6h+WzLMYEM0sQJiioKit3HuFnK7azoayGkQP68tjXpjF3wiBLDMYEiCUIE3AlpVX8bMU2PtxVSVZaH3524yS+ODWLmGh7+tmYQLIEYQJm64FaHlmxnbe2HaJ/Ujw/un4882cMJT4msp5oNyZYWYIwvW73kWP89xs7eGX9flISYvju3NHcdnEuiXH2z9GYYGL/I02vaW5t45HXt/PEyt3ERUdx9+yRLLx0JKmJwTeER484Xgk7XoPty+HYUeibAYn9oW9/n58ZJ5cTMyAmLtBRG3OCJQjTK/ZXN3DPc2tYs7eam2cM5e+vGM2A5DAcFqNqD2xbDtv+Bns/AG2DlCxIz4Ujn8CxVdBQ6az3Jz61QyLpkEA6rouzocuNeyxBGNe9s/0Qf//COpI8Nbw9/l1GHD8Aay6C3FmQNR1iQjhRqMKB9U4rYdvfoGKTs37geLj0H2HM1TB4CvjeidXmgYZqOH4Ejh3x+Xn01OXqvbB/rfO+rcX/+WMTvcnEf1Lpd7QCDmVCWo4lE9NlliCMazxtyn+/sYPfvrOF76a9w4K2l4nadRwyRsHON51C0fGQfaGTLHJmOe+D/YPM0wKl7zsJYdtyqC0DiYJhn4HP/xuMvhr6nWGAyKho5wO9bwYMGH3286lCU63/JHLK8mE4vM1Zbm0AYBLAxh87x+k70GnJpOc4P9NyTi6nZDlxGePDEoRxxaG6Rh54bg39S//Kh8kvkd5YARdcBVf8yPlQPF4Je1fBnvehdCUU/Qz0PyAq1mlVtCeMoTMhPgjGW2qqg51vOUnhkxXQWAMxfWDUHJj9EFww1/nAd4MIJKQ6r4yR57ZP8zE4doQ1Ra8xbUQGVJc6l7+q9sC+j2HTy6Cek+WjYiB1qP/kkT4c+qSf2goyEcEShOlxqz49yu+e+z0Ptv4vk+I+hYxJcOVvYcRnTxZK7AdjrnFe4Hzg7v3ISRZ73oeVv4D3fg4SDUOmQs7FkHsJDLvI+aDsDXUVsONVJynsKgRPM/TpB2Oucy4djZgdvK2duL4Q15fa1NEwseD07Z5Wp+VTtQeqvMmjPYls/YvTMjnleMmdtz7ShkFshI6mqwoHN8COFc5rxp0weX6go+oxliBMj2lrU5579W0GfPhvLI0upiVpMFz5G5h4E0Sd5aG3hFS44ErnBdBUD/s+ci7l7HkfPnwMPvilcyln0ETIucRpZQz7jJNsesqRT2DbX51LR2WrAXU+CGcsdC4dDZ0J0WHw3yY6xvsBn+t/e1OdkzhOtDy8P4/udC4PtjaeWj5p0OkJZPAkpy/mbH/7UNNUD7vfde5Q++QNqDsACGRNC7tEGQb/0k0wqD58gNVPP8hXav+CJyaOpsu+T/yse7r/DTs+ybl8M2qOs9zS4Hxg73nfSRrFT8CHjzrbBo4/eUkqZxYkDTj387S1QXmJkxS2L4cjO5z1g6fA7O87LZyBYyPv8kp8Mgya4Lw6UoX6Q6e2OtqTyJ73YcOLgHcq44Q0p/WXM8v5OWhSaCbYyt3wyetOK2HPe05rMj4FRs6GvM9D3hWQNDDQUfa4EPxLmaDS0kj5678gZfViLtcGdg67kQtu+imSnNmz54ntA8Mvc14ArU3OB3t7wlj7e/jYO6dU/9EnE0buJZA86NRjtTbB7iJvUngV6iuca/C5l3pbCldBanbPxh9ORCA503kNm3n69tYm5w6ssmLnb1P6vpN8wblUNcx7B1vOJTBkCkQH4XMwnhanBbvjNdjxOhzZ7qzPGOX8G8m70mm9hvlzK64mCBGZCyzGmVHucVV9uMP2dOBJYCTQCHxTVTeJyGjgBZ+iI4D/p6q/cDNe0wWq6KY/Uv+3fyGrcT/vR02n/xcfZvTEGb1z/ph47zfTi4HvOP+h96872Yex4f+g+EmnbL8RTrIYNJFxm/8MH6yH5nqIS3K++Y2+xvnZJ613Yg93MfHQP895TbnZWVe7H0o/gD0rnYTx5hvO+ti+MHSGN5kH+LbnY0dh5xtOK2HnW9BU49w0kTsL8hc4SeFcbxIIE64lCBGJBh4FrgDKgNUi8oqqbvEp9hCwTlW/KCJjvOXnqOp2YIrPccqBZW7FarqodBWe1x4i+sAa9rXlsHzIz7jz1gWBfSI6OhaGXui8LlnkdMJWbDzZwtj6F1j7DKlx6TDxyzDmWhh+aWg/gxFKUobAxBudFziXqEo/ONnH9M5PnPUxCc6tzu0JI/tC967rqzrPrbS3Etr7nPoOhHHXOZeORs52LrdFKDdbEDOAnaq6C0BEngfm4cw93W4c8O8AqrpNRHJFJFNVK3zKzAE+VdVSF2M15+Lop/DmD2DrX6iUDP6z9S7yPnc7//DZvOAbkjs6xrn7achUuPgep6+htoxVa3dSMPvyQEdnkgbC+C84L3Bue25PGKXvQ9F/wrttPX/bc/PxUzuYa8ud9UOmQsGDTith8JTw61jvJlFVdw4sciMwV1Xv8C7fAsxU1Xt8yvwbkKCqfy8iM4APvGVKfMo8CaxR1f/p5DwLgYUAmZmZ059//vluxVtfX09SUhDcb9+LzrXOMS215O55kSH7X6WVGP6n5TpejL6Gb05J4YL00Hq4KtL+zqFa3+jWY6TWbCWtehNp1ZtIrvsUoY02iaY+aSTVaROoThtPTepYPDF9T9m3Y50TGiroV1lMxtES0qs2EKUttEYnUJU+laMZ+VT2m05zfHpvV7FHnc/fefbs2SWqmu9vm5stCH9fKTtmo4eBxSKyDtgIrAVaTxxAJA64Hvinzk6iqkuBpQD5+flaUFDQrWALCwvp7r6h6qx1bm1yOn4//BnaVMeHqddw38GrGD1qFH+dP4X+SaF3eSbS/s6hXd9rTr5tqoN9HxNV+j4pe94npfwvDNv3st/bnt/9cC2fHR538tmEw1udY/QbATPvhLwricm5mAEx8XThfreg5tbf2c0EUQYM9VnOBvb7FlDVWmABgDjXKHZ7X+2uwmk9+F5yMm5Thc3L4M0fQnUpx4bNZlHlDbxRkcH9n8vj3svziI4KsktKJrzFJ59623PzcafPoPR959LU6se9tz0Ls6LjoajRuTMt52KY+nXnSff+owJahVDkZoJYDeSJyHCcTub5wFd9C4hIGnBcVZuBO4Aib9JodzPwBxdjNB3t/Qhe/77zny9zAh9c/Dh3rEymT2w0T39zCpfmhct3LhPS4hKdJ/Pbn873ue25YnsxWbNudjqYe+up+zDlWoJQ1VYRuQdYgXOb65OqullE7vJuXwKMBZ4WEQ9O5/Xt7fuLSCLOHVB/51aMxkflLqfFsOXPkDSIlmt+yY/2Teb3b5dzYW4Kv7p5GoNSEwIdpTH++dz2/IkWkjW+INARhQVXn4NQ1eXA8g7rlvi8XwXkdbLvccCl0c/MCccroegRp68hOhYK/ol9Y+7gW/+3lU3l5fzdZSP4x8+PJtbmhzYm4tiT1JGqtYnsfX+GD7/hDJQ39etw+T+zYi/845ISBPjtrflcMa6Hn4g2xoQMSxCRorUJDm5yrtOWF8OelYyqLYeRl8MVP6ZlwDj+49VtPL5yN5OyU3n0q9MY2i9IRyo1xvQKSxDhSNV5qK09GZSXwMGNzgBj4Iy8mZ3Phpw7mPSlv3emA/3NKtbsreaWi3L452vHEh8TWs83GGN6niWIcHDsiDMw2omEsAYaq51tsX2dYYgv+pbzRGpWvjPsgQiVhYW8u+MwDzy/lubWNn5581SunzwkoFUxxgQPSxChpvm4M0GJb0Ko3utskyhn6OvxXziZDAaM9juVpKdNefmTZv6y4mMuGJjMr78+jZEDQu+JW2OMeyxBBLM2jzM/QXnJyYRQsfnkVJGpw5zWwYyFTkIYPNmZSewsPG3KgqdWU/RpCzdOz+bH8ybQJ84uKRljTmUJIpjUHjjZZ1BW7Axf3VznbItPdZLBJYsgOx+GTHPG4++GTeU1FO04zA15sTzy5ck9F78xJqxYggik2gOw4XlvQiiBOu9IJFGxzkxek+c7LYPsfOg3ssdGmCwprQLgkiz78xtjOmefEIGyuwj+bwEcP+IMIpY7y+kzyJruDD4W695TyyWlVWSl9aFfgj38ZozpnCWI3qYKH/zKmVchYxTc9jcYOKYXT68Ul1Yyc3gGUNNr5zXGhB5LEL2pqQ7+fA9s+ROMvR6+8Oten62qvLqBitompuekQ7MlCGNM5+waQ2858gn8dg5sfQWu+Fe46emATGXY3v8wPSe0J0gxxrjPEkRv2PoXWDrb6W+45U8w634I0BSdxXuq6BsXzZhBkTvPrjHm3NglJje1eeDtH8PK/3Y6n296GlKzAxpSSWkVU4alEWOjsxpjzsI+Jdxy7Cj8/gYnOUy/DRa8GvDkUN/UyraDtUzP6RfQOIwxocFaEG4oXwMv3gr1h+D6X8G0WwMdEQDr9lbTppBv/Q/GmHPgagtCROaKyHYR2SkiD/rZni4iy0Rkg4h8LCITfLalichLIrJNRLaKyGfcjLXHrHkGnpzrvP/ma0GTHACKSysRgSnD0gIdijEmBLjWghCRaOBRnGlDy4DVIvKKqm7xKfYQsE5VvygiY7zlvbOSsxh4TVVvFJE4ILgnJ2htgle/CyVPwYgC+NKT0De4JsQrKa1idGYyKQmxgQ7FGBMC3GxBzAB2quouVW0GngfmdSgzDngLQFW3AbkikikiKcBlwBPebc2qWu1irOenpgx+d5WTHC5ZBF9/OeiSg6dNWbu3mvxcu7xkjDk3bvZBZAH7fJbLgJkdyqwHbgBWisgMIAfIBjzAYeB3IjIZKAHuV9VjHU8iIguBhQCZmZkUFhZ2K9j6+vpu7ZtWtYFxWx4hqq2ZbeMf5EjMZ6DovW7F4Ka9tR7qm1pJPF5BYeFRoPt1DmWRVudIqy9YnXuUqrryAr4MPO6zfAvwqw5lUoDfAeuAZ4DVwGQgH2gFZnrLLQZ+fLZzTp8+XbvrnXfe6doObW2qK3+h+sM01V9dqHpoe7fP3Rue/mC35nzvr7r36LET67pc5zAQaXWOtPqqWp27CijWTj5T3WxBlAFDfZazgf2+BVS1FlgAICIC7Pa+EoEyVf3IW/Ql4LRO7oBpqoM/3w1b/gzj5sG8RwPyVHRXlJRWMTA5nuz0PoEOxRgTItxMEKuBPBEZDpQD84Gv+hYQkTTguDp9FHcARd6kUSsi+0RktKpux+m43kIwOLwDXvg6HP3EGTLj4vsC9lR0VxSXVjE9Jx0JgViNMcHBtQShqq0icg+wAogGnlTVzSJyl3f7EmAs8LSIeHASwO0+h7gXeNZ7B9MuvC2NgNr6F1j2LYiJc4bMGPHZQEd0TipqGymrauC2i3MDHYoxJoS4+qCcqi4HlndYt8Tn/Sogr5N91+H0RQReEA6Z0RU2QJ8xpjvsSeqzOXYU/vhN2FXoDJlx1X9CTHygo+qS4j1VxMdEMX5IaqBDMcaEEEsQZ3LKkBn/A9NuCXRE3VJSWsnkoWnExdjQW8aYc2efGJ1Z83SHITNCMzk0NHvYvL/WLi8ZY7rMWhAdtTbB8u/Amv8N2iEzumJ9WTWtbWoD9BljuswShK+aMnjhFti/xhky4/J/gajoQEd1Xto7qKcNswRhjOkaSxBeaVUb4DffhNZm+MrvYex1gQ6pR5SUVjFyQF/S+8YFOhRjTIixPghVeH8xk9f/ABL7w51vh01yaGtTSkqryLcJgowx3WAJoqEKPnyMwwMugjvfggEXBDqiHvPp4XpqGlqYbiO4GmO6odMEISL/2f7Uc4f1i0TkP9wNqxcl9oM732bLuO8G/XhKXWUPyBljzseZWhDXAkv9rF8MXONOOAGSMiQkxlPqquLSKvr1jWNE/76BDsUYE4LOlCBUVdv8rGwDwu/TNAytKa1i2jAboM8Y0z1nShDHReS0cZK86xrcC8n0hKP1Tew6cswuLxljuu1Mt7n+P+BVEfkJzoxu4Aye90/AAy7HZc5Te/+DTTFqjOmuThOEqr4qIl8AvoMz9DbAJuBLqrqxF2Iz56FkbxWx0cLELBugzxjTPZ0mCBFJACpU9Rsd1g8UkQRVbXQ9OtNtJXuqmJCVSkJsaD8JbowJnDP1QfwSuNTP+iuA/3YnHNMTmlo9bCivsfGXjDHn5UwJ4hJVfbnjSlV9FrjsXA4uInNFZLuI7BSR0+aUFpF0EVkmIhtE5GMRmeCzbY+IbBSRdSJSfC7nM45N5bU0t7Yx3Z6gNsachzN1Up/p3sizPoEtItHAozgtjjJgtYi8oqq+c0s/BKxT1S+KyBhv+Tk+22er6pGzncucqqS0ErAH5Iwx5+dMH/SHRGRGx5XedYfP4dgzgJ2quktVm4HngXkdyowD3gJQ1W1ArohknlPkplPFe6rIyUhkQHJozXxnjAkuZ2pBfAd4UUSe4tTbXG8F5p/DsbOAfT7LZcDMDmXWAzcAK72JJwfIBioABV4XEQV+o6r+nupGRBYCCwEyMzMpLCw8h9BOV19f3+19g4mq8uHO40zsH3PW+oRLnbsi0uocafUFq3NPOtNtrh+LyEzg28BtOB/Ym4Fv4CSJj85ybH+XqLTD8sPAYhFZB2wE1gKt3m2zVHW/iAwE3hCRbapa5CfOpXiHBMnPz9eCgoKzhOVfYWEh3d03mOw5cozaFYVcPXMMBTNzzlg2XOrcFZFW50irL1ide9IZ54NQ1QrgByIyFbgZJzlcBvzxHI5dBgz1Wc4G9nc4fi2wAECc8SB2e1+o6n7vz0MisgznktVpCcKcqrj9ATnroDbGnKczPQdxAc6lpJuBo8ALgKjq7HM89mogT0SGA+XeY321wznSgOPePoo7gCJVrRWRvkCUqtZ5318J/GuXahahSkqrSE6IIW9gUqBDMcaEuDO1ILYB7wHXqepOcIb6PtcDq2qriNwDrACigSdVdXP7EOKqugQYCzwtIh5gC3C7d/dMYJl3kLkY4DlVfa1LNYtQJaWVTBuWTlSUDdBnjDk/Z0oQX8L51v+OiLyGcxdSlz51VHU5sLzDuiU+71cBpw0IqKq7gMldOZeBmuMt7Kio57pJQwIdijEmDHR6m6uqLlPVrwBjgEJgEZApIo+JyJW9FJ/pgjX7bIIgY0zPOesDb6p6TFWfVdVrcTqa1wGnPRVtAq9kTxXRUcKUYWmBDsUYEwa6NCe1qlaq6m9U9XK3AjLdV1xaybjBKSTGnfHmNGOMOSddShAmeLV42li/r8YuLxljeowliDCx9UAtDS0eSxDGmB5jCSJM2AxyxpieZgkiTBSXVjEkNYHBqX0CHYoxJkxYgggDqkrJniqm59rwGsaYnmMJIgzsr2nkYG2jzSBnjOlRliDCQPEemyDIGNPzLEGEgZLSKhLjohkzKDnQoRhjwogliDBQUlrF1GFpxETbn9MY03PsEyXE1Te1svVALdOH2eUlY0zPsgQR4tbtraZNsTuYjDE9zhJEiCsprUIEptoAfcaYHmYJIsQVl1YyOjOZlITYQIdijAkzriYIEZkrIttFZKeInDZEuIiki8gyEdkgIh+LyIQO26NFZK2I/NXNOEOVp01Zu7fabm81xrjCtQQhItHAo8BVwDjgZhEZ16HYQ8A6VZ0E3Aos7rD9fmCrWzGGuh0VddQ3tdr4S8YYV7jZgpgB7FTVXarajDNl6bwOZcYBbwGo6jYgV0QyAUQkG7gGeNzFGENasXeAvunDrIPaGNPz3JxZJgvY57NcBszsUGY9cAOwUkRmADk4s9ZVAL8Avguc8ekvEVkILATIzMyksLCwW8HW19d3e99AWb6+kdR44dMNH7FLujRdOBCadT5fkVbnSKsvWJ17kpsJwt8nlnZYfhhYLCLrgI3AWqBVRK4FDqlqiYgUnOkkqroUWAqQn5+vBQVnLN6pwsJCurtvoPzLx29zcV4qs2dP79b+oVjn8xVpdY60+oLVuSe5mSDKgKE+y9nAft8CqloLLAAQEQF2e1/zgetF5GogAUgRkd+r6tddjDekHKptZF9lA9/4TG6gQzHGhCk3+yBWA3kiMlxE4nA+9F/xLSAiad5tAHcARapaq6r/pKrZqprr3e9tSw6nap8gyO5gMsa4xbUWhKq2isg9wAogGnhSVTeLyF3e7UuAscDTIuIBtgC3uxVPuCkurSI+JorxQ1IDHYoxJky5eYkJVV0OLO+wbonP+1VA3lmOUQgUuhBeSCsurWJydhpxMfasozHGHfbpEoIaWzxsLq9huj3/YIxxkSWIELR+XzWtbWojuBpjXGUJIgQVWwe1MaYXWIIIQWtKqxg5oC/pfePOXtgYY7rJEkSIaWtTSvZWWevBGOM6SxAhZteReqqPt5CfY+MvGWPcZQkixJx4QM7uYDLGuMwSRIgp3lNFemIsI/r3DXQoxpgwZwkixJSUOv0P0o3RW40xpissQYSQymPN7DpyjOnW/2CM6QWWIEKIDdBnjOlNliBCSHFpJbHRwqRsG6DPGOM+SxAhZE1pFROyUkmIjQ50KMaYCGAJIkQ0tXpYX1Zj4y8ZY3qNJYgQsam8lubWNvLt+QdjTC+xBBEi1ng7qKdZB7Uxppe4miBEZK6IbBeRnSLyoJ/t6SKyTEQ2iMjHIjLBuz7Bu7xeRDaLyI/cjDMUFJdWMqxfIgOTEwIdijEmQriWIEQkGngUuAoYB9wsIuM6FHsIWKeqk4BbgcXe9U3A5ao6GZgCzBWRi9yKNdipKiWl1eRb68EY04vcbEHMAHaq6i5VbQaeB+Z1KDMOeAtAVbcBuSKSqY56b5lY70tdjDWo7a08zpH6Jht/yRjTq9yckzoL2OezXAbM7FBmPXADsFJEZgA5QDZQ4W2BlACjgEdV9SN/JxGRhcBCgMzMTAoLC7sVbH19fbf3ddv75S0AtB36lMLC3T123GCus1sirc6RVl+wOvckNxOEv8GCOrYCHgYWi8g6YCOwFmgFUFUPMEVE0oBlIjJBVTeddkDVpcBSgPz8fC0oKOhWsIWFhXR3X7e9vmwjyQn7+do1s4mK6rkxmIK5zm6JtDpHWn3B6tyT3EwQZcBQn+VsYL9vAVWtBRYAiDP63G7vy7dMtYgUAnOB0xJEJCjZU8W0Yek9mhyMMeZs3OyDWA3kichwEYkD5gOv+BYQkTTvNoA7gCJVrRWRAd6WAyLSB/gcsM3FWINWTUMLOw7V2fhLxphe51oLQlVbReQeYAUQDTypqptF5C7v9iXAWOBpEfEAW4DbvbsPBv7X2w8RBbyoqn91K9ZgtnZvFarYHUzGmF7n5iUmVHU5sLzDuiU+71cBeX722wBMdTO2UFFSWkV0lDB5aFqgQzHGRBh7kjrIFe+pYuzgZPrGu5rLjTHmNJYgglirp411+6rJtwmCjDEBYAkiiG09UEdDi8fGXzLGBIQliCBWXFoJWAe1MSYwLEEEsZLSKoakJjAkrU+gQzHGRCBLEEGspLTKLi8ZYwLGEkSQKq9u4EBNo11eMsYEjCWIIFXinSAoP9fuYDLGBIYliCBVsqeSxLhoxgxKDnQoxpgIZQkiSJXsrWLK0DRiou1PZIwJDPv0CULHmlrZeqDO+h+MMQFlCSIIrdtXjadN7Q4mY0xAWYIIQiWlVYhgCcIYE1CWIIJQcWkVozOTSUmIDXQoxpgIZgkiyHjalLX2gJwxJghYgggynxyqo66p1TqojTEB52qCEJG5IrJdRHaKyIN+tqeLyDIR2SAiH4vIBO/6oSLyjohsFZHNInK/m3EGk+I93gfkbIhvY0yAuZYgvNOFPgpcBYwDbhaRcR2KPQSsU9VJwK3AYu/6VuAfVHUscBFwt599w1JJaRX9k+IZ2s8G6DPGBJabLYgZwE5V3aWqzcDzwLwOZcYBbwGo6jYgV0QyVfWAqq7xrq8DtgJZLsYaNEpKq8jPSUdEAh2KMSbCuTmPZRawz2e5DJjZocx64AZgpYjMAHKAbKCivYCI5OLMT/2Rv5OIyEJgIUBmZiaFhYXdCra+vr7b+/aU6qY29lY2cPGA1l6JJRjq3Nsirc6RVl+wOvckNxOEv6/A2mH5YWCxiKwDNgJrcS4vOQcQSQL+CDygqrX+TqKqS4GlAPn5+VpQUNCtYAsLC+nuvj3l1Y0HgDXcNCefacPc76QOhjr3tkirc6TVF6zOPcnNBFEGDPVZzgb2+xbwfugvABDnmspu7wsRicVJDs+q6ssuxhk0SkqriI+JYsKQ1ECHYowxrvZBrAbyRGS4iMQB84FXfAuISJp3G8AdQJGq1nqTxRPAVlX9LxdjDCrFpVVMzk4jLsbuPjbGBJ5rn0Sq2grcA6zA6WR+UVU3i8hdInKXt9hYYLOIbMO526n9dtZZwC3A5SKyzvu62q1Yg0Fji4fN+2vsATljTNBw8xITqrocWN5h3RKf96uAPD/7rcR/H0bY2lBWQ4tH7QE5Y0zQsGsZQaK4tBKwAfqMMcHDEkSQKNlTxYgBfenXN+7shY0xphdYgggCqkrJ3iq7vGSMCSqWIILAp4ePUX28hemWIIwxQcQSRBBYU+oM0DfdBugzxgQRSxBBoLi0krTEWEYO6BvoUIwx5gRLEEGguLSK6cNsgD5jTHCxBBFglcea2XX4GNNzrf/BGBNcLEEEWHv/g00QZIwJNpYgAqy4tIrYaGFStg3QZ4wJLpYgAmxNaRXjh6SSEBsd6FCMMeYUliACqLm1jfVl1faAnDEmKFmCCKBN+2toam2zB+SMMUHJEkQAnXhAzu5gMsYEIUsQATQgOZ7rJg9hYHJCoEMxxpjTuDofhDmzeVOymDclK9BhGGOMX662IERkrohsF5GdIvKgn+3pIrJMRDaIyMciMsFn25MickhENrkZozHGGP9cSxAiEg08ijOV6DjgZhEZ16HYQ8A6VZ0E3Aos9tn2FDDXrfiMMcacmZstiBnATlXdparNwPPAvA5lxgFvAajqNiBXRDK9y0VApYvxGWOMOQM3+yCygH0+y2XAzA5l1gM3ACtFZAaQA2QDFed6EhFZCCwEyMzMpLCwsFvB1tfXd3vfUGV1Dn+RVl+wOvckNxOEv6FJtcPyw8BiEVkHbATWAq1dOYmqLgWWAuTn52tBQUGXAwUoLCyku/uGKqtz+Iu0+oLVuSe5mSDKgKE+y9nAft8CqloLLAAQZ6zr3d6XMcaYAHOzD2I1kCciw0UkDpgPvOJbQETSvNsA7gCKvEnDGGNMgLmWIFS1FbgHWAFsBV5U1c0icpeI3OUtNhbYLCLbcO52ur99fxH5A7AKGC0iZSJyu1uxGmOMOZ2oduwWCF0ichgo7ebu/YEjPRhOKLA6h79Iqy9YnbsqR1UH+NsQVgnifIhIsarmBzqO3mR1Dn+RVl+wOvckG4vJGGOMX5YgjDHG+GUJ4qSlgQ4gAKzO4S/S6gtW5x5jfRDGGGP8shaEMcYYvyxBGGOM8SviE8TZ5qwINyIyVETeEZGtIrJZRO4/+17hQUSiRWStiPw10LH0Bu9IBS+JyDbv3/szgY7JbSKyyPvvepOI/EFEwm66Rn9z5YhIPxF5Q0Q+8f7skXmMIzpBnOOcFeGmFfgHVR0LXATcHQF1bnc/zlP9kWIx8JqqjgEmE+Z1F5Es4D4gX1UnANE4Q/yEm6c4fa6cB4G3VDUPZwqFHvmyG9EJgnObsyKsqOoBVV3jfV+H86ER9vOeikg2cA3weKBj6Q0ikgJcBjwBoKrNqlod0KB6RwzQR0RigEQ6DBAaDjqZK2ce8L/e9/8LfKEnzhXpCcLfnBVh/2HZTkRyganARwEOpTf8Avgu0BbgOHrLCOAw8DvvZbXHRaRvoINyk6qWA48Ae4EDQI2qvh7YqHpNpqoeAOdLIDCwJw4a6QniXOasCEsikgT8EXgg3EfQFZFrgUOqWhLoWHpRDDANeExVpwLH6KHLDsHKe919HjAcGAL0FZGvBzaq0BbpCeKsc1aEIxGJxUkOz6rqy4GOpxfMAq4XkT04lxEvF5HfBzYk15UBZara3jp8CSdhhLPPAbtV9bCqtgAvAxcHOKbeUiEigwG8Pw/1xEEjPUGcdc6KcOOdmOkJYKuq/leg4+kNqvpPqpqtqrk4f+O3VTWsv1mq6kFgn4iM9q6aA2wJYEi9YS9wkYgkev+dzyHMO+Z9vAJ8w/v+G8Cfe+Kgbs4oF/RUtVVE2uesiAaeVNXNAQ7LbbOAW4CN3qleAR5S1eWBC8m45F7gWe+Xn114Z28MV6r6kYi8BKzBuVtvLWE47IZ3rpwCoL+IlAE/wJm++UXvvDl7gS/3yLlsqA1jjDH+RPolJmOMMZ2wBGGMMcYvSxDGGGP8sgRhjDHGL0sQxhhj/LIEYUwXiIhHRNb5vHrs6WQRyfUdodOYQIvo5yCM6YYGVZ0S6CCM6Q3WgjCmB4jIHhH5DxH52Psa5V2fIyJvicgG789h3vWZIrJMRNZ7X+1DQkSLyG+9cxq8LiJ9AlYpE/EsQRjTNX06XGL6is+2WlWdAfwPzuixeN8/raqTgGeBX3rX/xJ4V1Un44yR1P4Efx7wqKqOB6qBL7laG2POwJ6kNqYLRKReVZP8rN8DXK6qu7yDIR5U1QwROQIMVtUW7/oDqtpfRA4D2ara5HOMXOAN76QviMj3gFhV/UkvVM2Y01gLwpieo52876yMP00+7z1YP6EJIEsQxvScr/j8XOV9/wEnp738GrDS+/4t4FtwYq7slN4K0phzZd9OjOmaPj6j4IIz53P7ra7xIvIRzhevm73r7gOeFJHv4Mzw1j6i6v3AUu/omx6cZHHA7eCN6QrrgzCmB3j7IPJV9UigYzGmp9glJmOMMX5ZC8IYY4xf1oIwxhjjlyUIY4wxflmCMMYY45clCGOMMX5ZgjDGGOPX/wfTVJtUcUCsCwAAAABJRU5ErkJggg==\n",
|
|
"text/plain": [
|
|
"<Figure size 432x288 with 1 Axes>"
|
|
]
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"\n",
|
|
"metrics = pd.read_csv(f\"{trainer.logger.log_dir}/metrics.csv\")\n",
|
|
"\n",
|
|
"aggreg_metrics = []\n",
|
|
"agg_col = \"epoch\"\n",
|
|
"for i, dfg in metrics.groupby(agg_col):\n",
|
|
" agg = dict(dfg.mean())\n",
|
|
" agg[agg_col] = i\n",
|
|
" aggreg_metrics.append(agg)\n",
|
|
"\n",
|
|
"df_metrics = pd.DataFrame(aggreg_metrics)\n",
|
|
"df_metrics[[\"train_loss\", \"valid_loss\"]].plot(\n",
|
|
" grid=True, legend=True, xlabel='Epoch', ylabel='Loss')\n",
|
|
"df_metrics[[\"train_acc\", \"valid_acc\"]].plot(\n",
|
|
" grid=True, legend=True, xlabel='Epoch', ylabel='ACC')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "dc29d109",
|
|
"metadata": {},
|
|
"source": [
|
|
"- The `trainer` automatically saves the model with the best validation accuracy automatically for us, we which we can load from the checkpoint via the `ckpt_path='best'` argument; below we use the `trainer` instance to evaluate the best model on the test set:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 14,
|
|
"id": "d86f091d",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Restoring states from the checkpoint path at logs/my-mlp/version_18/checkpoints/epoch=5-step=1283.ckpt\n",
|
|
"LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
|
|
"Loaded model weights from checkpoint at logs/my-mlp/version_18/checkpoints/epoch=5-step=1283.ckpt\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "f4a06c0213a3434e837518f331d62f71",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Testing: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
|
|
"┃<span style=\"font-weight: bold\"> Test metric </span>┃<span style=\"font-weight: bold\"> DataLoader 0 </span>┃\n",
|
|
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
|
|
"│<span style=\"color: #008080; text-decoration-color: #008080\"> test_acc </span>│<span style=\"color: #800080; text-decoration-color: #800080\"> 0.9732000231742859 </span>│\n",
|
|
"└───────────────────────────┴───────────────────────────┘\n",
|
|
"</pre>\n"
|
|
],
|
|
"text/plain": [
|
|
"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
|
|
"┃\u001b[1m \u001b[0m\u001b[1m Test metric \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m DataLoader 0 \u001b[0m\u001b[1m \u001b[0m┃\n",
|
|
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
|
|
"│\u001b[36m \u001b[0m\u001b[36m test_acc \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.9732000231742859 \u001b[0m\u001b[35m \u001b[0m│\n",
|
|
"└───────────────────────────┴───────────────────────────┘\n"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"[{'test_acc': 0.9732000231742859}]"
|
|
]
|
|
},
|
|
"execution_count": 14,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"trainer.test(model=lightning_model, datamodule=data_module, ckpt_path='best')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "23bdd961",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Predicting labels of new data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "9210fe8a",
|
|
"metadata": {},
|
|
"source": [
|
|
"- You can use the `trainer.predict` method on a new `DataLoader` or `DataModule` to apply the model to new data.\n",
|
|
"- Alternatively, you can also manually load the best model from a checkpoint as shown below:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 15,
|
|
"id": "3a7806e0",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"logs/my-mlp/version_18/checkpoints/epoch=5-step=1283.ckpt\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"path = trainer.checkpoint_callback.best_model_path\n",
|
|
"print(path)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 16,
|
|
"id": "b032ab2b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"lightning_model = LightningMLP.load_from_checkpoint(path, model=pytorch_model)\n",
|
|
"lightning_model.eval();"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "286b69ff",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Note that our PyTorch model, which is passed to the Lightning model requires input arguments. However, this is automatically being taken care of since we used `self.save_hyperparameters()` in our PyTorch model's `__init__` method.\n",
|
|
"- Now, below is an example applying the model manually. Here, pretend that the `test_dataloader` is a new data loader."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 17,
|
|
"id": "02178666",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor([7, 2, 1, 0, 4])"
|
|
]
|
|
},
|
|
"execution_count": 17,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"test_dataloader = data_module.test_dataloader()\n",
|
|
"\n",
|
|
"all_true_labels = []\n",
|
|
"all_predicted_labels = []\n",
|
|
"for batch in test_dataloader:\n",
|
|
" features, labels = batch\n",
|
|
" \n",
|
|
" with torch.no_grad(): # since we don't need to backprop\n",
|
|
" logits = lightning_model(features)\n",
|
|
" \n",
|
|
" predicted_labels = torch.argmax(logits, dim=1)\n",
|
|
" all_predicted_labels.append(predicted_labels)\n",
|
|
" all_true_labels.append(labels)\n",
|
|
" \n",
|
|
"all_predicted_labels = torch.cat(all_predicted_labels)\n",
|
|
"all_true_labels = torch.cat(all_true_labels)\n",
|
|
"all_predicted_labels[:5]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "4ad0d2e6",
|
|
"metadata": {},
|
|
"source": [
|
|
"Just as an internal check, if the model was loaded correctly, the test accuracy below should be identical to the test accuracy we saw earlier in the previous section."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 18,
|
|
"id": "8749f5ae",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Test accuracy: 0.9732 (97.32%)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"test_acc = torch.mean((all_predicted_labels == all_true_labels).float())\n",
|
|
"print(f'Test accuracy: {test_acc:.4f} ({test_acc*100:.2f}%)')"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|