rasbt--deeplearning-models
Sebastian Raschka
1153 行
77 KiB
Plaintext
1153 行
77 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "e33c69ee",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"torch : 1.10.1\n",
|
|
"pytorch_lightning: 1.6.0.dev0\n",
|
|
"torchmetrics : 0.6.2\n",
|
|
"matplotlib : 3.3.4\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"%load_ext watermark\n",
|
|
"%watermark -p torch,pytorch_lightning,torchmetrics,matplotlib"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "d292dc3e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%load_ext pycodestyle_magic\n",
|
|
"%flake8_on --ignore W291,W293,E703"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "49accc67",
|
|
"metadata": {},
|
|
"source": [
|
|
"<a href=\"https://pytorch.org\"><img src=\"https://raw.githubusercontent.com/pytorch/pytorch/master/docs/source/_static/img/pytorch-logo-dark.svg\" width=\"90\"/></a> <a href=\"https://www.pytorchlightning.ai\"><img src=\"https://raw.githubusercontent.com/PyTorchLightning/pytorch-lightning/master/docs/source/_static/images/logo.svg\" width=\"150\"/></a>\n",
|
|
"\n",
|
|
"# Model Zoo -- Multilayer Perceptron with BatchNorm"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "fdcdb08c",
|
|
"metadata": {},
|
|
"source": [
|
|
"## General settings and hyperparameters"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "a90854c6",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Here, we specify some general hyperparameter values and general settings\n",
|
|
"- Note that for small datatsets, it is not necessary and better not to use multiple workers as it can sometimes cause issues with too many open files in PyTorch. So, if you have problems with the data loader later, try setting `NUM_WORKERS = 0` instead."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"id": "aa4864f0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"BATCH_SIZE = 256\n",
|
|
"NUM_EPOCHS = 20\n",
|
|
"LEARNING_RATE = 0.005\n",
|
|
"NUM_WORKERS = 4"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "591a95c9",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Implementing a Neural Network using PyTorch Lightning's `LightningModule`"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "30bb2fb3",
|
|
"metadata": {},
|
|
"source": [
|
|
"- In this section, we set up the main model architecture using the `LightningModule` from PyTorch Lightning.\n",
|
|
"- We start with defining our neural network model in pure PyTorch, and then we use it in the `LightningModule` to get all the extra benefits that PyTorch Lightning provides."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "6071e991",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import torch\n",
|
|
"import torch.nn.functional as F\n",
|
|
"\n",
|
|
"\n",
|
|
"# Regular PyTorch Module\n",
|
|
"class PyTorchMLP(torch.nn.Module):\n",
|
|
" def __init__(self, input_size, hidden_units, num_classes):\n",
|
|
" super().__init__()\n",
|
|
"\n",
|
|
" # Initialize MLP layers\n",
|
|
" all_layers = []\n",
|
|
" for hidden_unit in hidden_units:\n",
|
|
" layer = torch.nn.Linear(input_size, hidden_unit, bias=False)\n",
|
|
" all_layers.append(layer)\n",
|
|
" all_layers.append(torch.nn.BatchNorm1d(hidden_unit))\n",
|
|
" all_layers.append(torch.nn.ReLU())\n",
|
|
" input_size = hidden_unit\n",
|
|
"\n",
|
|
" output_layer = torch.nn.Linear(\n",
|
|
" in_features=hidden_units[-1],\n",
|
|
" out_features=num_classes)\n",
|
|
"\n",
|
|
" all_layers.append(output_layer)\n",
|
|
" self.layers = torch.nn.Sequential(*all_layers)\n",
|
|
"\n",
|
|
" def forward(self, x):\n",
|
|
" x = torch.flatten(x, start_dim=1) # to make it work for image inputs\n",
|
|
" x = self.layers(x)\n",
|
|
" return x # x are the model's logits"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"id": "83308f9e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pytorch_lightning as pl\n",
|
|
"import torchmetrics\n",
|
|
"\n",
|
|
"\n",
|
|
"# LightningModule that receives a PyTorch model as input\n",
|
|
"class LightningMLP(pl.LightningModule):\n",
|
|
" def __init__(self, model, learning_rate):\n",
|
|
" super().__init__()\n",
|
|
"\n",
|
|
" self.learning_rate = learning_rate\n",
|
|
" # The inherited PyTorch module\n",
|
|
" self.model = model\n",
|
|
"\n",
|
|
" # Save settings and hyperparameters to the log directory\n",
|
|
" # but skip the model parameters\n",
|
|
" self.save_hyperparameters(ignore=['model'])\n",
|
|
"\n",
|
|
" # Set up attributes for computing the accuracy\n",
|
|
" self.train_acc = torchmetrics.Accuracy()\n",
|
|
" self.valid_acc = torchmetrics.Accuracy()\n",
|
|
" self.test_acc = torchmetrics.Accuracy()\n",
|
|
" \n",
|
|
" # Defining the forward method is only necessary \n",
|
|
" # if you want to use a Trainer's .predict() method (optional)\n",
|
|
" def forward(self, x):\n",
|
|
" return self.model(x)\n",
|
|
" \n",
|
|
" # A common forward step to compute the loss and labels\n",
|
|
" # this is used for training, validation, and testing below\n",
|
|
" def _shared_step(self, batch):\n",
|
|
" features, true_labels = batch\n",
|
|
" logits = self(features)\n",
|
|
" loss = torch.nn.functional.cross_entropy(logits, true_labels)\n",
|
|
" predicted_labels = torch.argmax(logits, dim=1)\n",
|
|
"\n",
|
|
" return loss, true_labels, predicted_labels\n",
|
|
"\n",
|
|
" def training_step(self, batch, batch_idx):\n",
|
|
" loss, true_labels, predicted_labels = self._shared_step(batch)\n",
|
|
" self.log(\"train_loss\", loss)\n",
|
|
" \n",
|
|
" # To account for BatchNorm behavior during evaluation\n",
|
|
" self.model.eval()\n",
|
|
" with torch.no_grad():\n",
|
|
" _, true_labels, predicted_labels = self._shared_step(batch)\n",
|
|
" self.train_acc.update(predicted_labels, true_labels)\n",
|
|
" self.log(\"train_acc\", self.train_acc, on_epoch=True, on_step=False)\n",
|
|
" self.model.train()\n",
|
|
" return loss # this is passed to the optimzer for training\n",
|
|
"\n",
|
|
" def validation_step(self, batch, batch_idx):\n",
|
|
" loss, true_labels, predicted_labels = self._shared_step(batch)\n",
|
|
" self.log(\"valid_loss\", loss)\n",
|
|
" self.valid_acc(predicted_labels, true_labels)\n",
|
|
" self.log(\"valid_acc\", self.valid_acc,\n",
|
|
" on_epoch=True, on_step=False, prog_bar=True)\n",
|
|
"\n",
|
|
" def test_step(self, batch, batch_idx):\n",
|
|
" loss, true_labels, predicted_labels = self._shared_step(batch)\n",
|
|
" self.test_acc(predicted_labels, true_labels)\n",
|
|
" self.log(\"test_acc\", self.test_acc, on_epoch=True, on_step=False)\n",
|
|
"\n",
|
|
" def configure_optimizers(self):\n",
|
|
" optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)\n",
|
|
" return optimizer"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "80b8d600",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Setting up the dataset"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "ca24d761",
|
|
"metadata": {},
|
|
"source": [
|
|
"- In this section, we are going to set up our dataset."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "71cfb0bd",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Inspecting the dataset"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"id": "bcd17554",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import torch\n",
|
|
"\n",
|
|
"from torchvision import datasets\n",
|
|
"from torchvision import transforms\n",
|
|
"from torch.utils.data import DataLoader\n",
|
|
"\n",
|
|
"\n",
|
|
"train_dataset = datasets.MNIST(root='./data', \n",
|
|
" train=True, \n",
|
|
" transform=transforms.ToTensor(),\n",
|
|
" download=True)\n",
|
|
"\n",
|
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
|
" batch_size=BATCH_SIZE, \n",
|
|
" num_workers=NUM_WORKERS,\n",
|
|
" drop_last=True,\n",
|
|
" shuffle=True)\n",
|
|
"\n",
|
|
"test_dataset = datasets.MNIST(root='./data', \n",
|
|
" train=False,\n",
|
|
" transform=transforms.ToTensor())\n",
|
|
"\n",
|
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
|
" batch_size=BATCH_SIZE,\n",
|
|
" num_workers=NUM_WORKERS,\n",
|
|
" drop_last=False,\n",
|
|
" shuffle=False)\n",
|
|
"\n",
|
|
"# Checking the dataset\n",
|
|
"all_train_labels = []\n",
|
|
"all_test_labels = []\n",
|
|
"\n",
|
|
"for images, labels in train_loader: \n",
|
|
" all_train_labels.append(labels)\n",
|
|
"all_train_labels = torch.cat(all_train_labels)\n",
|
|
" \n",
|
|
"for images, labels in test_loader: \n",
|
|
" all_test_labels.append(labels)\n",
|
|
"all_test_labels = torch.cat(all_test_labels)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"id": "40a491ee",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Training labels: tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])\n",
|
|
"Training label distribution: tensor([5914, 6730, 5946, 6124, 5835, 5413, 5906, 6259, 5840, 5937])\n",
|
|
"\n",
|
|
"Test labels: tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])\n",
|
|
"Test label distribution: tensor([ 980, 1135, 1032, 1010, 982, 892, 958, 1028, 974, 1009])\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print('Training labels:', torch.unique(all_train_labels))\n",
|
|
"print('Training label distribution:', torch.bincount(all_train_labels))\n",
|
|
"\n",
|
|
"print('\\nTest labels:', torch.unique(all_test_labels))\n",
|
|
"print('Test label distribution:', torch.bincount(all_test_labels))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "624d7cda",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Performance baseline"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "f38859d7",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Especially for imbalanced datasets, it's quite useful to compute a performance baseline.\n",
|
|
"- In classification contexts, a useful baseline is to compute the accuracy for a scenario where the model always predicts the majority class -- you want your model to be better than that!"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"id": "3d4153c8",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Baseline ACC: 11.35%\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"majority_prediction = torch.argmax(torch.bincount(all_test_labels))\n",
|
|
"baseline_acc = torch.mean((all_test_labels == majority_prediction).float())\n",
|
|
"print(f'Baseline ACC: {baseline_acc*100:.2f}%')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "5301af8a",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Setting up a `DataModule`"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "6998d8b7",
|
|
"metadata": {},
|
|
"source": [
|
|
"- There are three main ways we can prepare the dataset for Lightning. We can\n",
|
|
" 1. make the dataset part of the model;\n",
|
|
" 2. set up the data loaders as usual and feed them to the fit method of a Lightning Trainer -- the Trainer is introduced in the next subsection;\n",
|
|
" 3. create a LightningDataModule.\n",
|
|
"- Here, we are going to use approach 3, which is the most organized approach. The `LightningDataModule` consists of several self-explanatory methods as we can see below:\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"id": "faff5cea",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import os\n",
|
|
"\n",
|
|
"from torch.utils.data.dataset import random_split\n",
|
|
"from torch.utils.data import DataLoader\n",
|
|
"\n",
|
|
"\n",
|
|
"class DataModule(pl.LightningDataModule):\n",
|
|
" def __init__(self, data_path='./'):\n",
|
|
" super().__init__()\n",
|
|
" self.data_path = data_path\n",
|
|
" \n",
|
|
" def prepare_data(self):\n",
|
|
" datasets.MNIST(root=self.data_path,\n",
|
|
" download=True)\n",
|
|
" return\n",
|
|
"\n",
|
|
" def setup(self, stage=None):\n",
|
|
" # Note transforms.ToTensor() scales input images\n",
|
|
" # to 0-1 range\n",
|
|
" train = datasets.MNIST(root=self.data_path, \n",
|
|
" train=True, \n",
|
|
" transform=transforms.ToTensor(),\n",
|
|
" download=False)\n",
|
|
"\n",
|
|
" self.test = datasets.MNIST(root=self.data_path, \n",
|
|
" train=False, \n",
|
|
" transform=transforms.ToTensor(),\n",
|
|
" download=False)\n",
|
|
"\n",
|
|
" self.train, self.valid = random_split(train, lengths=[55000, 5000])\n",
|
|
"\n",
|
|
" def train_dataloader(self):\n",
|
|
" train_loader = DataLoader(dataset=self.train, \n",
|
|
" batch_size=BATCH_SIZE, \n",
|
|
" drop_last=True,\n",
|
|
" shuffle=True,\n",
|
|
" num_workers=NUM_WORKERS)\n",
|
|
" return train_loader\n",
|
|
"\n",
|
|
" def val_dataloader(self):\n",
|
|
" valid_loader = DataLoader(dataset=self.valid, \n",
|
|
" batch_size=BATCH_SIZE, \n",
|
|
" drop_last=False,\n",
|
|
" shuffle=False,\n",
|
|
" num_workers=NUM_WORKERS)\n",
|
|
" return valid_loader\n",
|
|
"\n",
|
|
" def test_dataloader(self):\n",
|
|
" test_loader = DataLoader(dataset=self.test, \n",
|
|
" batch_size=BATCH_SIZE, \n",
|
|
" drop_last=False,\n",
|
|
" shuffle=False,\n",
|
|
" num_workers=NUM_WORKERS)\n",
|
|
" return test_loader"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "04e59026",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Note that the `prepare_data` method is usually used for steps that only need to be executed once, for example, downloading the dataset; the `setup` method defines the the dataset loading -- if you run your code in a distributed setting, this will be called on each node / GPU. \n",
|
|
"- Next, lets initialize the `DataModule`; we use a random seed for reproducibility (so that the data set is shuffled the same way when we re-execute this code):"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"id": "0672a3c9",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"torch.manual_seed(1) \n",
|
|
"data_module = DataModule(data_path='./data')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "07ae2a37",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Training the model using the PyTorch Lightning Trainer class"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "eaf36ed9",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Next, we initialize our model.\n",
|
|
"- Also, we define a call back so that we can obtain the model with the best validation set performance after training.\n",
|
|
"- PyTorch Lightning offers [many advanced logging services](https://pytorch-lightning.readthedocs.io/en/latest/extensions/logging.html) like Weights & Biases. Here, we will keep things simple and use the `CSVLogger`:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"id": "323dc94b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pytorch_lightning.callbacks import ModelCheckpoint\n",
|
|
"from pytorch_lightning.loggers import CSVLogger\n",
|
|
"\n",
|
|
"\n",
|
|
"pytorch_model = PyTorchMLP(\n",
|
|
" input_size=28*28,\n",
|
|
" hidden_units=(128, 256),\n",
|
|
" num_classes=10\n",
|
|
")\n",
|
|
"\n",
|
|
"lightning_model = LightningMLP(\n",
|
|
" pytorch_model, learning_rate=LEARNING_RATE)\n",
|
|
"\n",
|
|
"callbacks = [ModelCheckpoint(\n",
|
|
" save_top_k=1, mode='max', monitor=\"valid_acc\")] # save top 1 model \n",
|
|
"logger = CSVLogger(save_dir=\"logs/\", name=\"my-mlp\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b779d5db",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Now it's time to train our model:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"id": "3807f123",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"GPU available: True, used: True\n",
|
|
"TPU available: False, using: 0 TPU cores\n",
|
|
"IPU available: False, using: 0 IPUs\n",
|
|
"LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
|
|
"\n",
|
|
" | Name | Type | Params\n",
|
|
"-----------------------------------------\n",
|
|
"0 | model | PyTorchMLP | 136 K \n",
|
|
"1 | train_acc | Accuracy | 0 \n",
|
|
"2 | valid_acc | Accuracy | 0 \n",
|
|
"3 | test_acc | Accuracy | 0 \n",
|
|
"-----------------------------------------\n",
|
|
"136 K Trainable params\n",
|
|
"0 Non-trainable params\n",
|
|
"136 K Total params\n",
|
|
"0.546 Total estimated model params size (MB)\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validation sanity check: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "67d119753c214434a09ace4c7e69a42f",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Training: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Training took 1.38 min in total.\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import time\n",
|
|
"\n",
|
|
"\n",
|
|
"trainer = pl.Trainer(\n",
|
|
" max_epochs=NUM_EPOCHS,\n",
|
|
" callbacks=callbacks,\n",
|
|
" progress_bar_refresh_rate=50, # recommended for notebooks\n",
|
|
" accelerator=\"auto\", # Uses GPUs or TPUs if available\n",
|
|
" devices=\"auto\", # Uses all available GPUs/TPUs if applicable\n",
|
|
" logger=logger,\n",
|
|
" deterministic=True,\n",
|
|
" log_every_n_steps=100)\n",
|
|
"\n",
|
|
"start_time = time.time()\n",
|
|
"trainer.fit(model=lightning_model, datamodule=data_module)\n",
|
|
"\n",
|
|
"runtime = (time.time() - start_time)/60\n",
|
|
"print(f\"Training took {runtime:.2f} min in total.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "3a37d475",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Evaluating the model"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cc676417",
|
|
"metadata": {},
|
|
"source": [
|
|
"- After training, let's plot our training ACC and validation ACC using pandas, which, in turn, uses matplotlib for plotting (you may want to consider a [more advanced logger](https://pytorch-lightning.readthedocs.io/en/latest/extensions/logging.html) that does that for you):"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"id": "8370390b",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"<AxesSubplot:xlabel='Epoch', ylabel='ACC'>"
|
|
]
|
|
},
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
},
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABGnElEQVR4nO3dd3hUZfbA8e9J7wkQCGkQpEkvCUURBStgwQKICIiry1r42VZX3aruuqLrWhfBhqKroqgoiyAqEhBRqvTeCaGFEhKSkPb+/rgTHELKTDI3k3I+zzNP5pb33jOXMCf3vW8RYwxKKaWUq3y8HYBSSqm6RROHUkopt2jiUEop5RZNHEoppdyiiUMppZRb/LwdQE2Ijo42SUlJVSp76tQpQkNDPRuQB2l81aPxVY/GV321OcaVK1dmGGOanrPBGFPvX8nJyaaqFixYUOWyNUHjqx6Nr3o0vuqrzTECK0wZ36laVaWUUsotmjiUUkq5RROHUkoptzSIh+NKqfqloKCAtLQ08vLyKtwvMjKSTZs21VBUVVMbYgwKCiIhIQF/f3+X9tfEoZSqc9LS0ggPDycpKQkRKXe/rKwswsPDazAy93k7RmMMR48eJS0tjVatWrlURquqlFJ1Tl5eHk2aNKkwaSjXiAhNmjSp9O7NmSYOpVSdpEnDc9y9lpo4KvD95kPM3pnv7TCUUqpW0cRRgcXbjjJ7RwFG5yxRSqkzNHFUIC4qiLwiOJlb6O1QlFK1yIkTJ3jttdfcLjdkyBBOnDjhdrlx48bx6aeful3OLpo4KhAfFQzA/hO5Xo5EKVWblJc4ioqKKiw3Z84coqKibIqq5tjaHFdEBgEvA77AW8aYiaW2nw+8A/QE/mSMed6xvj3wsdOu5wF/Nca8JCJPAL8Fjji2/dEYM8eO+OMciSP9RC4d4yLsOIVSqpqe/N8GNqafLHNbUVERvr6+bh+zY1wEf7u2U7nbH3vsMXbs2EH37t3x9/cnLCyM2NhYVq9ezcaNG7n++uvZt28feXl53H///YwfPx6ApKQkVqxYQXZ2NoMHD+aiiy5i8eLFJCYm8uWXXxIcHFxpbPPnz+fhhx+msLCQXr16MXnyZAIDA3nssceYNWsWfn5+XHnllTz//PPMmDGDJ598El9fXyIjI1m0aJHb16IstiUOEfEFJgFXAGnAchGZZYzZ6LTbMeA+4HrnssaYLUB3p+PsB2Y67fJiSZKx05nEkal3HEqpX02cOJH169ezevVqUlNTufrqq1m/fv2ZfhBTp06lcePG5Obm0qtXL2666SaaNGly1jG2bdvGRx99xAsvvMAdd9zBZ599xujRoys8b15eHuPGjWP+/Pm0a9eOsWPHMnnyZMaOHcvMmTPZvHkzInKmOuypp55i3rx5xMfHV6mKrDx23nH0BrYbY3YCiMh0YChwJnEYYw4Dh0Xk6gqOcxmwwxizx8ZYy9QkNAA/H62qUqo2q+jOoKY61/Xu3fusznOvvPIKM2daf+vu27ePbdu2nZM4WrVqRffu3cnKyiI5OZndu3dXep4tW7bQqlUr2rVrB8Btt93GpEmTmDBhAkFBQdx5551cffXVXHPNNQD069ePcePGMWLECG688UYPfVp7E0c8sM9pOQ3oU4XjjAQ+KrVugoiMBVYAvzfGHC9dSETGA+MBYmJiSE1NrcKpoVGAYfWWPaQGH6pSebtlZ2dX+bPVBI2vejS+skVGRpKVlVXpfkVFRS7t567s7GyKi4vJysoiJyeHwMDAM+f54YcfmDdvHt988w0hISEMGTKEY8eOkZWVhTGG7OxssrOz8ff3Jysri6KiIgoLCzl16lS5sRYUFJCbm0t2dvZZnyknJ4fCwkJyc3OZP38+qampzJgxg5dffpnZs2fzr3/9i+XLlzNv3jy6devG4sWLz0lgJfLy8lz+t7QzcZTVo8Stdq0iEgBcBzzutHoy8HfHsf4O/Bv4zTknMuYN4A2AlJQUM2DAAHdOfUb0srkUBkYwYMCFVSpvt9TUVKr62WqCxlc9Gl/ZNm3a5NKdhF13HLGxsZw6dYrw8HBCQkLw8/M7c56CggKio6OJiYlh8+bNLF++nJCQEMLDwxERwsLCAPDx8SE8PJysrCwCAwMpKCgoN1Z/f3+Cg4NJTk5m3759HDp0iDZt2vDZZ59x2WWXISIUFxczbNgwLr30Utq0aUN4eDg7duzg0ksv5dJLL+Wbb77hxIkTlDepXVBQED169HDp89uZONKARKflBCDdzWMMBlYZY878ue/8XkTeBGZXJ8jKNAn2YftxrapSSv2qSZMm9OvXj86dOxMcHExMTMyZbYMGDWLKlCl07dqV9u3b07dvX4+dNygoiHfeeYfhw4efeTh+1113cezYMYYOHUpeXh7GGF588UUAHnnkEbZt24Yxhssuu4xu3bp5JA47E8dyoK2ItMJ6uD0SGOXmMW6hVDWViMQaYw44Fm8A1lc30Io0CRJ+TM+joKgYf19tvayUsnz44Ydlrg8MDGTu3Lllbit5jhEdHc369b9+dT388MMVnuvdd9898/6yyy7jl19+OWt7bGwsy5YtO6fc559/XuFxq8q2xGGMKRSRCcA8rOa4U40xG0TkLsf2KSLSHOs5RQRQLCIPAB2NMSdFJASrRdbvSh36ORHpjlVVtbuM7R7VOFgwBg5m5pHYOMTOUymlVJ1gaz8OR/+KOaXWTXF6fxCrCqussjnAOU9xjDFjPBxmhZoEWXcZ6SdyNXEopWx177338uOPP5617v777+f222/3UkRl0/k4KtEkyHrGr305lFJ2mzRpkrdDcIlW2leicbAjcZxwfax6pZSqzzRxVCLQV2gcGqCdAJVSykEThwviooJI18ShlFKAJg6XxEUGa+JQSikHTRwuiIsKZv/xXJ3QSSlVZSU9xtPT0xk2bFiZ+wwYMIAVK1aUe4ykpCQyMjJsic8dmjhcEB8VzKn8Ik7m6YROSqnqiYuLq1WTMlWFNsd1gfO8HJHB/l6ORil1lrmPwcF1ZW4KLioE3yp8zTXvAoMnVrjLo48+SsuWLbnnnnsAeOKJJxARFi1axPHjxykoKOAf//gHQ4cOPavc7t27ueaaa1i/fj25ubmMGzeObdu20aFDB3JzXa8Sf+GFF5g6dSoAd955Jw888ACnTp1ixIgRpKWlUVRUxF/+8hduvvnmMufqqA5NHC6IiwoCrMTRIVYndFJKwciRI3nggQfOJI5PPvmEr7/+mgcffJCIiAgyMjLo27cv1113HSJljfkKkydPJiQkhLVr17J27Vp69uzp0rlXrlzJO++8w9KlSzHG0KdPHy655BJ27txJXFwcX331FQCZmZkcO3aszLk6qkMThwvine44lFK1TAV3Brk2zsfRo0cPDh8+THp6OkeOHKFRo0bExsby4IMPsmjRInx8fNi/fz+HDh2iefPmZR5j0aJF3HnnnQB07dqVrl27unTuxYsXc8MNNxAaGgrAjTfeyA8//MCgQYN4+OGHefTRR7nmmmvo378/hYWFZc7VUR36jMMF0WGBBPj6sF87ASqlnAwbNoxPP/2Ujz/+mJEjR/LBBx9w5MgRVq5cyerVq4mJiSEvr+LvjfLuRipSXkOddu3asXLlSrp06cLjjz/OU089hZ+fH8uWLeOmm27iiy++YNCgQW6frzRNHC7w8RFitS+HUqqUkSNHMn36dD799FOGDRtGZmYmzZo1w9/fnwULFrBnT8UTl1588cV88sknAKxfv561a9e6dN6LL76YL774gpycHE6dOsXMmTPp378/6enphISEMHr0aB5++GFWrVpFdnY2mZmZDBkyhJdeeonVq1dX92NrVZWrtC+HUqq0Tp06kZWVRXx8PLGxsdx6661ce+21pKSk0L17d84///wKy999992MHj2arl270r17d3r37u3SeXv27Mm4cePO7H/nnXfSo0cP5s2bxyOPPIKPjw/+/v5MnjyZrKysMufqqA5NHC6Kiwrmpx3ebz+tlKpd1q37tUVXdHQ0P/30U5n7ZWdnA1ZfjJK5OIKDg3n33Xddfg7jPC/5Qw89xEMPPXTW9quuuoqrrrrqnHJlzdVRHVpV5aL4qCAOnsyjsKjY26EopZRX6R2Hi+Kigik2cCjr9JlWVkopZYc+ffpw+vTps9a9//77dOnSxUsRnU0Th4ucOwFq4lDK+4wxVWqRVBcsXbq0Rs/n7nBKWlXlojjty6FUrREUFMTRo0d1/DgPMMZw9OhRgoKCXC6jdxwuKuk9rvNyKOV9CQkJpKWlceTIkQr3y8vLc+sL0RtqQ4xBQUEkJJQ5i3eZbE0cIjIIeBnwBd4yxkwstf184B2gJ/AnY8zzTtt2A1lAEVBojElxrG8MfAwkAbuBEcaY43Z+DoCQAD8ahfjrHYdStYC/vz+tWrWqdL/U1FR69OhRAxFVXV2IsTTbqqpExBeYBAwGOgK3iEjHUrsdA+4Dyhtxa6AxpntJ0nB4DJhvjGkLzHcs14i4qGCdQlYp1eDZ+YyjN7DdGLPTGJMPTAfOGibSGHPYGLMcKHDjuEOBaY7304DrPRCrS6zEoXccSqmGTex6uCQiw4BBxpg7HctjgD7GmAll7PsEkF2qqmoXcBwwwOvGmDcc608YY6Kc9jtujGlUxjHHA+MBYmJikqdPn16lz5GdnX1mApb/bjzNj+mFTL48tErHsoNzfLWRxlc9Gl/11Pb4oHbHOHDgwJWlanwAe59xlNVOzp0s1c8Yky4izYBvRWSzMWaRq4UdieYNgJSUFDNgwAA3Tv2r1NRUSspu9dnBd3s307NvPyKCase8HM7x1UYaX/VofNVT2+ODuhFjaXZWVaUBiU7LCUC6q4WNMemOn4eBmVhVXwCHRCQWwPHzsEeidUFJk9wD+pxDKdWA2Zk4lgNtRaSViAQAI4FZrhQUkVARCS95D1wJrHdsngXc5nh/G/ClR6OugPblUEopG6uqjDGFIjIBmIfVHHeqMWaDiNzl2D5FRJoDK4AIoFhEHsBqgRUNzHT0CvUDPjTGfO049ETgExG5A9gLDLfrM5RW0mM8TROHUqoBs7UfhzFmDjCn1LopTu8PYlVhlXYS6FbOMY8Cl3kwTJc1DQvE31f0jkMp1aDpkCNu8PERmkfqhE5KqYZNE4ebdEInpVRDp4nDTfHae1wp1cBp4nBTXFSwTuiklGrQNHG4KS4qmKJiw+Gs05XvrJRS9ZAmDjeVDK+uzzmUUg2VJg43lfTl0Hk5lFINlSYON8We6T2uD8iVUg2TJg43hQX6ERmsEzoppRouTRxVoPNyKKUaMk0cVRAfFaTPOJRSDZYmjirQOw6lVEOmiaMK4qKCOZlXSFaeOzPeKqVU/aCJowrOTOiUqS2rlFINjyaOKoh3dALU5xxKqYZIE0cV6EyASqmGTBNHFTQLD8LXRyd0Uko1TJo4qsDXR2geEaS9x5VSDZImjiqKjwrWZxxKqQbJ1sQhIoNEZIuIbBeRx8rYfr6I/CQip0XkYaf1iSKyQEQ2icgGEbnfadsTIrJfRFY7XkPs/AzliYvSKWSVUg2TbYlDRHyBScBgoCNwi4h0LLXbMeA+4PlS6wuB3xtjOgB9gXtLlX3RGNPd8ZpjzyeoWHyjYA5m5lFUbLxxeqWU8ho77zh6A9uNMTuNMfnAdGCo8w7GmMPGmOVAQan1B4wxqxzvs4BNQLyNsbotLiqYwmLDEZ3QSSnVwPjZeOx4YJ/TchrQx92DiEgS0ANY6rR6goiMBVZg3ZkcL6PceGA8QExMDKmpqe6eGoDs7Owyyx49UgjA7O9/pE0j3yod2xPKi6+20PiqR+OrntoeH9SNGM9hjLHlBQwH3nJaHgO8Ws6+TwAPl7E+DFgJ3Oi0LgbwxbpbehqYWlksycnJpqoWLFhQ5votB0+alo/ONrNW76/ysT2hvPhqC42vejS+6qnt8RlTu2MEVpgyvlPtrKpKAxKdlhOAdFcLi4g/8BnwgTHm85L1xphDxpgiY0wx8CZWlViNi43UKWSVUjYxBvYsgR/+DSdd/tqsMXZWVS0H2opIK2A/MBIY5UpBERHgbWCTMeaFUttijTEHHIs3AOs9F7LrwoP8iQjy08ShlPKc7COw5iNY9R4c3Wat++k1uOlNaH2pd2NzYlviMMYUisgEYB5W1dJUY8wGEbnLsX2KiDTHek4RARSLyANYLbC6YlVtrROR1Y5D/tFYLaieE5HugAF2A7+z6zNUJi4qmP3aCVApVR2mGLZ/ZyWLzXOguAAS+8BFr0FMR5h5N7x/Iwx4DC5+BHy890y1hJ13HDi+6OeUWjfF6f1BrCqs0hYDUs4xx3gyxsr4FJWfGOJ1Xg6lVFVlpsEvH9D357dg4WEIbgy9x0PPsdDs/F/3++18mP0QpD4De3+Gm96C0GjvxY3NiaPO+/avJP8yEwZeUWaWj4sKZsWecxp0KaVU2YoKYOvX1t3F9u/AFJPTqBtB1z4L518NfoHnlgkIhRumQMsLYM4fYEp/GDbVWvYSHXKkInE9CM3ZC2s/KXtzVDCZuQVkny6s4cCUUnXK0R3w7d/ghY7w8Wg4uA76/x7uX8Pabk9B5xvLTholRCB5HNz5nbXfu1fDj69YD9G9QO84KtJhKFlhrQlP/Sd0vgn8As7aHOeYl+PAiVzaxoR7I0KllKfln4JDG+HQOsjYbq3z8QVff/DxO/t1Zp0v+Pg7rXMs5x6HtR/D7h9AfKHdIKsqqs3l4Fvy9bvb9dhiu8LvFsKX98K3f7Gqrq6fBMGNPH0VKqSJoyI+PuxqNZqu656EVdOg92/P2hzvmJdjvyYOpeoeYyDrgPXX/8F1cGi99fPoDqy2N4BfsJUEigutaiZT5P55GiXBZX+FbqMgIrb6cQdFwoj3YekU+ObP8PolMGIaxPWo/rFdpImjEsca94CW/WDhc9B9lFXf6PDrhE7askqpWq2oAI5s+TU5lLxyj/26T1RLaN4Fugy3fsZ0hqgWVjVRCWOguMhq+VSSTIqLrPdn1hU6lgutpNO0A/h4+KmACPS9G+JTYMY4ePtKGPQMpNxxdrw20cRRGRHrr4WpV8GyN+CiB89sahYeqBM6KVVbHd9D+82vwuY/W0mjKN9a7xcEzTpAh2sgpgs07wwxnay/5CsjYlUx+daSr87EXvC7RTBzPHz1e6vq6pqXIDDM1tPWkk9fy7XoC22vgsUvQfLtEBwFgJ+vj2NCJ00cStUq2+fDZ3fQND8XkvpZneead7XuIpq0qT1f/J4Q2gRGzYDF/4YF/4QDa2DEe1ZytIm2qnLVpX+GvBOw5NWzVsdFBemETvWZl1qtqCoyxhqm4783QXgcK5NfgjGfwxVPQZdhVv+I+pQ0Svj4WJ0Dx34JuSfgzUthzXT7Tmfbkeub2K7Q6Ub4eTJkHz6zOi4qmPRMTRz1TmG+1Wb+3+fD/pXejka5Iu+k1dR1vqN5653fkhvigYfRdUmri+GuHyCuJ8z8Hcy6Dwo8//2kicMdA/8EhXnWXzQOcVE6oVO9c/IATLsGlr0ORafh/Rus239Vex3ZYv2VvWUuXPVPuOntsxqyNCjhza07j4seslqDbvvW46fQxOGO6DbQ41ZYMRVO7AWsxFFQZMjI1gmd6oU9S+CNS+Dgehj+LoxfCAHh8N5Qa52qfTb9z0oaucetL8wL7q2RlkW1mq8fXP43uGsxdLzO44fXxOGuSx4FBFKfBSDe0QlQn3PUccbAz1Ng2rUQGG6ND9TpBmjUEsb9z2rP/951cHiTtyNVJYqL4Lsnreqppu2t1kWt+ns7qtqleRdbDquJw12RCdDrTljzIRzZ4tSXQxNHnZWfA5//Fr5+1Go999vvz26R0vg8uO1/Vq/gaddBxjbvxVpXnM6GXz6AWf8HG2Z6vp4955j1AHzxC9DzNrh9LkTWqtml67V62LygBvR31B0ueJq4694GNHHUVUG5B+DtK+DQBrj0L1a9cFmdtaLbWMnj3autu5JxX0GT1jUfcG1mjNWPYPV/Yf1MKDhl3amtes+q7us4FLoOh6T+1Rsa/MAa6y4j6yBc+7I1hpOqUZo4qiI02qpHXfgsERc9SHign/Yer4u2fkPyyt+DfwCM/tQaP6giTdvD2FmO5HEd3P6VNZxEQ3fygDX50C//hWM7ICDMatXUY7TVs3nPYmug0I1fWkklPNZqGttlhFWV4s7ziDXT4X/3Q0gTuP1rSEi273OpcmniqKoLJlg9yef/nbioB/QZR11SXAyLnoPUieSFtcL/jpmuJ4CYjtYD2GnXwrvXwu1zICqx8nL1TWE+bJ1Ll7Uvw8JfrMmIWvazRnztOPTsnsvnDbBeV//bavW09hOrWfuSV63hOLoOt4b5iGpR8fm++ZP1f67lRVbDhbCmNn9IVR5NHFUVFGFVa3z7FwbGDOaHE+dXXkZ5X+5x+Px3sG0edBvFLxHXc7G7dw2xXWHsFzBtqNVs9/a5EBFnR7S1z8F11rOLtR9D7jHCAppY/w+6j6q86s4/2LoT6XwjnDoKG2fC2hlWv4v5T0GLC6HrCCvxhDT+tVzWQWs8pr0/WX+wXf5k/ezEV4fo1a+O3r+Fn1/jlqxpfJz7F29HoypzcL1VN56ZZv31m3IHxQsXVu1YcT2sHsnvXf/rM4/w5h4Nt9bIOQbrP4Nf3reeL/gGWJMO9RjNT/uEAQMvc/+YoU2sRia97oTju2HdDOtOZPYDMOcRaHeVdRcS0hg++y2cPmn1zegyzNOfTlWBrYlDRAYBL2PNOf6WMWZiqe3nA+8APYE/GWOer6ysiDQGPgaSsAayH2GM8c40fP7BcMkfaDn7QXrkLyMn/zJCAjQX10prZ1gtfIKjrOqlxN7VP2ZCivVs5P0brWce42ZDWLPqH7eqCk9bSTFzH5zYZ/3MPsyZIcIRp+cJjp/Oy2VtO5luzVhXlG+N9TT4X9aXd8kdQVpq9eNulGQNl9H/YTiw2vq3Wv8pbJ796/bRn1mDEapawbZvORHxBSYBVwBpwHIRmWWM2ei02zHgPuB6N8o+Bsw3xkwUkcccy4/a9Tkq1WMM2Qte4JHiT0g/fj9tYiK8FooqQ1EBfPMXWDrZqoMf9g6Ex3ju+C36wq0zrKah7w2F22Zbf03bIfeEU1JIg8y9Tu/3QfahUgXEeojs4+sYc8uRQM6Mv1XGcult/iGQ8hvofqtVRWcnEetOLq6HNbbU7kXWXWLPMTU+UZGqmJ1/HvcGthtjdgKIyHRgKHAmcRhjDgOHReRqN8oOBQY49psGpOLNxOHrz6Gev6fDDw+wac2ncOVvvBaKKuXIVqsFzt4l0Pce68vI19/z50nqB6Omw4c3w/tDrZZXznX07irMh/0rYPdiOq/7BjY+biWH0yfP3s830OpXFJUIba+AyBbW+8gEiEyEiPhzZq2sM3z9rBFtW1/q7UhUGcTYNPqniAwDBhlj7nQsjwH6GGMmlLHvE0B2SVVVRWVF5IQxJsqp7HFjzDl/jojIeGA8QExMTPL06VUbKTI7O5uwsIrHts/IKaTTzw/QLLCA9RdMwvjUXHWVK/F5U03H51N0mqZHfiT2wLdEZW6kyCeQLe0ncDjmYtvja3RsFV3WPc2p0Jas6fYUhf6uHVeKCwjP2k7UiXVEnVhPZOYmfIvzMQhZQfHkh8aRF9SU04FNyQtqRl6Q9bPAPwLEu3149fev+mpzjAMHDlxpjEkpvd7Ob7iyGme7mqWqU9ba2Zg3gDcAUlJSzIABA9wpfkZqaiqVlS0oKubuH0fwVsG/uSQyrUY7JLkSnzfVWHwH18HKadYD1tOZ0Lg1XP4kvt1H0TGsGR1rJL4B0LkT4dNv5aLdL8KYmVbru9KKCiB9tTUP9e4frE5zBTnWtmadoNdvIKk/0vJCVi1bq/++1VDb44O6EWNpLiUOEQkFco0xxSLSDjgfmGuMKaigWBrg3MA9AUh3Ma6Kyh4SkVhjzAERiQUOn1O6hvn7+rAh9EJ2+35NUuqz0PVm68G5stfpLKu1z8ppkL7KqrrpeJ01BEXSRd4Z6K7dVdb8z5+MhQ+GWw91/YLg4BrY5ZQo8rOt/Zt2sDrKJV1k9U+w6/mIUh7k6h3HIqC/iDQC5gMrgJuBWysosxxoKyKtgP3ASGCUi+erqOws4DZgouPnly4e01ZxjUJ4r/A2/nr0D7D8bbjwnBo55QnGwP5VsOpdWPeZNaxF0w4waKKVsKvzbMFTzr8ahk2FGbfD5AutviMlzyei20O3kb8mCu3EpuogVxOHGGNyROQO4FVjzHMi8ktFBYwxhSIyAZiH1aR2qjFmg4jc5dg+RUSaYyWhCKBYRB4AOhpjTpZV1nHoicAnjlj2AsPd+sQ2iYsKZn5aO/7a+lJrvo6eY8uuplBVk3vcaqa5ahocWm+19ul0IyTfBgm9at8w2h2HwrC34afXrAe8rfpbicKTLbqU8hKXE4eIXIB1h3GHq2WNMXOAOaXWTXF6fxCrGsqlso71R4Eq9DiyV1xUEPPW51F8y1/weWsg/PwaDHjM22F5T1EhbJhJ+83TIftLq7qm5OXv/D4Y/AKtwfD8As9e9g+ympuueg82fmFNohXbDa5+weocVtsTc6cbrJdS9YyrieMB4HFgpuOu4TxggW1R1UHxUcHkFxWTEdmRZh2ugyX/gV6/bXh11gW51mB3S16BE3tp4h8BWWugIM/64i+u6LFYOQLCrSEtet4Gcd09HrJSyj0uJQ5jzEJgIYCI+AAZxpj77AysromLLJmXI49ml/7Z6vW6+AW46mkvR1ZDck/A8reswetyMqzqo0HPsuRAEAMGOrXFLy6yEkhBHhTmWr2dCxw/C3N/TTAlL/8Q64FzQ50GVKlayNVWVR8CdwFFwEogUkReMMb8y87g6hLnCZ26d2kP3W6BZW9aHc/q8wQzWQetarnlUyE/C9pcARc9CC0vtJ47HEw9e38fXysJaCJQqs5ytfdQR2PMSayhQeYALYAxdgVVF8U3shLH/uOO4dUvedQaanre41Yfg6IqVNHUZkd3WL2yX+piDY/d7iprfuPRn1o9qWvbw2qllMe4+ozDX0T8sRLHf4wxBSJiT5fzOioiyI+wQL9f5+Vo1NJqkrv4RWsCG99Aay6H2G6/vpp1sh4A1yXpq+HHl6zP5OMPPcZYn7Pxed6OTClVQ1xNHK9jjUS7BlgkIi2BkxWWaGBEhLiooLOnkL3sb9BtFBxca436eWCNNf/yyncdhXytua2dk0lM57MnwakNjLE6ri1+EXZ8D4ER0O9+6HO3Ni9VqgFy9eH4K8ArTqv2iMhAe0Kqu+KigknPdEocItC0nfUqmUfAGDixx0oiJa+t82D1ByWFILrtr4kksa81fLc3qn6Ki2HLHOsh//6VENoMLn/CGi01KLLm41FK1QquPhyPBP4GlIwUtxB4Csi0Ka46KS4qmLVplVwSEWt+gUZJVicxsJJJ1gFHIllr/dzzkzW5DUCzjtaXddeba6bvQtZBq0ntqmlwYq8V6zUvWndPda1qTSnlca5WVU0F1gMjHMtjsCZgutGOoOqq+Khgjp3KJze/iOAAX9cLilhTj0bEQfvBv64/lWH9xb/8bZjzMHz7N2tqzV53QPMung2+uBh2fm9Vo22ZC8WF0Opia5rODtfpVJ1KqTNc/TZobYy5yWn5SRFZbUM8dVpclPXXeHpmLq2beuA5RWi0NXRJz7FWVdHyt2HNR7DyHUjoDb3uxKeomhPclL67CGliNSHueRtEt6n+Z1BK1TuuJo5cEbnIGLMYQET6AbmVlGlwfu0E6KHE4Sw+2Xpd+Q8reSx/G2aO5wK/cCi6HVJud71lU8ndxYp3rLsLU+S4u3gCzr/GGvJDKaXK4WriuAt4z/GsA+A41si0yolzJ0DbhDSGC+61WjTtWsiJuc/S9KdJ1hAfbS6HlDusPhU+ZVSVZR2EX963xn4qubu44F69u1BKucXVVlVrgG4iEuFYPukYyXatjbHVOc0jgxCB/Sfy7D+Zjw+0HsiGzsKAnu2sZLDyXZh+C0QkWJNJ9RwLoU2tJrQr9e5CKeUZbj3xdPQeL/EQ8JJHo6nj/H19iAkPsveOoywRcdZIvP1/byWHFW/Dgn/AwolW4sg68OvdRfI4aNK6ZuNTStUr1Wkqo2NKlOGcToA1ydffmgGv43WQsR1WTIXju6DL03p3oZTymOokDh1ypAxxUcGs318LurdEt4FB//R2FEqpeqjCxCEiWZSdIATQSbXLEB8VzDcbD1FcbPDx0ZsypVT9U2HiMMaE11Qg9UVcVDD5hcUcPZVP03CtGlJK1T+uDquuXFQjTXKVUsqLbE0cIjJIRLaIyHYROWcCbrG84ti+VkR6Ota3F5HVTq+S5r+IyBMist9p2xA7P4O7zvQe18ShlKqnbBuASER8gUnAFUAasFxEZhljNjrtNhho63j1ASYDfYwxW4DuTsfZD8x0KveiMeZ5u2KvjnjHHcd+TRxKqXrKzjuO3sB2Y8xOY0w+MB0YWmqfocB7xvIzECUisaX2uQzYYYzZY2OsHhMZ7E9IgC/pNdEJUCmlvMDOIU/jgX1Oy2lYdxWV7RMPHHBaNxL4qFS5CSIyFlgB/N4Yc7z0yUVkPDAeICYmhtTU1Cp8BMjOzna7bJR/MWu27yU19XCVzumOqsRXkzS+6tH4qqe2xwd1I8ZzGGNseQHDgbeclscAr5ba5yvgIqfl+UCy03IAkAHEOK2LAXyx7paeBqZWFktycrKpqgULFrhdZszbS821r/5Q5XO6oyrx1SSNr3o0vuqp7fEZU7tjBFaYMr5T7ayqSgMSnZYTgHQ39xkMrDLGHCpZYYw5ZIwpMsYUA29iVYnVKvHe7D2ulFI2szNxLAfaikgrEQnAqnKaVWqfWcBYR+uqvkCmMca5muoWSlVTlXoGcgPWBFO1SlxkMBnZ+eQVFHk7FKWU8jjbnnEYYwpFZAIwD6tqaaoxZoOI3OXYPgWYAwwBtgM5wO0l5UUkBKtF1u9KHfo5EemO1aN9dxnbva6kL8eBzDxaRYd6ORqllPIsW+cDNcbMwUoOzuumOL03wL3llM0BmpSxfoyHw/Q4506AmjiUUvWN9hy3gfblUErVZ5o4bBATGYiI9h5XStVPmjhsEOjnS9OwQE0cSql6SROHTeKigrX3uFKqXtLEYZP4qGC941BK1UuaOGwSFxXE/hO5Jb3dlVKq3tDEYZO4qGBOOyZ0Ukqp+kQTh010QielVH2licMm8Zo4lFL1lCYOm8Sd6QSoLauUUvWLJg6bNArxJ8jfR+84lFL1jiYOm4iIoy+HJg6lVP2iicNG2pdDKVUf2To6bkMXHxXMkh1HGfzyD8REBBITHmT9jAxyvLeWm4QF4usj3g5XKaVcoonDRuP6JSEiHD6Zx6GsPDaknyQj+zSl+wT6CDQNDyQmIohmjuTSPMJKLM0jg0hqEkpcVBB+vnqDqJTyPk0cNjq/eQTP3NjlrHWFRcVkZOdz6GSe9co6bSWWk3kcOnmatOM5rNp7nGOlOg76+wqJjUJo2SSEpOhQkpqE0rJJCIdziiksKtakopSqMZo4apifrw/NI607iYqcLiziSNZp9h/PZc/RHHYdPcWeo6fYnZHD0l3HyMn/dVraPy7+moRGwWcllJL3CY2C8dekopTyIE0ctVSgny8JjUJIaBRCn/POngjRGMOR7NPszshh3o8rCYxOZM/RHHYfPcXyXcc45ZRU2jYL46v7+hPgp8lDKeUZmjjqIBGhWbj1PCRnjz8DBpx/ZpsxhozsfHYfPcWS7Ud58butzN90iMFdYr0YsVKqPrH1z1ARGSQiW0Rku4g8VsZ2EZFXHNvXikhPp227RWSdiKwWkRVO6xuLyLciss3xs5Gdn6GuERGahgfSK6kxEy5tQ/OIID5Zsc/bYSml6hHbEoeI+AKTgMFAR+AWEelYarfBQFvHazwwudT2gcaY7saYFKd1jwHzjTFtgfmOZVUGXx9hWHICC7ce4WCmDn2ilPIMO+84egPbjTE7jTH5wHRgaKl9hgLvGcvPQJSIVFanMhSY5ng/DbjegzHXO8OSEyg28NmqNG+HopSqJ8SuiYZEZBgwyBhzp2N5DNDHGDPBaZ/ZwERjzGLH8nzgUWPMChHZBRwHDPC6MeYNxz4njDFRTsc4bow5p7pKRMZj3cUQExOTPH369Cp9juzsbMLCwqpUtia4Et8zS3M5cdowsX8wIjXb0bA+XD9v0viqp7bHB7U7xoEDB64sVeMD2PtwvKxvqNJZqqJ9+hlj0kWkGfCtiGw2xixy9eSORPMGQEpKihkwYICrRc+SmppKVcvWBFfiywhP4+EZawhN6kbvVo1rJjCH+nD9vEnjq57aHh/UjRhLs7OqKg1IdFpOANJd3ccYU/LzMDATq+oL4FBJdZbj52GPR17PDOnSnNAAX2boQ3KllAfYmTiWA21FpJWIBAAjgVml9pkFjHW0ruoLZBpjDohIqIiEA4hIKHAlsN6pzG2O97cBX9r4GeqFkAA/ru0Wx1frDpB9utDb4Sil6jjbEocxphCYAMwDNgGfGGM2iMhdInKXY7c5wE5gO/AmcI9jfQywWETWAMuAr4wxXzu2TQSuEJFtwBWOZVWJ4SmJ5OQXMWftAW+HopSq42ztAGiMmYOVHJzXTXF6b4B7yyi3E+hWzjGPApd5NtL6r2eLKFo3DeWTFfsY0Sux8gJKKVUOHYeigRARRqQksmLPcXYcyfZ2OEqpOkwTRwNyQ894fH2EGSu0T4dSquo0cTQgzcKDGNi+GZ+tSqOwqNjb4Sil6ihNHA3MiJQEjmSdZuHWI94ORSlVR2niaGAGnt+M6LAAHfhQKVVlmjgaGH9fH27oEc/8TYc5mn3a2+EopeogTRwN0PCURAqLDTN/2e/tUJRSdZAmjgaoXUw43ROj+GTFPuwa5FIpVX9p4migRqQksvVQNmvTMr0dilKqjtHE0UBd0y2WIH8fWx+SnzpdyPbjRXpXo1Q9o4mjgYoI8mdI51hmrU4nN7/I48cvKjbc9d+V/GNpHkNeWcysNekUFWsCUao+0MTRgA1PSSTrdCHzNhz0+LEnLdjOD9syGJDoR35hEfd99AuX/TuVj5bt5XSh5xOVUqrmaOJowPq0akyLxiEer65asj2DF7/byg094rmtYwDfPngJU0b3JDzIn8c/X8fFzy3gzUU7OaVDvCtVJ2niaMB8fIRhyQks2XGUfcdyPHLMwyfzuG/6as6LDuUf13dGRPDxEQZ1jmXWhH68f0dvzosO4+k5m+j37Pe8+O1Wjp/K98i5lVI1QxNHA3dTcgIiMGNl9Qc+LCwq5v8++oXs0wVMHp1MaODZo/aLCP3bNuWj8X35/J4L6ZXUmJfnb6Pfs9/z99kbOZiZV+0YlFL208TRwMVHBXNRm2g+W5lGcTUfXr/03TaW7jrGP67vQruY8Ar37dmiEW+OTWHeAxdzVafmvLtkN/2f+57HPlvLroxT1YpDKWWxq0WjJg7FiJRE9p/IZcmOo1U+RuqWw/xnwXZGpCQwLDnB5XLtm4fz4s3dSX14ACN7teDzX/Zz2b9TuffDVWxI1z4mSlXVgcxcrnhxEct3H/P4sTVxKK7oGENksH+VH5Knn8jlwY9Xc37zcJ68rnOVjpHYOIS/X9+ZHx+9lN9d0ppFW45w9SuLGf/eCvIKtBWWUu568dut7D2aQ/OIII8f29bEISKDRGSLiGwXkcfK2C4i8opj+1oR6elYnygiC0Rkk4hsEJH7nco8ISL7RWS14zXEzs/QEAT5+3J99zi+3nCQzJwCt8oWOJ5r5BcWM+nWngQH+FYrlqbhgTw66HwWP3YpD17ejm83HeLxz9d5pRPhroxTvL5wh85douqcLQez+HRlGmMuaEli4xCPH9+2xCEivsAkYDDQEbhFRDqW2m0w0NbxGg9MdqwvBH5vjOkA9AXuLVX2RWNMd8frrDnNVdUMT0kkv7CYWWvcG/jw+XlbWLnnOM/c1JXWTcM8Fk9ksD/3X96Why5vx8xf9vP24l0eO7YrDmflMfqtpTwzdzMvz99Wo+dWqrqe/XozoYF+TBjYxpbj23nH0RvYbozZaYzJB6YDQ0vtMxR4z1h+BqJEJNYYc8AYswrAGJMFbALibYy1wescH0nH2Ag+cWNa2e82HuL1RTsZ3bcF13WLsyWuewe2YVCn5vxzziYWb8uw5Ryl5eYX8dtpKzh2Kp8B7ZvynwXba+zcSlXXzzuP8v3mw9w9oDWNQgNsOYediSMecK40T+PcL/9K9xGRJKAHsNRp9QRH1dZUEWnksYgbuOEpCazbn8nG9JOV7rvvWA6/n7GGTnER/Pnq0jeSnuPjI/x7RDfaNgtnwker2HvUM/1NylNcbHjok9Ws3Z/JyyO789qtPWnTNIwHPl7N4SxtLqxqN2MMz8zdTPOIIH7Tr5Vt5xG76o5FZDhwlTHmTsfyGKC3Meb/nPb5CnjGGLPYsTwf+IMxZqVjOQxYCDxtjPncsS4GyAAM8Hcg1hjzmzLOPx6r+ouYmJjk6dOnV+lzZGdnExbmuSoYT/NkfNn5hgcW5DCwhR+3dggsd7/CYsPTS/M4eKqYJy8MpllI+X9/eCq+wznFPPlTLo0ChT/3DSbIT6p9TDg3vk+25DNnVwEj2wcwqJU/APuzrHO3jvLhkV5B+Ihnzl2V+Gobja/6PBnjsoOFvLb6NHd0DqB/gn+1jzdw4MCVxpiUczYYY2x5ARcA85yWHwceL7XP68AtTstbsBIBgD8wD3iognMkAesriyU5OdlU1YIFC6pctiZ4Or57/rvSdH9ynjldUFTuPn/7cr1p+ehsM3ddeqXH82R8i7YeNq0em23uen+FKS4u9sgxneP7cOke0/LR2eZPM9eec/yPl+01LR+dbV7+bqtHzluV+Gojja/6PBXj6YIic8lz35srX1hoCos88/8DWGHK+E61s6pqOdBWRFqJSAAwEphVap9ZwFhH66q+QKYx5oCICPA2sMkY84JzARGJdVq8AVhv30doeIanJHA8p4D5mw6VuX3uugO8u2Q3t/dLYlDn2DL3sUv/tk15fHAH5q4/yKQF2z167B+2HeHPX6znknZNeeLaTkipu4rhKQnc0COel77bys87q97fRSm7fLRsL7uP5vDo4Pb4+th7V2xb4jDGFAITsO4aNgGfGGM2iMhdInKXY7c5wE5gO/AmcI9jfT9gDHBpGc1unxORdSKyFhgIPGjXZ2iI+rdtSvOIoDL7dOzOOMUfPl1Lt8QoHh/cwQvRwZ39W3FDj3j+/e3WcpObu7YeyuKe/66ibbMw/jOqB36+5/63EBH+fn1nkpqEct9Hv+h87apWycor4JX52+jTqjED2zez/Xy29uMwxswxxrQzxrQ2xjztWDfFGDPF8d4YY+51bO9ijFnhWL/YGCPGmK6mVLNbY8wYx75djTHXGWMO2PkZGhpfx8CHC7ceOWvsqLyCIu75YBU+PsKkUT0I8PNO31ER4Zkbu9ApLoL7p69m++Hsah0v87Th9neWExTgy9vjehEeVH69cFigH/8Z1ZMTuQU8+Mmaag/RopSnvLloJ0dP5fP4kA7n3C3bQXuOq3MMS06g2MBnq35tmvv32RvZeOAkL4zoRkIjz3cockeQvy+vj0kh0M+H8e+t4GSee50WS+TmF/HyqjyOnjrN27elEB8VXGmZjnER/PWajizaeoTXF+2s0nmV8qTDJ/N484ddXN01lu6JUTVyTk0c6hxJ0aH0adWYGSv2YYzhy9X7+WDpXn53yXlc1iHG2+EB1uCMk0cns/dYDg9MX+327IIlzW53ZRbz8sgedE2IcrnsrX1acHXXWJ7/ZgsrbBgHSCl3vDR/GwVFxTxyZfsaO6cmDlWmESmJ7D6aw/Tl+3j883WktGzEwzX4i+mK3q0a87frOvH95sO88O0Wt8o+N28Lc9cfZET7AK7q1NytsiXVZfFRwdz30S86n4jymu2Hs/l4+T5u7dOCpOjQGjuvJg5VpsFdmhMW6Mfjn68jyN+XV0f1wL+Mh8beNrpPC0b2SmTSgh18tda1x13Tl+1lysIdjOrTgkFJfpUXKENEkD+TRvXkSPZpHvl0jVfG0lLqua83E+Tnw/9d1rZGz1v7vglUrRAS4Me13eIQgRdv7k5sZOX1/94gIjw5tBM9W0Tx8Iw1bDpQca/3xdsy+PMX67m4XVOeuu7cZrfu6JIQyR+HdOC7TYdrfCwtpVbsPsY3Gw9x1yWtiQ4rv8OuHTRxqHL99ZqO/G/CRVzSrqm3Q6lQoJ8vU0YnExHsx/j3V5RbdbTtUBZ3f7CS1k3DmFROs1t3jbswias6xTBx7mZW7ztR7eMpyC8sZtmuY7z47VYembHGY9Ma1yfGMbRIs/BA7uhv39Ai5dHEocoVHOBL5/hIb4fhkmYRQUwZncyhzNPc++Gqc4ZCP5J1mtvfXU6gny9vj0upsNmtO0SE527qRkxEEBM+XEVmbtVaeDVkRcWGdWmZvL5wB2OnLqPbk98w4vWfePX7bcxak84Nr/2oSbmUeRsOsXLPcR64vB0hAVWrbq0OTRyq3ujRohFP39CZJTuO8szczWfW5xUU8dv3VpCRbTW79XRz4sgQf/4zqgcHM/N49NO1+ryjEsYYth/O5v2fdnPX+yvp+fdvufY/i3lm7mbST+QyIiWB18ck88tfruSr+/oTHODLyDd+Yt6Gg94OvVYoLCrmuXmbad00lBEprs+26Uk1n6qUstHwlEQ2pJ/k7cW76BgbwQ094vn9J2tYk3aCybcm082mdu49WjTi0UHn8/ScTbz30x5uuzDJlvPUVemOqYmXbM/gxx0ZHDpp9byPjwrmyo4x9GsTzYWtm9Cs1Gx1kSH+zLynH3dOW8Fd/13JX67uyG8uqvmqmdrk4xX72HnkFK+PSfZIdWtVaOJQ9c6fru7AloNZPD5zHQu3HuGrdQf445DzGdTZvWa37rrjolb8tPMoT3+1ieSWjepMNZ9dNqRn8tGyvfy4/Si7Mk4B0Dg0gAtaN6Ff62j6tWlCi8YhlTZQiA4L5KPf9uWBj3/hqdkb2Xssh79c09H28Zhqo5z8Ql76bhspLRtxZUfv9anSxKHqHX9fHybd2pNrX13MrDXp3NK7Bb/tf57t5/XxEf49vBtDXvmBez9cxez/u8hjz1LqkuW7jzFpwXZStxwhJMCXvuc14dY+LejXJpr2MeH4VOELPzjAl9duTeaZOZt4a/Eu0o7n8sot3W2r3y8uNny2Ko3/rT3AxW2jGZ6cSGSI9/8t3/phF0eyTjNldM8aGVqkPJo4VL3UODSAab/pzbwNBxl/8Xk19p+sUWgAr9zSg5Fv/Mzjn6/j1Vt6ePU/eE0xxrBw6xFeW7CDZbuP0SQ0gEeuas+YC1oS4aHk6esj/PmajrRoEsITszZw8+s/8/a4FJqFB1Ve2A1Ltmfwj682sfHASZqFB7Jo6xGe/2YL13ePZ8wFLekU5507yYzs07y+cAdXdYohuWVjr8RQQhOHqrfaNAujTTN75lyuSK+kxjx0RTv+NW8LF7aOZlSfFjUeQ00pNoav1h7gtdTtbEg/SVxkEE9c25Gbe7UgOMDXlnOOvSCJ+KhgJnz4CzdMWsI7t/eiXUx4tY+740g2z8zZxHebDhMfFczLI7tzbdc4Nh08yfs/7eGL1fuZvnwfKS0bMfbCJAZ1al6jg32+Mn8beYXF/GHQ+TV2zvJo4lDKBndf0pqfdx7lT1+s4+MV+7i4bTT92zalR4uoWtkD3135hcV8sXo/L/yQy8GcVZwXHcpzw7pyfff4GvkyvaxDDJ/87gJ+M205N01ewuujk7mwTXSVjnXsVD4vf7eVD5buJcjflz8Mas9v+rUiyN9KfJ3iIpl4U1ceH9yBGSv38f7Pe7jvo1+IDgtkVO9ERvVpSfNIz971lLYr4xQfLt3LyF6JtG7q/RkNNXEoZQMfH+E/o3oydfEufth2hEkLtvPq99sJC/TjgtZNziSSlk0qfzhcFacLi9idkcPhrDwSGoWQ0CjYIwkrN7+I6cv38uainaRn5tEi3IdJo3owqHPzGn9Y3SUhki/u7cft7yxj7NRlTLypK8OSXW+eerqwiGlLdvPq99s5dbqQUX1a8MDl7crthR0Z4s+d/c/jN/1asXDbEd7/aQ+vLtjOpFSr+mjsBUn0adXYln/P5+dtIcDPh/svr9mhRcqjiUMpm0QG+/PgFe148Ip2ZOYW8NOODBZty2DR1iN8u9GahCqxcTD92zbl4rbRXNA6mshg954H5OYXseNINtsPZ7PtcBbbDlnv9xzLOWvEYD8foUXjEFpFh3Je01BaRYeded8sPLDSL7vM3AL++/Mepi7exdFT+fROasw/b+yCSd/AwK41OxOks/ioYD69+0Lu+e8qHp6xhr3Hcnjw8rYVfh5jDHPWHWTi15vYdyyXge2b8schHWjrYnWXj48wsH0zBrZvxt6jOfx36R4+Xr6POesO0i4mjLEXJHFDj3hCAz3z9frL3uN8te4A913W1uPPc6pKE4dSNSAy2J9BnWMZ1DkWYwx7jubww7YjLNqWwazV6Xy4dC8+At0To6xE0i6abk5DvWflFTiSg5UYShJF2vFcSvob+vkILZuE0C4mnKu7xtKmWRgxEUGkHc9lV0Y2uzJOsfPIKRZvz+B04a8960MCfGkVHWolkuhQzmtqJZWk6FAKioqZungX7/+0h6zThQxo35R7BrShdyvr4WzqgY01eRnLFBHkzzu39+KPn6/jlfnbSDuWw8SbupZZZbZq73Ge/moTK/cc5/zm4bx/R2/6t636kDotmoTwxyEdePDydvxvTTrTftrNn79Yz7NzN3NTcgKj+rSgTdOwKrUkg1+HFokOC2D8xfa3DHSVJg6lapiIkOT4Yh5zQRIFRcWs3neCH7ZaieTV77fx8vxthAf6ERtSzGNL5nPw5K+zMQb4+nBe01C6JzZiWM9E2saE0bZZGC2bhLr0fKG42HDgZB67jpxiV0Y2OzNOsSvjFOv2ZzJn3QGcpzbxETDAkC6x3H1J61rbN8Xf14fnhnWlZZMQnv9mK+mZubw+OuXM9n3Hcnj2683MXnuApuGBTLyxC8NTEj1WvRYc4MuIXokMT0lg1d4TvP/Tbj5Yuod3l+zG31eIiQgiNjKI5pHBxEYGnXk1jwzmeF4xRcWmzFi+33yYZbuO8dTQToR56A7GE2pPJEo1UP6+PvRKamy1xrqyPSdy8lmy4yg/bDvC8q37ubB1E9rEhNG2WThtmoWR2Ci4Wj2GfXyE+Khg4qOCuajt2Q+U8wuL2Xssh10ZVlLJzC3gxp4JteKBbGVEhAmXtiWhUQh/+HQtN07+kdvaFvHM3E288+NufATuu7QNv7uktceqkcqKIbllI5JbNuJPV3dk3oaD7D+Ry4ETuRzIzGNt2gnmbcgjv/DssdQeWTSXmIggmkdar9iIIGKjgvlo2V5aRYdyS+/a1TLP1sQhIoOAlwFf4C1jzMRS28WxfQiQA4wzxqyqqKyINAY+BpKA3cAIY8xxOz+HUjUpKiSAIV1iGdIlltTUYwwY0L3Gzh3g5+NoxhwG1I7ZHt11fY94mkcG8bv3V/LXJQXATm7sGc8jV7Wv0ekBmoYHMrpvy3PWG2M4nlNA+olcDmbmsWjFWsJjEjlwIo8DmXlsTD/JdxsPnalOnDK6Z61riWdb4hARX2AScAWQBiwXkVnGGOdK0cFAW8erDzAZ6FNJ2ceA+caYiSLymGP5Ubs+h1Kq7ul7XhM+u/tC/jnjRx4c2pcuCbWnik1EaBwaQOPQADrHR+J32J8BA87um2GMITO3gKy8QhIbe3ZQTk+wM431BrYbY3YaY/KB6cDQUvsMBd4zlp+BKBGJraTsUGCa4/004HobP4NSqo5q0yyMsZ0Ca1XScJWIEBUSUCuTBthbVRUP7HNaTsO6q6hsn/hKysYYYw4AGGMOiEizsk4uIuOB8QAxMTGkpqZW6UNkZ2dXuWxN0PiqR+OrHo2v+upCjKXZmTjKaq5QeqKC8vZxpWyFjDFvAG8ApKSkmAEDBrhT/IzU1FSqWrYmaHzVo/FVj8ZXfXUhxtLsrKpKAxKdlhOAdBf3qajsIUd1Fo6fhz0Ys1JKqUrYmTiWA21FpJWIBAAjgVml9pkFjBVLXyDTUQ1VUdlZwG2O97cBX9r4GZRSSpViW1WVMaZQRCYA87Ca1E41xmwQkbsc26cAc7Ca4m7Hao57e0VlHYeeCHwiIncAe4Hhdn0GpZRS57K1H4cxZg5WcnBeN8XpvQHudbWsY/1R4DLPRqqUUspVtatXiVJKqVpPE4dSSim3iDFutXKtk0TkCLCnisWjgQwPhuNpGl/1aHzVo/FVX22OsaUx5pzhgxtE4qgOEVlhjEmpfE/v0PiqR+OrHo2v+upCjKVpVZVSSim3aOJQSinlFk0clXvD2wFUQuOrHo2vejS+6qsLMZ5Fn3EopZRyi95xKKWUcosmDqWUUm7RxOEgIoNEZIuIbHfMLFh6u4jIK47ta0WkZw3GligiC0Rkk4hsEJH7y9hngIhkishqx+uvNRWf4/y7RWSd49wrytjuzevX3um6rBaRkyLyQKl9avT6ichUETksIuud1jUWkW9FZJvjZ6Nyylb4u2pjfP8Skc2Of7+ZIhJVTtkKfxdsjO8JEdnv9G84pJyy3rp+HzvFtltEVpdT1vbrV23GmAb/whpIcQdwHhAArAE6ltpnCDAXa66QvsDSGowvFujpeB8ObC0jvgHAbC9ew91AdAXbvXb9yvi3PojVsclr1w+4GOgJrHda9xzwmOP9Y8Cz5cRf4e+qjfFdCfg53j9bVnyu/C7YGN8TwMMu/Pt75fqV2v5v4K/eun7Vfekdh6U609zazhhzwBizyvE+C9iENUtiXeK161fKZcAOY0xVRxLwCGPMIuBYqdWuTIvsyu+qLfEZY74xxhQ6Fn/GmifHK8q5fq7w2vUrISICjAA+8vR5a4omDkt5U9i6u4/tRCQJ6AEsLWPzBSKyRkTmikinmo0MA3wjIivFmra3tFpx/bDmdinvP6w3rx+UmhYZKGta5NpyHX+DdQdZlsp+F+w0wVGVNrWcqr7acP36A4eMMdvK2e7N6+cSTRyW6kxzW2NEJAz4DHjAGHOy1OZVWNUv3YBXgS9qMjagnzGmJzAYuFdELi61vTZcvwDgOmBGGZu9ff1cVRuu45+AQuCDcnap7HfBLpOB1kB34ABWdVBpXr9+wC1UfLfhrevnMk0clupMc1sjRMQfK2l8YIz5vPR2Y8xJY0y24/0cwF9EomsqPmNMuuPnYWAmVpWAM69eP4fBwCpjzKHSG7x9/RxcmRbZ27+HtwHXALcaR4V8aS78LtjCGHPIGFNkjCkG3iznvN6+fn7AjcDH5e3jrevnDk0clupMc2s7R53o28AmY8wL5ezT3LEfItIb69/2aA3FFyoi4SXvsR6iri+1m9eun5Ny/9Lz5vVz4sq0yK78rtpCRAYBjwLXGWNyytnHld8Fu+JzfmZ2Qznn9dr1c7gc2GyMSStrozevn1u8/XS+trywWv1sxWpx8SfHuruAuxzvBZjk2L4OSKnB2C7Cup1eC6x2vIaUim8CsAGrlcjPwIU1GN95jvOuccRQq66f4/whWIkg0mmd164fVgI7ABRg/RV8B9AEmA9sc/xs7Ng3DphT0e9qDcW3Hev5QMnv4JTS8ZX3u1BD8b3v+N1ai5UMYmvT9XOsf7fkd85p3xq/ftV96ZAjSiml3KJVVUoppdyiiUMppZRbNHEopZRyiyYOpZRSbtHEoZRSyi2aOJTyABEpkrNH4PXYqKsikuQ8yqpS3ubn7QCUqidyjTHdvR2EUjVB7ziUspFjboVnRWSZ49XGsb6liMx3DMg3X0RaONbHOOa6WON4Xeg4lK+IvCnWfCzfiEiw1z6UavA0cSjlGcGlqqpudtp20hjTG/gP8JJj3X+whpnvijVY4CuO9a8AC4012GJPrN7DAG2BScaYTsAJ4CZbP41SFdCe40p5gIhkG2PCyli/G7jUGLPTMVDlQWNMExHJwBoSo8Cx/oAxJlpEjgAJxpjTTsdIAr41xrR1LD8K+Btj/lEDH02pc+gdh1L2M+W8L2+fspx2el+EPp9UXqSJQyn73ez08yfH+yVYI7MC3AosdryfD9wNICK+IhJRU0Eq5Sr9q0UpzwgWkdVOy18bY0qa5AaKyFKsP9Rucay7D5gqIo8AR4DbHevvB94QkTuw7izuxhplValaQ59xKGUjxzOOFGNMhrdjUcpTtKpKKaWUW/SOQymllFv0jkMppZRbNHEopZRyiyYOpZRSbtHEoZRSyi2aOJRSSrnl/wGgHWEPkrfE+gAAAABJRU5ErkJggg==\n",
|
|
"text/plain": [
|
|
"<Figure size 432x288 with 1 Axes>"
|
|
]
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA69UlEQVR4nO3deXxU9dX48c/Jvi8kEHYCgrImQNgEF5BqUevWakWtC27VFrc+9qe2T2tba2uXR6vW6oN1qVaLVh+VKm7VUFRQAVlkFQSEkBAIZF/Idn5/3EkY4iRkkrmZTDjv12teM3Pv9849uUnumfvdrqgqxhhjTEthwQ7AGGNM92QJwhhjjE+WIIwxxvhkCcIYY4xPliCMMcb4FBHsAAIpPT1dMzMzO7RtZWUl8fHxgQ0oACwu/1hc/rG4/NMT41q1alWRqvb2uVJVXXkATwL7gPWtrBfgIWAbsA6Y6LVuDrDFs+7O9u4zJydHOyo3N7fD27rJ4vKPxeUfi8s/PTEuYKW2ck51s4rpac+JvjVnAiM8j+uBRwFEJBx4xLN+NHCJiIx2MU5jjDE+uJYgVHUpcLCNIucBz3iS2MdAioj0A6YA21R1u6rWAgs9ZY0xxnQhURdHUotIJvC6qo71se514D5V/dDz/j3gDiATmKOq13qWXw5MVdX5rezjepwrEDIyMnIWLlzYoVgrKipISEjo0LZusrj8Y3H5x+LyT0+Ma9asWatUdZKvdcFspBYfy7SN5T6p6gJgAcCkSZN05syZHQpmyZIldHRbN1lc/rG4/GNx+edYiyuYCSIPGOT1fiCQD0S1stwYY0wXCuY4iEXAFeKYBpSqagGwAhghIkNFJAqY6ylrjDGmC7l2BSEi/wBmAukikgfcDUQCqOpjwGLgLJyurFXAPM+6ehGZD7wNhANPquoGt+I0xhjjm2sJQlUvOcp6BX7YyrrFOAnEGGMCpr6hkYpD9ZRV11NWU0dZdZ3n2XlfeaiB5NgIeiVEkx4fRVpCNL3io0iNiyQi/NibeKJHjaQ2xhy78kuq+XBbERvzyyit/vrJv6y6jsrahg59tgikxkURQx2DtiwnLSGKtPhoz7OTSNLioxjUK46+STGEhfnqaxN6LEEYY0JSaVUdy7cX8eG2IpZtO8D2okoAEqIjSI2PJCnGeWSmxzmvYz3LYiO83kc4z57XsZHhlFbXcbCylqKKWg5W1nKg8pDn9SE279iDAlv2lnOw8gDFVXVfiys6Ioyh6fFkpsUztHc8Qz3PmWnxpCdEIRI6ycMShDEmoGrqGth9sIqI8DD6JccQExkesM9d9VUxH24r4qNtRazfU0qjQlxUONOGpXHZtCHMGJ7GCRmJnToJpyVEk5YQzYiMr69bsuQAM2ee2Py+vqGRg1VOIikqr2XXwSp2FFWwo6iSrfvKeW9zIXUNh3vpJ0ZHkJke7ySQ9HiGeb1Ojo3scMxusQRhjPFbTV2D52RYyc6iSnYeqPI8V1JQWnNE2dS4SPomx9IvOYa+yTH0T4454n2/5Bjior5+KmpoVDbklzYnhBU7i6mtbyQiTJgwOIWbZ4/gpOHpZA9KITJI7QMR4WH0SYyhT2IM9P36+vqGRvJLatjuSRo7iyrZXlTJ6t3F/GtdPt7jlPskRjO6fxKj+yU1P2emxQe1usoShDHdVOWherbtq6CqtoGxA5JIjOnab5hNVwI7PCf+5iRQVElBWc0RJ7fUuEgy0+M5cVgaQ9LiyUyPo75B2VtWQ0FpNXtLa8gvqWHN7hIOVtZ+bV/JsZFHJIwvvqrh5iXvUFZTD8DIvolcPm0IJw1PZ/LQXiREh8apKyI8jMFpcQxOi2PmCUeu8z6+24sq+aKwnI35ZXy4tYj6RufgxkWFM7JvoidhJDO6fxInZCQSGxWYq7Kjxt8lezHGtKq8po5t+yrYuq+Cbfsq+KKwnK2FFewpqW4uIwLH90lk4pAUJgxKZeKQFIalJwTs22VNXQObCspYv6eUdXmlfL6nlK37KmhoPJwFUuMiGZIWz9RhaWR6kkBmmlO3nhzX/uRVU9dAYZmTMPaWVVNQWsPe0hoKSp1k8nleKdLYyJyxA5gxPJ3px6XTOzE6ID9ndxITGc6IjERGZCQesfxQfQNbCyvYWFDGxvwyNhaU8drqfP7+8S4AwgSG9U444kqj7JA7UyZZgjCmi5TV1LG1sIJt+8r5otCTEArLyfeqkomOCOO43glMykzl0ozBDO+TQHREGGt2l7B6VwlvrCvgH5/uBiApJoLxg1OZMCiFCYOdxNGeE/Wh+ga27C1nXV4p76w/xO/XfsAXheXN31p7xUcxbkAy3xiVwfA+CWSmx5OZFkdKXFRAjkNMZDhD0uIZktb6/QucqSOyA7K/UBMdEc7YAcmMHZDcvExVySuuZoMnYWzML2PVV8UsWutMMhEbAeecoQFvALcEYUw7NTYqlbX1VByqp6KmnnLPs/f78pq65mXe67fvraL4rXeaPysmMozhfRKYOiyNERkJjOiTyIg+CQzqFUe4j6uCmSf0aY5he1Eln+0qZvWuElbvKubh97fS9EX/uN7xTBycyoTBzlVGZlo82/ZV8HnzlUEJW/aWNzecxkfCxMwovj9yGOMGJDNuYAr9k2NCqqfNsUBEGNQrjkG94pgz9nBjR0lVLZsKyln66WpXfmeWIIzxUnGovrkhcWdRJTs8j50HKinx0aXRl/iocBJiIkiIjiAhJpLE6AhGp4VzcvZwRvRxksHA1NgOVQ+FhQnD+yQwvE8C3500qDnmdbtLmpPGvzcV8s9VeV/bNikmgnEDk7nmpGFkDUxm3IBktq39hFmzpvodh+keUuKiOPG4NA7tdudUbgnCHHMO1Tew60AV25tO/p6EsKOokv3lh44o2z85hsz0eM4a14/0hGgSoyOaT/6JMc4jITrycEKIjvB5BbBkyRJmnnqcKz9PQnQE04enM314OuBUR3x1oIrVu4vZUVTFiD4JZA1MZnCvuK99y/zSrhRMGyxBmB4vv6Satzfs5T9f7GfbPqfx17sHTnpCFJlp8cw8vvcRA5uG9Irvst4igSQiTrtBeve7d7IJLZYgTI/05f4K3t6wl7fX72VtXikAw/skkDMklQtzBjLUa4BSUhd3HzUmVFiCMD2CqrIhv4yXt9Zy72f/Yeu+CgCyB6Vwx5yRfHNMBsN6d787gRnTnVmCMCGroVH5bFcxb63fy9sb9pJXXI0A04Yl8b1pQzhjTAb9kmODHaYxIcsShAkptfWNfLz9AG9t2Ms7GwopqjhEVHgYJ41I5+bTRhBbvI1zzpgW7DCN6REsQZhuRVUpq6knv6S6+bGnpIY9ntdfFJZTXlNPXFQ4s0b2Yc6Yvsw8oXfzNBRLlnwZ5J/AmJ7DEoTpUo2NSn5pNfklNZ6Tf3Xzyd951FBxqP6IbaLCw+iXEkP/5Fi+ldWf2SP7cNKI9IDNEmqM8c0ShOkSuw9W8c9Veby0cvcRU0uAM7VD/5QYhqbHM2N4OgNSYunf/IghPT66x9yAxZhQYgnCuOZQfQPvbizkhRW7+XBbEQAnj+jND08bzuBecU4CSI4NybEGxhwLLEGYgNuyt5wXVuzmldV5FFfVMSAllltmj+CiSYMYkGK9iowJFZYgTEBUHKrn9bX5vLByN6t3lRAZLpwxui/fnTyIk4an+5x+whjTvVmCMB2mqny2q4QXV+zmX+vyqaptYESfBP777FFcMGEAaQk9bw5/Y44lriYIEZkDPAiEA39V1ftarE8FngSOA2qAq1V1vWfdLcB1gACPq+qf3IzVtF9xZS1v76zj3s+WsnVfBXFR4ZyT1Z/vTh7ExMEpNlW0MT2EawlCRMKBR4DTgTxghYgsUtWNXsV+AqxR1QtEZKSn/GwRGYuTHKYAtcBbIvKGqm51K17TPos/L+DOl9dRVlPP+EFx3PftcXwru3/I3ALSGNN+bv5XTwG2qep2ABFZCJwHeCeI0cBvAVR1s4hkikgGMAr4WFWrPNv+B7gA+L2L8Zo2VNXW88tFG3lh5W6yBybzncGHuOLcGcEOyxjjIlF1516mInIhMEdVr/W8vxyYqqrzvcr8BohR1R+JyBRgGTAVqAJeA04EqoH3gJWqepOP/VwPXA+QkZGRs3Dhwg7FW1FRQUJC95vMrTvEtbO0gcfWHqKwSjl7WCTnD4+kpqoy6HH50h2Oly8Wl38sLv90Jq5Zs2atUtVJPleqqisP4CKcdoem95cDD7cokwQ8BawBngVWANmeddcAnwFLgceAB462z5ycHO2o3NzcDm/rpmDG1dDQqAv+86UO/8kbOvXef+tH2/Z3i7jaYnH5x+LyT0+MC+fLt89zqptVTHnAIK/3A4F87wKqWgbMAxCnZXOH54GqPgE84Vn3G8/nmS6yr7yG/3pxLR9sLeKM0Rn87jtZpMYH5qb1xpjQ4GaCWAGMEJGhwB5gLnCpdwERSQGqVLUWuBZY6kkaiEgfVd0nIoOBb+NUN5ku8P7mQn78z3VU1tZz7wVjuXTKYOuZZMwxyLUEoar1IjIfeBunm+uTqrpBRG7wrH8MpzH6GRFpwGm8vsbrI14WkTSgDvihqha7Fatx1NQ1cN+bm3l62U5G9k1k4SXTGJGRGOywjDFB4mrfRFVdDCxusewxr9fLgRGtbHuym7GZI31RWM7N/1jN5r3lXD1jKP9vzgk2W6oxxzjrvH6MU1X+/skufv36RhKiI3jqqsnMGtkn2GEZY7oBSxDHsIOVtdzx8jre3VjIKcf35o8XZdEnMSbYYRljuglLEMeoZduKuO3FNRRX1vGzb41m3vRMu+eCMeYIliCOQU99tINfvb6RYenxPHHlZMYOSA52SMaYbsgSxDGkvqGRe17fyN+Wf8U3x2TwwMXjiYuyPwFjjG92djhGVB6q56Z/rOb9zfu47uSh3HnmKLtHgzGmTZYgjgF7S2u4+ukVbCks59fnj+V704YEOyRjTAiwBNHDbcgv5ZqnV1JxqJ4nrpzEzBOsC6sxpn0sQfRg728uZP7zq0mOjeSfN5zIqH5JwQ7JGBNCLEH0UM8s38kvFm1gdP8knrhyMhlJNr7BGOMfSxA9TEOjcu8bm3jyox18Y1QfHrpkgvVUMsZ0iJ05epCq2npu/sca/r2pkHkzMvnvs0dbTyVjTIdZgughCstquOZvK9iYX8YvzhnNVTOGBjskY0yIswTRA2wqKOOap1dQUl3H41dMYvaojGCHZIzpASxBhLglW/Yx//nVxEeH8+L3T7RpM4wxAWMJIoT9/eOvuHvRBo7PSOTJqybRLzk22CEZY3oQSxAhavHnBfz3q+uZdUJvHr50IgnR9qs0xgSWnVVC0KH6Bn775iZG9k3k8SsmEREeFuyQjDE9kJ1ZQtCzy79i98Fq7jprlCUHY4xr7OwSYkqr6nj4/W2cPCKdU4/vHexwjDE9mCWIEPPIkm2U1dRx15mjgh2KMaaHczVBiMgcEdkiIttE5E4f61NF5BURWScin4rIWK91t4nIBhFZLyL/EJFjfjKh3QerePqjnXx7wkBG97eJ94zL6g8FOwITZK4lCBEJBx4BzgRGA5eIyOgWxX4CrFHVLOAK4EHPtgOAm4FJqjoWCAfmuhVrqPjjO1sQgdu/eXywQzE92aEKePfn8JsBsOhmSxTHMDevIKYA21R1u6rWAguB81qUGQ28B6Cqm4FMEWkaBhwBxIpIBBAH5LsYa7e3Lq+E19bkc+3JQ228g3GHKmx4FR6ZAh89CIOnwWd/g6e/BeV7gx2dCQJRVXc+WORCYI6qXut5fzkwVVXne5X5DRCjqj8SkSnAMk+ZVSJyC3AvUA28o6qXtbKf64HrATIyMnIWLlzYoXgrKipISEjo0LZuqqioID4+nvs+rSG/spHfnxJHbETwJ+DrzsfL4mq/prhiq/YwYusCehWvoTxhKFtH3EBZ8kh67/uIkZsfpD4ijg1j7qIs+YQujau76YlxzZo1a5WqTvK5UlVdeQAXAX/1en858HCLMknAU8Aa4FlgBZANpALvA72BSOBV4HtH22dOTo52VG5uboe3dVNubq6+u2GvDrnjdf3bsh3BDqdZdz5e3VF3jes//35L9d+/VP1lmupvBql+/L+q9XVHFir4XPWBcaq/Sldd9UyXxNVdj1dPjAtYqa2cU90cKJcHDPJ6P5AW1USqWgbMAxARAXZ4Ht8Edqjqfs+6/wOmA393Md5uqaFR+e2bmxiWHs8lUwYHOxzTU6jC5jeY8umtcGg/ZM2FM+6BBB+3pO07Fq5fAi/Ng0XzoWAtzPkthEd2ddTuaqiDsnxIGQzSxVfpZQVQWwHpI7p2v0fhZoJYAYwQkaHAHpxG5ku9C4hIClClThvFtcBSVS0TkV3ANBGJw6limg2sdDHWbmtpXj1f7q/lse/lEGmD4kwgHNwOb94BW9+hPn4IXLIYMme0vU1cL7jsZXjvF7DsYdi3ES76GySE8FicumrYswq+WgZffQS7P4W6Khg0DWb//OjHJBCqDsKHD8CnC2DQVLhykfv79INrCUJV60VkPvA2Ti+kJ1V1g4jc4Fn/GDAKeEZEGoCNwDWedZ+IyEvAZ0A9sBpY4Fas3VXFoXpe2VbHpCGpfHOMTeF9TKk/BJv+BauehqIvIGMs9B8P/SdAv/GQPND/b7l11fDhn5wTUngkfPM3rKo+gVPbeyIMj4Azfg19s50riQUzYe7fnZhCQU2ZkwS++gh2LXeSQ0MtIM7xnfA9SOznnKyfPguOmw2zf+bOz3eoHJb/BZb/2XmdPRdOvSPw++kkV+diUtXFwOIWyx7zer0c8HlNpap3A3e7GV93t2DpdspqlZ+cPQrp6kteExwHvoRVT8Ga56HqAKRmwrBZsG+D07Oosd4pF5funLi8k0ZS/9aTxhdvw5v/D4p3wtgLnRN9Uj90yRL/Y8y6yKkKeeF78OQcOOchyL64Qz+uqyoPwK5l8NVyJynsXQfaCGERzvGaegMMmQGDp0Js6uHtpt0IK/4KH9zvJMFR58Ksn0KfkZ2Pqa4aVjwBH97v/H5HneP57O458NUm6+umCstqeHzpdib3DWfi4NSjb2DcU7QVtr4DcWkwZLpTRx1I9bWw+XUnMexYChIOI8+CnHlOcgjzVC3WVUPhBshfDflroGANfPA+aIOzPr7PkQmj/wTnG/Jbd8GWNyD9eLhiEQw7tfMx9x/vtEv88yp45Xrn5PuNXzpXGV1JFWpKoGQ3lO52nvdvdq4Q9m92ykTEwMDJcMqPnd/fwMkQFd/6Z0bGwvSbYOKV8PFfYNmfnd9P1lxiYmZ2LM6GOlj9d/jP76E8H447DU77bxiQ07HP6yKWILqpB979gvrGRi46/pgfQN71VGHv504Vz6ZFh080TZIHOSeaIdOdb6BpwzvWqHlwh1OFtOY5qNwPyYOdk8aEyyGx79fLR8bCwEnOo0ldNexd7ySNgjXO87Z/O9+UARBnu2/8Eqb9ACKi/I+zNfHpcPkr8M5/O1Ulez+HC5+C+LTA7aOxESoKPSf/XYeTQGne4de15UduE53k1OdnXez8jvpPgIho//cdkwQz74TJ18FHD8CnjzOl4UVo/NRJNr5+R77iX/8y5N4LxTucuL69AIae7H88QWAJohv6orCcF1fu5qrpQ+kTty/Y4RwbGhthz0onIWz6l1MVI2FOAph0NZxwlvNNtalB88tcWPeCs218bxh8olN2yHTIGANh4b7301AHWxbDyqdge65ztXD8HJg0z/lW2dp2rYmMhUGTnUeT2konaRSscU6uk6522izcEB4JZ/4O+mbB67fB4zNh7vPQd1z7tq+vhbK8w1cApc7r7J3rYG05lO3xtBN4iUmBlEGQOhSGnuIk7JRBnufBzpVeIKtk49OcKrlpP6Bg4W0MWPU0rH4Opl4PM251GvBbUoUtb8L7v3aqBzPGwaUvwogzur6HVCdYguiGfrt4E/HREdx02nDWruhGCaK+1um9svdzEstqgJnBjqhzGuqdk/2mfzlVCOUFEBYJw2bCST+CkWc735KbDXJOfFO/75wADnzpbP/VMuexydMDJTrJGYXcdIXRbzwx1YXw3q+caoaKQkgaCDN/AhMvd9oOAikq3qlXHzw1sJ/blgmXQe+RTrvEX0+H8x+Bsd9xGmCbq392HZEEKN3tGaHtPVhXILEvYZIEgyfC6PM8J//BTpJLGQTRiV33c3lL6s/W429kwIW/hSX3wUcPOYl++k1Ou0VTXNv/4/yu96x0ri4vfApGn3+4qjCEWILoZpZtKyJ3y37uOnMkqfEBrA7wV0Md7Nvkqe/2VF8Ubmj+NpcDoGudqotAVim4rf6Q8w+86TXYvBiqD0JELAyf7ZyMRpwBsSlH/xwRSB/uPHKudJaV7HbqvpuSxtZ3nOURMUytP+RsM+IMp21hxOn+Xy10dwNznHaJF6+Al652rihqSo8sExZ5+ER/3OzDr5uuApIGQEQ0q5csYebMmcH4KY6u1zCnmmjGrU7VUe698MljMPVG2PkB7PiP8wXg3D9D9iVd3y4TQKEbeQ/U2Kjcu3gTA1JiuXJ6ZtftuKHOqWfPX3M4GexdDw2eSdqik6FfltPro/946DOaXf+6j8Fr/+F8+/7G3U6DXlec8Brqnb7qddVQV+l5rnaqVeqq6b1vJazO8yyvgtqqw+UrCp2qodpy51v+8d90eqgMn912o2V7pXhOclnfdd5X7HcSxq7l7NxbzNDzf+qs78kSM+DKfzldaSsKj6z6SR4ECRkh+U3ap4zRMPc5yFsF798Dub92qhvn/A5yroLI0G8/tATRjby2dg8b8sv408XjiYl06WSr6jSW7foY9nzmJITC9VBf46yPSnSSwJTrPN0oJzh1vS3+qbcfN4/B59wJb9zufFP87Fk4+39gwER3Yv7yfVj+iPN8RJXEkcaAM6LGW0SMU1cfnQhjzneSwrBTO9Zw6Y+E3jD6XBh9Ll8tWcLQnp4cmkREwczu16ffNQNz4IpXoWib03Ad3f3mauooSxDdRE1dA394awtjByRxbnYA66QbGz3d/pYdrisvL3DWRSVAv2yYfO3hbpG9hrX/G16fUXDV6/D5P+Htn8LjpzmNraf9zHfDnb/qapyG4I8fhf2bnG+f029yniNjITIOouKc58hYiIzl0zUbmDL91MPrImJ7zjdW072lDw92BAFnCaKbeOqjneSX1vDH72YTFtaJXg4N9U6f9KZksGsZVBc76xL7eXranAiDpzuNip09eYo4VSrHfxNyfwuf/i9sfM1pmxh/Wcc+v2KfM1BpxRNQVeT0ADn/UafR8yjf+qu2lkPqkA7+MMYYb5YguoGDlbX8JXcbp43sw/Tj0o++gbe6Gsj/7HDD6O5PnUm/wLkaOOHsw332UzPd62IXkwxn3uf0Znnjdmcqhs+ecaqd+mW17zMKNzjTD3z+otMYfvyZcOIPIPPkkOoaaExPYQmiG3jova1U1tZz15ntHMpfWeQMvtn4GuStPNyY3GeMM6fLkOnOFUJSP/eCbk3fcTDvTVi3EN75GSw41RloNOsnvnsHNTY6A7uW/9np/REZBxOvcHqE9MBLdmNCiSWIINtZVMnfP/6KiycPZkRGG/2766qdAVZrX/CMlG1wJhibcp1nPplpgan3D4SwMBh/KZxwJrx/L6x4HDa84kwnnXWxczVQWwVr/+G0LxzY6lR/zb7b6f3RXX4OY45xliCC7PdvbyYqIozbTvcxZ2FjIynF6+DVfzpXC7XlTj/x6Tc5J9qMlrf47mZiU+HsPzqzZL7xX/DK92HV32DQFOdWltXFTuP4tx93BhIFchoIY0ynWYIIos92FbP4873c+o0R9En06jO9bxOsXQif/5PxZXucrqejz3MagzNPCr0BVv3HwzXvwpq/w7t3O2MDRp4NJ/7QmaLC2heM6ZYsQQTR62sLiIkM47qThzlTDnz+klN3v/dzZ46e4bPZOGAuoy+43emyGcrCwpy2hTEXONVLiXZ/C2O6O0sQQVR4oJh5CZ8Q/+LjzsRt2uiMRZjzO6dLZ0Jv9i1ZwuhQTw7eohODN5eOMcYvliCC4VA5rHiCe756gF5aAkWDncnhsi6G3scHOzpjjAEsQXSt6mL4ZIFzE5KaEjaTxebjfsXVl11uo32NMd2OJYiuULEfPn4EPv2r0xPphLOomHorly4o4q7BARjNbIwxLrAE4abSPbDsYeeuYfU1TgPtyf8Ffceyu6AM+ICBqT2ofcEY06NYgnDDwR3w0Z+cG883NjhtCyf/yLnRu0decTUAA1NjgxSkMca0zRJEIO3fAh/c78xuGhbuDBCbcYszB1ILecVVAAywBGGM6aZcTRAiMgd4EAgH/qqq97VYnwo8CRwH1ABXq+p6ETkBeMGr6DDg56r6Jzfj7bCCdfDB/zijnSNinBvrTJ/f5q0k84qriYkMIy2Yd40zxpg2uJYgRCQceAQ4HcgDVojIIlX1vp3LT4A1qnqBiIz0lJ+tqluA8V6fswd4xa1YO6yxAV6/1Zm1NCrRqUaa9oMW9zH2bU9xNQNT4xAbRWyM6abcvIKYAmxT1e0AIrIQOI8j7/c1GvgtgKpuFpFMEclQ1UKvMrOBL1X1Kxdj9V9jI/zrFlj9LJw4H0653Zl7qJ3ySqqs/cEY062Jauu3b+zUB4tcCMxR1Ws97y8HpqrqfK8yvwFiVPVHIjIFWOYps8qrzJPAZ6r651b2cz1wPUBGRkbOwoULOxRvRUUFCQntvFWgKsO3Pc7APW+wc8h32Tn0Mr/398P3KpnSN4Irx7R9Axy/4upCFpd/LC7/WFz+6Uxcs2bNWqWqk3yuVFVXHsBFOO0OTe8vBx5uUSYJeApYAzwLrACyvdZHAUVARnv2mZOTox2Vm5vb/sLv/kL17iTVN+9SbWz0e1/lNXU65I7X9S+52wIbVxeyuPxjcfnH4vJPZ+ICVmor51Q3q5jyAO+7tA8E8r0LqGoZMA9AnMr4HZ5HkzNxrh68q5yCa+kf4MP7nfsWfPPeDs1Euse6uBpjQoCbQ3hXACNEZKiIRAFzgUXeBUQkxbMO4FpgqSdpNLkE+IeLMfpn+V/g/V874xrOfqDD01Q3dXG1BGGM6c5cu4JQ1XoRmQ+8jdPN9UlV3SAiN3jWPwaMAp4RkQacxutrmrYXkTicHlDfdytGv6x8Ct6+C0adC+f9pVPTYzQNkrMxEMaY7szVcRCquhhY3GLZY16vlwM+bqUGqloFpLkZX7utfQFevw2Gnw7feQLCO3fY8oqriI4Io3dC2w3UxhgTTK1+DRaR3zd922+x/DYR+Z27YXUjGxfBqzc6d3K7+NmA3BZzT0k1A1JjbQyEMaZba6ue5FvAAh/LHwTOdiecbmbru/DS1TAgBy5ZCJGBqRLK8wySM8aY7qytBKGq2uhjYSPQ87/67vgAXvgeZIyGy/4J0YHr+5xXXM2AFGt/MMZ0b20liCoR+Vr7gGdZtXshdQO7P4XnL4bUofC9VyA2JWAfXXmonoOVtdaDyRjT7bXV2vpz4E0R+TXQNLJ5EnAXcKvLcQVPwVr4+4WQmAFXvArxgW0n31NiYyCMMaGh1QShqm+KyPnAj4GbPIvXA99R1c+7ILaut28zPHsBxCTBFYsgsW/Ad3F4kJy1QRhjurdWE4SIxACFqnpli+V9RCRGVWtcj64LxVYVwDPfh7BIuOI1SBl09I06wAbJGWNCRVttEA8BJ/tYfjrwgDvhBEnJbrLX/gwa65zkkHaca7vKK64mKtzGQBhjur+2EsRJqvp/LReq6nPAKe6F1MWqDsIz5xJRXwWXvwJ9Rrq6u7xiZwxEWFjP7whmjAltbTVSt3UGc3MOp64VkwKjzmFdVX8m9st2fXd5JdVWvWSMCQltnej3ee7RcATPsv3uhdTFwsLg9F9RluzulUOTPcVVNgbCGBMS2rqC+DHwoog8zZHdXK/AmZnV+Km6toGiChsDYYwJDa1eQajqp8BUnKqmq4Cm3kxX4iQJ46c9JU09mKyLqzGm+2tzWlLPjXruFpEJOPdmuBKngfrlLoitx8mzGwUZY0JIW+MgjsepSroEOAC8gHMP61ldFFuPk2eD5IwxIaStK4jNwAfAOaq6DZypvrskqh4qr7iayHChT6KNgTDGdH9t9WL6DrAXyBWRx0VkNsfCLK4uyiuuon+KjYEwxoSGthqpX1HVi4GRwBLgNiBDRB4VkTO6KL4eZY+NgTDGhJCjDnhT1UpVfU5VvwUMBNYAd7odWE+UV1zNwBRrfzDGhAa/RkSr6kFV/V9VPc2tgHqqmroG9pcfYoBdQRhjQkTPmTKjm7P7QBhjQo0liC5i94EwxoQaVxOEiMwRkS0isk1EvtZuISKpIvKKiKwTkU9FZKzXuhQReUlENovIJhE50c1Y3WaD5Iwxoca1BCEi4cAjwJnAaOASERndothPgDWqmoUzfceDXuseBN5S1ZFANrDJrVi7Ql5xFRFhQkZSTLBDMcaYdnHzCmIKsE1Vt6tqLbAQOK9FmdHAewCquhnIFJEMEUnCmdLjCc+6WlUtcTFW1+UVV9MvJYZwGwNhjAkRoqrufLDIhcAcVb3W8/5yYKqqzvcq8xsgRlV/5JlGfBnOBIENwAJgI87VwyrgFlWt9LGf64HrATIyMnIWLlzYoXgrKipISEjo0Lbt8euPq4kMgzum+FfF5HZcHWVx+cfi8o/F5Z/OxDVr1qxVqjrJ50pVdeUBXAT81ev95cDDLcokAU/hjK14FliBkxAmAfU4CQWc6qZ7jrbPnJwc7ajc3NwOb9seU+59V29/cY3f27kdV0dZXP6xuPxjcfmnM3EBK7WVc2qbs7l2Uh4wyOv9QCDfu4CqlgHzAEREgB2eRxyQp6qfeIq+RAgPzjtU30BhmY2BMMaEFjfbIFYAI0RkqIhE4cwMu8i7gKenUpTn7bXAUlUtU9W9wG4ROcGzbjZOdVNIyi+pAayLqzEmtLh2BaGq9SIyH3gbCAeeVNUNInKDZ/1jwCjgGRFpwEkA13h9xE3Ac54Esh3PlUYo2mNdXI0xIcjNKiZUdTGwuMWyx7xeLwdGtLLtGpy2iJCXV9x0JzlLEMaY0GEjqbtAXnE14WFCXxsDYYwJIZYgukBecRV9k2KICLfDbYwJHXbG6gJ2HwhjTCiyBNEF8oqrrQeTMSbkWIJwWW19I3vLamwMhDEm5FiCcFlBaTWq1oPJGBN6LEG4zMZAGGNClSUIlzXdB2KQtUEYY0KMJQiX5RVXESbQN9nGQBhjQoslCJflFVfTNymGSBsDYYwJMXbWclleiXVxNcaEJksQLttTbIPkjDGhyRKEi+oaGikotQRhjAlNliBctLe0hkbFBskZY0KSJQgX5TWPgbA2CGNM6LEE4SK7D4QxJpRZgnBRXnE1ItAv2RKEMSb0WIJwUV5xNRmJMURF2GE2xoQeO3O5aE9JlVUvGWNCliUIF+XZGAhjTAizBOGS+oZGCkprrAeTMSZkuZogRGSOiGwRkW0icqeP9aki8oqIrBORT0VkrNe6nSLyuYisEZGVbsbphr1lNTQ0qo2BMMaErAi3PlhEwoFHgNOBPGCFiCxS1Y1exX4CrFHVC0RkpKf8bK/1s1S1yK0Y3ZRn94EwxoQ4N68gpgDbVHW7qtYCC4HzWpQZDbwHoKqbgUwRyXAxpi6zxwbJGWNCnKiqOx8sciEwR1Wv9by/HJiqqvO9yvwGiFHVH4nIFGCZp8wqEdkBFAMK/K+qLmhlP9cD1wNkZGTkLFy4sEPxVlRUkJCQ0KFtfXltWy2vbKvj8TPiiAyTDn9OoOMKFIvLPxaXfywu/3QmrlmzZq1S1Uk+V6qqKw/gIuCvXu8vBx5uUSYJeApYAzwLrACyPev6e577AGuBU462z5ycHO2o3NzcDm/ry+0vrtHJv363058T6LgCxeLyj8XlH4vLP52JC1iprZxTXWuDwGl3GOT1fiCQ711AVcuAeQAiIsAOzwNVzfc87xORV3CqrJa6GG9AWRdXY0yoc7MNYgUwQkSGikgUMBdY5F1ARFI86wCuBZaqapmIxItIoqdMPHAGsN7FWANuj90oyBgT4ly7glDVehGZD7wNhANPquoGEbnBs/4xYBTwjIg0ABuBazybZwCvOBcVRADPq+pbbsUaaA2NSn5JNd/K6hfsUIwxpsPcrGJCVRcDi1sse8zr9XJghI/ttgPZbsbmpsKyGuptDIQxJsTZSGoX2H0gjDE9gSUIF+wpsftAGGNCnyUIF+QddK4gBqRYgjDGhC5LEC7IK64mPSGamMjwYIdijDEdZgnCBXl2HwhjTA9gCcIFe2yQnDGmB7AEEWCNjWqD5IwxPYIliADbV36IugYbA2GMCX2WIAIsr9i6uBpjegZLEAG2p8Tp4jrIEoQxJsRZggiwplHUA1KsDcIYE9osQQRYXnEV6QlRxEbZGAhjTGizBBFgecXVNoLaGNMjWIIIMGcMhFUvGWNCnyWIAGpsVPJKbJCcMaZncPV+EMeaoopD1NY3WoIwJsDq6urIy8ujpqYmqHEkJyezadOmoMbgS3viiomJYeDAgURGRrb7cy1BBNDuph5MliCMCai8vDwSExPJzMzEc6fJoCgvLycxMTFo+2/N0eJSVQ4cOEBeXh5Dhw5t9+daFVMANY2BsDYIYwKrpqaGtLS0oCaHUCYipKWl+X0FZgkigJpGUVsvJmMCz5JD53Tk+FmCCKC84mp6xUcRH201d8aY0GcJIoBsDIQxpiexBBFAe4rtRkHG9EQlJSX85S9/8Xu7s846i5KSksAH1EVcrQsRkTnAg0A48FdVva/F+lTgSeA4oAa4WlXXe60PB1YCe1T1W27G2lmqSl5xNaeN7BPsUIzp0X75rw1szC8L6GeO7p/E3eeMaXV9U4K4/PLLj1je0NBAeHjr0+osXrw4YDEGg2tXEJ6T+yPAmcBo4BIRGd2i2E+ANaqaBVyBk0y83QJ0v07HPhRV1HKovtF6MBnTA9155518+eWXzJgxg8mTJzNr1iwuvfRSxo0bB8D5559PTk4OY8aMYcGCBc3bZWZmUlRUxM6dOxk1ahTXXXcdY8aM4YwzzqC6urrV/T3++ONMnjyZ7OxsvvOd71BV5XSAKSws5IILLiA7O5vs7GyWLVsGwPPPP09WVhbZ2dlfS2KdoqquPIATgbe93t8F3NWizBvASV7vvwQyPK8HAu8BpwGvt2efOTk52lG5ubkd3lZV9bOvDuqQO17Xdzfs7dTntNTZuNxicfnH4vJPy7g2btwYnEA8duzYoWPGjNGysjLNzc3VuLg43b59e/P6AwcOqKpqVVWVjhkzRouKilRVdciQIbp//37dsWOHhoeH6+rVq1VV9aKLLtJnn3221f01ba+q+tOf/lQfeughVVX97ne/qw888ICqqtbX12tJSYmuX79ehw8frvv37z8iFl98HUdgpbZyTnWzimkAsNvrfR4wtUWZtcC3gQ9FZAowxJMYCoE/Af8PaHNUiohcD1wPkJGRwZIlSzoUbEVFRYe3Bfi0oB6Agi83sGRf4C56OhuXWywu/1hc/mkZV3JyMuXl5UGNp7GxkYaGBqqqqsjJySE9Pb05pj/84Q+8/vrrAOzevZs1a9YwZcoUVJWKigoqKioYMmQIxx13HOXl5YwdO5YtW7a0+jN9+umn3HPPPZSWllJZWcns2bMpLy/nvffe45FHHmneLiwsjMWLF3PuuecSHR1NeXk5kZGRrX5uTU2NX79vNxOEr0632uL9fcCDIrIG+BxYDdSLyLeAfaq6SkRmtrUTVV0ALACYNGmSzpzZZvFWLVmyhI5uC7D5P1/C2s2cf/rJJMa0fyi723G5xeLyj8Xln5Zxbdq0KagjmBMSEggLCyM8PJy4uDiSkpKa41myZAkffPABn3zyCXFxccycOZPw8HASExMRERISEgCIjY1t3iYuLo6KiopWf6Yf/OAHvPrqq2RnZ/P000+zZMmS5s9LTEwkOjq6uWx0dDRhYWHtOj4xMTFMmDCh3T+3m72Y8oBBXu8HAvneBVS1TFXnqep4nDaI3sAOYAZwrojsBBYCp4nI312MtdPyiqtIiYsMaHIwxnQPiYmJrX4rLy0tJTU1lbi4ODZv3szHH3/c6f2Vl5fTr18/6urqeO6555qXz549m0cffRRwGsjLysqYPXs2r7zyCgcOHADg4MGDnd5/EzcTxApghIgMFZEoYC6wyLuAiKR41gFcCyz1JI27VHWgqmZ6tntfVb/nYqydZmMgjOm50tLSmDFjBlOnTuXHP/7xEevmzJlDfX09WVlZ/OxnP2PatGmd3t8999zD1KlTOf300xk5cmTz8gcffJDc3FzGjRtHTk4OGzZsYMyYMdx+++2ceuqpZGdn86Mf/ajT+2/iWhWTqtaLyHzgbZxurk+q6gYRucGz/jFgFPCMiDQAG4Fr3IrHbXuKqxnWOz7YYRhjXPL888/7nBQvOjqaN9980+c2O3fuBCA9PZ3165t78HP77be3ua8bb7yRG2+88WvLMzIyeO211762/LLLLuOGG2442o/gN1fHQajqYmBxi2WPeb1eDow4ymcsAZa4EF7AqGcMxCnH9w52KMYYEzA2aVAAHKyspbquwUZRG2P88sMf/pCPPvroiGW33HIL8+bNC1JER7IEEQB5TfeBsDYIY4wfHnnkkWCH0CabiykA7D4QxpieyBJEADTfB8KqmIwxPYgliADIK64mKSaC5FgbA2GM6TksQQRAXnE1A6x6yRjTw1iCCIA9xdXWg8kY06xpeo38/HwuvPBCn2VmzpzJypUruzIsv1kvpk5yxkBUMX14WrBDMebY8OadsPfzwH5m33Fw5n1HL+en/v3789JLLwX8c7uKXUF0UklVHZW1DdaDyZge7I477jjijnK/+MUv+OUvf8ns2bOZOHEi48aN8znCeefOnYwdOxaA6upq5s6dS1ZWFhdffHGb94MAZzT1pEmTGDNmDHfffXfz8hUrVjB9+nSys7OZMmUK5eXlNDQ0cPvttzNu3DiysrJ4+OGHA/Jz2xVEJzWNgbAqJmO6iAvf9I9m7ty53Hrrrc0343nxxRd56623uO2220hKSqKoqIhp06Zx7rnnIuJrImt49NFHiYuLY926daxbt46JEye2uc97772XXr160dDQwOzZs1m3bh0jR47k4osv5oUXXmDy5MmUlZURGxvLww8/zI4dO1i9ejUREREBm7DPEkQn7SnxdHG1QXLG9FgTJkxg3759FBQUsH37dlJTU+nXrx+33XYbS5cuJSwsjD179lBYWEjfvn19fsbSpUu5+eabAcjKyiIrK6vNfb744ossWLCA+vp6CgoK2LhxIyJCv379mDx5MgBJSUmAM+X4/PnziYhwTum9evUKyM9tCaKTmq4gBlkVkzE92oUXXsirr75KSUkJc+fO5bnnnmP//v2sWrWKyMhIMjMzqampafMzWru6aGnHjh388Y9/ZMWKFaSmpnLVVVdRU1ODqvr8jNaWd5a1QXRSXnE1idERJMVarjWmJ5s7dy4vv/wyL730EhdeeCGlpaX06dOHyMhIcnNz+eqrr9rc/pRTTmm+t8P69etZt25dq2XLysqIj48nOTmZwsLC5tliR44cSX5+PitWrACc+0bU19dz2mmn8dhjj1Ff79zZ0qqYuom84ioGpMa6kr2NMd3HmDFjqKioYMCAAfTr14/LLruMc845h0mTJjF+/Pgj7tvgy4033si8efPIyspi/PjxTJkypdWy2dnZTJgwgTFjxjBs2DBmzJgBQFRUFC+88AI33XQT1dXVxMbG8u9//5srr7ySXbt2kZWVRWRkJNdddx3z58/v9M9sCaKTxg1IYXS/pGCHYYzpAh9//HHz/SDS09NZvny5z3IVFRUAZGZmNt8HIjY2loULF7Z7X08//bTP5ZMnT/7aXevKy8u5//77uf/++9v9+e1hCaKTbvlGm7ezMMaYkGUJwhhjgmjq1KkcOnToiGXPPvss48aNC1JEh1mCMMaEBLd66gTbJ5980iX7UVW/t7FeTMaYbi8mJoYDBw506CRnnORw4MABYmJi/NrOriCMMd3ewIEDycvLY//+/UGNo6amxu+TbFdoT1wxMTEMHDjQr8+1BGGM6fYiIyMZOnRosMNgyZIlTJgwIdhhfI1bcVkVkzHGGJ8sQRhjjPHJEoQxxhifpCf1ChCR/UDbE6K0Lh0oCmA4gWJx+cfi8o/F5Z+eGNcQVe3ta0WPShCdISIrVXVSsONoyeLyj8XlH4vLP8daXFbFZIwxxidLEMYYY3yyBHHYgmAH0AqLyz8Wl38sLv8cU3FZG4Qxxhif7ArCGGOMT5YgjDHG+HRMJQgRmSMiW0Rkm4jc6WO9iMhDnvXrRGRiF8U1SERyRWSTiGwQkVt8lJkpIqUissbz+HkXxbZTRD737HOlj/VdfsxE5ASv47BGRMpE5NYWZbrkeInIkyKyT0TWey3rJSLvishWz3NqK9u2+ffoQlx/EJHNnt/TKyKS0sq2bf7OXYjrFyKyx+t3dVYr23b18XrBK6adIrKmlW3dPF4+zw1d9jemqsfEAwgHvgSGAVHAWmB0izJnAW8CAkwDPumi2PoBEz2vE4EvfMQ2E3g9CMdtJ5DexvqgHLMWv9e9OIN9uvx4AacAE4H1Xst+D9zpeX0n8LuO/D26ENcZQITn9e98xdWe37kLcf0CuL0dv+cuPV4t1v8P8PMgHC+f54au+hs7lq4gpgDbVHW7qtYCC4HzWpQ5D3hGHR8DKSLSz+3AVLVAVT/zvC4HNgED3N5vgATlmHmZDXypqh0dQd8pqroUONhi8XnA3zyv/wac72PT9vw9BjQuVX1HVes9bz8G/Jv72aW42qnLj1cTce5S9F3gH4HaX3u1cW7okr+xYylBDAB2e73P4+sn4faUcZWIZAITAF+3mTpRRNaKyJsiMqaLQlLgHRFZJSLX+1gf7GM2l9b/cYNxvAAyVLUAnH9woI+PMsE+blfjXPn5crTfuRvme6q+nmyluiSYx+tkoFBVt7ayvkuOV4tzQ5f8jR1LCcLXvQpb9vFtTxnXiEgC8DJwq6qWtVj9GU41SjbwMPBqF4U1Q1UnAmcCPxSRU1qsD9oxE5Eo4Fzgnz5WB+t4tVcwj9tPgXrguVaKHO13HmiPAscB44ECnOqcloL5v3kJbV89uH68jnJuaHUzH8v8OmbHUoLIAwZ5vR8I5HegjCtEJBLnD+A5Vf2/lutVtUxVKzyvFwORIpLudlyqmu953ge8gnPZ6i1oxwznH/IzVS1suSJYx8ujsKmazfO8z0eZoBw3EbkS+BZwmXoqqltqx+88oFS1UFUbVLUReLyV/QXreEUA3wZeaK2M28erlXNDl/yNHUsJYgUwQkSGer55zgUWtSizCLjC0zNnGlDadBnnJk8d5xPAJlW9v5UyfT3lEJEpOL+7Ay7HFS8iiU2vcRo517coFpRj5tHqN7tgHC8vi4ArPa+vBF7zUaY9f48BJSJzgDuAc1W1qpUy7fmdBzou7zarC1rZX5cfL49vAJtVNc/XSrePVxvnhq75G3Oj5b27PnB63HyB07L/U8+yG4AbPK8FeMSz/nNgUhfFdRLOpd86YI3ncVaL2OYDG3B6InwMTO+CuIZ59rfWs+/udMzicE74yV7Luvx44SSoAqAO5xvbNUAa8B6w1fPcy1O2P7C4rb9Hl+PahlMn3fQ39ljLuFr7nbsc17Oev511OCewft3heHmWP930N+VVtiuPV2vnhi75G7OpNowxxvh0LFUxGWOM8YMlCGOMMT5ZgjDGGOOTJQhjjDE+WYIwxhjjkyUIY/wgIg1y5EyyAZtVVEQyvWcTNSbYIoIdgDEhplpVxwc7CGO6gl1BGBMAnnsC/E5EPvU8hnuWDxGR9zwT0b0nIoM9yzPEuSfDWs9juuejwkXkcc/c/++ISGzQfihzzLMEYYx/YltUMV3sta5MVacAfwb+5Fn2Z5zp0LNwJsd7yLP8IeA/6kwmOBFnFC7ACOARVR0DlADfcfWnMaYNNpLaGD+ISIWqJvhYvhM4TVW3eyZX26uqaSJShDN1RJ1neYGqpovIfmCgqh7y+oxM4F1VHeF5fwcQqaq/7oIfzZivsSsIYwJHW3ndWhlfDnm9bsDaCU0QWYIwJnAu9npe7nm9DGcWTYDLgA89r98DbgQQkXARSeqqII1pL/t2Yox/YuXIm9e/papNXV2jReQTnC9el3iW3Qw8KSI/BvYD8zzLbwEWiMg1OFcKN+LMJmpMt2FtEMYEgKcNYpKqFgU7FmMCxaqYjDHG+GRXEMYYY3yyKwhjjDE+WYIwxhjjkyUIY4wxPlmCMMYY45MlCGOMMT79f3LITJdi//QQAAAAAElFTkSuQmCC\n",
|
|
"text/plain": [
|
|
"<Figure size 432x288 with 1 Axes>"
|
|
]
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"\n",
|
|
"metrics = pd.read_csv(f\"{trainer.logger.log_dir}/metrics.csv\")\n",
|
|
"\n",
|
|
"aggreg_metrics = []\n",
|
|
"agg_col = \"epoch\"\n",
|
|
"for i, dfg in metrics.groupby(agg_col):\n",
|
|
" agg = dict(dfg.mean())\n",
|
|
" agg[agg_col] = i\n",
|
|
" aggreg_metrics.append(agg)\n",
|
|
"\n",
|
|
"df_metrics = pd.DataFrame(aggreg_metrics)\n",
|
|
"df_metrics[[\"train_loss\", \"valid_loss\"]].plot(\n",
|
|
" grid=True, legend=True, xlabel='Epoch', ylabel='Loss')\n",
|
|
"df_metrics[[\"train_acc\", \"valid_acc\"]].plot(\n",
|
|
" grid=True, legend=True, xlabel='Epoch', ylabel='ACC')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "c4541cce",
|
|
"metadata": {},
|
|
"source": [
|
|
"- The `trainer` automatically saves the model with the best validation accuracy automatically for us, we which we can load from the checkpoint via the `ckpt_path='best'` argument; below we use the `trainer` instance to evaluate the best model on the test set:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 14,
|
|
"id": "ebca4d75",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Restoring states from the checkpoint path at logs/my-mlp/version_16/checkpoints/epoch=13-step=2995.ckpt\n",
|
|
"LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
|
|
"Loaded model weights from checkpoint at logs/my-mlp/version_16/checkpoints/epoch=13-step=2995.ckpt\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "e9003a604b2f4c82973189b372e67313",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Testing: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
|
|
"┃<span style=\"font-weight: bold\"> Test metric </span>┃<span style=\"font-weight: bold\"> DataLoader 0 </span>┃\n",
|
|
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
|
|
"│<span style=\"color: #008080; text-decoration-color: #008080\"> test_acc </span>│<span style=\"color: #800080; text-decoration-color: #800080\"> 0.9812999963760376 </span>│\n",
|
|
"└───────────────────────────┴───────────────────────────┘\n",
|
|
"</pre>\n"
|
|
],
|
|
"text/plain": [
|
|
"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
|
|
"┃\u001b[1m \u001b[0m\u001b[1m Test metric \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m DataLoader 0 \u001b[0m\u001b[1m \u001b[0m┃\n",
|
|
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
|
|
"│\u001b[36m \u001b[0m\u001b[36m test_acc \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.9812999963760376 \u001b[0m\u001b[35m \u001b[0m│\n",
|
|
"└───────────────────────────┴───────────────────────────┘\n"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"[{'test_acc': 0.9812999963760376}]"
|
|
]
|
|
},
|
|
"execution_count": 14,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"trainer.test(model=lightning_model, datamodule=data_module, ckpt_path='best')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "28072fc9",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Predicting labels of new data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b556f70a",
|
|
"metadata": {},
|
|
"source": [
|
|
"- You can use the `trainer.predict` method on a new `DataLoader` or `DataModule` to apply the model to new data.\n",
|
|
"- Alternatively, you can also manually load the best model from a checkpoint as shown below:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 15,
|
|
"id": "7aef985a",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"logs/my-mlp/version_16/checkpoints/epoch=13-step=2995.ckpt\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"path = trainer.checkpoint_callback.best_model_path\n",
|
|
"print(path)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 16,
|
|
"id": "51181422",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"lightning_model = LightningMLP.load_from_checkpoint(path, model=pytorch_model)\n",
|
|
"lightning_model.eval();"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "1ec1e35f",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Note that our PyTorch model, which is passed to the Lightning model requires input arguments. However, this is automatically being taken care of since we used `self.save_hyperparameters()` in our PyTorch model's `__init__` method.\n",
|
|
"- Now, below is an example applying the model manually. Here, pretend that the `test_dataloader` is a new data loader."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 17,
|
|
"id": "692ff4b3",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor([7, 2, 1, 0, 4])"
|
|
]
|
|
},
|
|
"execution_count": 17,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"test_dataloader = data_module.test_dataloader()\n",
|
|
"\n",
|
|
"all_true_labels = []\n",
|
|
"all_predicted_labels = []\n",
|
|
"for batch in test_dataloader:\n",
|
|
" features, labels = batch\n",
|
|
" \n",
|
|
" with torch.no_grad(): # since we don't need to backprop\n",
|
|
" logits = lightning_model(features)\n",
|
|
" \n",
|
|
" predicted_labels = torch.argmax(logits, dim=1)\n",
|
|
" all_predicted_labels.append(predicted_labels)\n",
|
|
" all_true_labels.append(labels)\n",
|
|
" \n",
|
|
"all_predicted_labels = torch.cat(all_predicted_labels)\n",
|
|
"all_true_labels = torch.cat(all_true_labels)\n",
|
|
"all_predicted_labels[:5]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "e93a8f92",
|
|
"metadata": {},
|
|
"source": [
|
|
"Just as an internal check, if the model was loaded correctly, the test accuracy below should be identical to the test accuracy we saw earlier in the previous section."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 18,
|
|
"id": "b83d551c",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Test accuracy: 0.9813 (98.13%)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"test_acc = torch.mean((all_predicted_labels == all_true_labels).float())\n",
|
|
"print(f'Test accuracy: {test_acc:.4f} ({test_acc*100:.2f}%)')"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|