rasbt--deeplearning-models
Sebastian Raschka
1154 行
83 KiB
Plaintext
1154 行
83 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "0a2c67d8",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"torch : 1.10.1\n",
|
|
"pytorch_lightning: 1.6.0.dev0\n",
|
|
"torchmetrics : 0.6.2\n",
|
|
"matplotlib : 3.3.4\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"%load_ext watermark\n",
|
|
"%watermark -p torch,pytorch_lightning,torchmetrics,matplotlib"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "99b8d17e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%load_ext pycodestyle_magic\n",
|
|
"%flake8_on --ignore W291,W293,E703"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b0b654b0",
|
|
"metadata": {},
|
|
"source": [
|
|
"<a href=\"https://pytorch.org\"><img src=\"https://raw.githubusercontent.com/pytorch/pytorch/master/docs/source/_static/img/pytorch-logo-dark.svg\" width=\"90\"/></a> <a href=\"https://www.pytorchlightning.ai\"><img src=\"https://raw.githubusercontent.com/PyTorchLightning/pytorch-lightning/master/docs/source/_static/images/logo.svg\" width=\"150\"/></a>\n",
|
|
"\n",
|
|
"# Model Zoo -- Multilayer Perceptron with Dropout"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "415fa190",
|
|
"metadata": {},
|
|
"source": [
|
|
"## General settings and hyperparameters"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "8eaa4c61",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Here, we specify some general hyperparameter values and general settings\n",
|
|
"- Note that for small datatsets, it is not necessary and better not to use multiple workers as it can sometimes cause issues with too many open files in PyTorch. So, if you have problems with the data loader later, try setting `NUM_WORKERS = 0` instead."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"id": "879bbeeb",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"BATCH_SIZE = 256\n",
|
|
"NUM_EPOCHS = 20\n",
|
|
"LEARNING_RATE = 0.005\n",
|
|
"NUM_WORKERS = 4"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b6e90ead",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Implementing a Neural Network using PyTorch Lightning's `LightningModule`"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "a903d1eb",
|
|
"metadata": {},
|
|
"source": [
|
|
"- In this section, we set up the main model architecture using the `LightningModule` from PyTorch Lightning.\n",
|
|
"- We start with defining our neural network model in pure PyTorch, and then we use it in the `LightningModule` to get all the extra benefits that PyTorch Lightning provides."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "25711ffb",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import torch\n",
|
|
"import torch.nn.functional as F\n",
|
|
"\n",
|
|
"\n",
|
|
"# Regular PyTorch Module\n",
|
|
"class PyTorchMLP(torch.nn.Module):\n",
|
|
" def __init__(self, input_size, hidden_units, dropout_proba, num_classes):\n",
|
|
" super().__init__()\n",
|
|
" \n",
|
|
" self.dropout_proba = dropout_proba\n",
|
|
" # Initialize MLP layers\n",
|
|
" all_layers = []\n",
|
|
" for hidden_unit in hidden_units:\n",
|
|
" layer = torch.nn.Linear(input_size, hidden_unit)\n",
|
|
" all_layers.append(layer)\n",
|
|
" all_layers.append(torch.nn.ReLU())\n",
|
|
" all_layers.append(torch.nn.Dropout(dropout_proba))\n",
|
|
" input_size = hidden_unit\n",
|
|
"\n",
|
|
" output_layer = torch.nn.Linear(\n",
|
|
" in_features=hidden_units[-1],\n",
|
|
" out_features=num_classes)\n",
|
|
"\n",
|
|
" all_layers.append(output_layer)\n",
|
|
" self.layers = torch.nn.Sequential(*all_layers)\n",
|
|
"\n",
|
|
" def forward(self, x):\n",
|
|
" x = torch.flatten(x, start_dim=1) # to make it work for image inputs\n",
|
|
" x = self.layers(x)\n",
|
|
" return x # x are the model's logits"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"id": "cc50aaa8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pytorch_lightning as pl\n",
|
|
"import torchmetrics\n",
|
|
"\n",
|
|
"\n",
|
|
"# LightningModule that receives a PyTorch model as input\n",
|
|
"class LightningMLP(pl.LightningModule):\n",
|
|
" def __init__(self, model, learning_rate):\n",
|
|
" super().__init__()\n",
|
|
"\n",
|
|
" self.learning_rate = learning_rate\n",
|
|
" # The inherited PyTorch module\n",
|
|
" self.model = model\n",
|
|
" self.dropout_proba = model.dropout_proba\n",
|
|
"\n",
|
|
" # Save settings and hyperparameters to the log directory\n",
|
|
" # but skip the model parameters\n",
|
|
" self.save_hyperparameters(ignore=['model'])\n",
|
|
"\n",
|
|
" # Set up attributes for computing the accuracy\n",
|
|
" self.train_acc = torchmetrics.Accuracy()\n",
|
|
" self.valid_acc = torchmetrics.Accuracy()\n",
|
|
" self.test_acc = torchmetrics.Accuracy()\n",
|
|
" \n",
|
|
" # Defining the forward method is only necessary \n",
|
|
" # if you want to use a Trainer's .predict() method (optional)\n",
|
|
" def forward(self, x):\n",
|
|
" return self.model(x)\n",
|
|
" \n",
|
|
" # A common forward step to compute the loss and labels\n",
|
|
" # this is used for training, validation, and testing below\n",
|
|
" def _shared_step(self, batch):\n",
|
|
" features, true_labels = batch\n",
|
|
" logits = self(features)\n",
|
|
" loss = torch.nn.functional.cross_entropy(logits, true_labels)\n",
|
|
" predicted_labels = torch.argmax(logits, dim=1)\n",
|
|
"\n",
|
|
" return loss, true_labels, predicted_labels\n",
|
|
"\n",
|
|
" def training_step(self, batch, batch_idx):\n",
|
|
" loss, true_labels, predicted_labels = self._shared_step(batch)\n",
|
|
" self.log(\"train_loss\", loss)\n",
|
|
" \n",
|
|
" # To account for DropOut behavior during evaluation\n",
|
|
" self.model.eval()\n",
|
|
" with torch.no_grad():\n",
|
|
" _, true_labels, predicted_labels = self._shared_step(batch)\n",
|
|
" self.train_acc(predicted_labels, true_labels)\n",
|
|
" self.log(\"train_acc\", self.train_acc, on_epoch=True, on_step=False)\n",
|
|
" self.model.train()\n",
|
|
" return loss # this is passed to the optimzer for training\n",
|
|
"\n",
|
|
" def validation_step(self, batch, batch_idx):\n",
|
|
" loss, true_labels, predicted_labels = self._shared_step(batch)\n",
|
|
" self.log(\"valid_loss\", loss)\n",
|
|
" self.valid_acc(predicted_labels, true_labels)\n",
|
|
" self.log(\"valid_acc\", self.valid_acc,\n",
|
|
" on_epoch=True, on_step=False, prog_bar=True)\n",
|
|
"\n",
|
|
" def test_step(self, batch, batch_idx):\n",
|
|
" loss, true_labels, predicted_labels = self._shared_step(batch)\n",
|
|
" self.test_acc(predicted_labels, true_labels)\n",
|
|
" self.log(\"test_acc\", self.test_acc, on_epoch=True, on_step=False)\n",
|
|
"\n",
|
|
" def configure_optimizers(self):\n",
|
|
" optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)\n",
|
|
" return optimizer"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "7f8c19ce",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Setting up the dataset"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b8430555",
|
|
"metadata": {},
|
|
"source": [
|
|
"- In this section, we are going to set up our dataset."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "031d3a36",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Inspecting the dataset"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"id": "40a6fd65",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import torch\n",
|
|
"\n",
|
|
"from torchvision import datasets\n",
|
|
"from torchvision import transforms\n",
|
|
"from torch.utils.data import DataLoader\n",
|
|
"\n",
|
|
"\n",
|
|
"train_dataset = datasets.MNIST(root='./data', \n",
|
|
" train=True, \n",
|
|
" transform=transforms.ToTensor(),\n",
|
|
" download=True)\n",
|
|
"\n",
|
|
"train_loader = DataLoader(dataset=train_dataset, \n",
|
|
" batch_size=BATCH_SIZE, \n",
|
|
" num_workers=NUM_WORKERS,\n",
|
|
" drop_last=True,\n",
|
|
" shuffle=True)\n",
|
|
"\n",
|
|
"test_dataset = datasets.MNIST(root='./data', \n",
|
|
" train=False,\n",
|
|
" transform=transforms.ToTensor())\n",
|
|
"\n",
|
|
"test_loader = DataLoader(dataset=test_dataset, \n",
|
|
" batch_size=BATCH_SIZE,\n",
|
|
" num_workers=NUM_WORKERS,\n",
|
|
" drop_last=False,\n",
|
|
" shuffle=False)\n",
|
|
"\n",
|
|
"# Checking the dataset\n",
|
|
"all_train_labels = []\n",
|
|
"all_test_labels = []\n",
|
|
"\n",
|
|
"for images, labels in train_loader: \n",
|
|
" all_train_labels.append(labels)\n",
|
|
"all_train_labels = torch.cat(all_train_labels)\n",
|
|
" \n",
|
|
"for images, labels in test_loader: \n",
|
|
" all_test_labels.append(labels)\n",
|
|
"all_test_labels = torch.cat(all_test_labels)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"id": "d6f6e3ae",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Training labels: tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])\n",
|
|
"Training label distribution: tensor([5918, 6728, 5951, 6124, 5827, 5414, 5906, 6253, 5840, 5943])\n",
|
|
"\n",
|
|
"Test labels: tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])\n",
|
|
"Test label distribution: tensor([ 980, 1135, 1032, 1010, 982, 892, 958, 1028, 974, 1009])\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print('Training labels:', torch.unique(all_train_labels))\n",
|
|
"print('Training label distribution:', torch.bincount(all_train_labels))\n",
|
|
"\n",
|
|
"print('\\nTest labels:', torch.unique(all_test_labels))\n",
|
|
"print('Test label distribution:', torch.bincount(all_test_labels))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "26e8dcf2",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Performance baseline"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "04432348",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Especially for imbalanced datasets, it's quite useful to compute a performance baseline.\n",
|
|
"- In classification contexts, a useful baseline is to compute the accuracy for a scenario where the model always predicts the majority class -- you want your model to be better than that!"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"id": "f90f746f",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Baseline ACC: 11.35%\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"majority_prediction = torch.argmax(torch.bincount(all_test_labels))\n",
|
|
"baseline_acc = torch.mean((all_test_labels == majority_prediction).float())\n",
|
|
"print(f'Baseline ACC: {baseline_acc*100:.2f}%')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "771bc2c2",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Setting up a `DataModule`"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "ead00f13",
|
|
"metadata": {},
|
|
"source": [
|
|
"- There are three main ways we can prepare the dataset for Lightning. We can\n",
|
|
" 1. make the dataset part of the model;\n",
|
|
" 2. set up the data loaders as usual and feed them to the fit method of a Lightning Trainer -- the Trainer is introduced in the next subsection;\n",
|
|
" 3. create a LightningDataModule.\n",
|
|
"- Here, we are going to use approach 3, which is the most organized approach. The `LightningDataModule` consists of several self-explanatory methods as we can see below:\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"id": "bf79ab1e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import os\n",
|
|
"\n",
|
|
"from torch.utils.data.dataset import random_split\n",
|
|
"from torch.utils.data import DataLoader\n",
|
|
"\n",
|
|
"\n",
|
|
"class DataModule(pl.LightningDataModule):\n",
|
|
" def __init__(self, data_path='./'):\n",
|
|
" super().__init__()\n",
|
|
" self.data_path = data_path\n",
|
|
" \n",
|
|
" def prepare_data(self):\n",
|
|
" datasets.MNIST(root=self.data_path,\n",
|
|
" download=True)\n",
|
|
" return\n",
|
|
"\n",
|
|
" def setup(self, stage=None):\n",
|
|
" # Note transforms.ToTensor() scales input images\n",
|
|
" # to 0-1 range\n",
|
|
" train = datasets.MNIST(root=self.data_path, \n",
|
|
" train=True, \n",
|
|
" transform=transforms.ToTensor(),\n",
|
|
" download=False)\n",
|
|
"\n",
|
|
" self.test = datasets.MNIST(root=self.data_path, \n",
|
|
" train=False, \n",
|
|
" transform=transforms.ToTensor(),\n",
|
|
" download=False)\n",
|
|
"\n",
|
|
" self.train, self.valid = random_split(train, lengths=[55000, 5000])\n",
|
|
"\n",
|
|
" def train_dataloader(self):\n",
|
|
" train_loader = DataLoader(dataset=self.train, \n",
|
|
" batch_size=BATCH_SIZE, \n",
|
|
" drop_last=True,\n",
|
|
" shuffle=True,\n",
|
|
" num_workers=NUM_WORKERS)\n",
|
|
" return train_loader\n",
|
|
"\n",
|
|
" def val_dataloader(self):\n",
|
|
" valid_loader = DataLoader(dataset=self.valid, \n",
|
|
" batch_size=BATCH_SIZE, \n",
|
|
" drop_last=False,\n",
|
|
" shuffle=False,\n",
|
|
" num_workers=NUM_WORKERS)\n",
|
|
" return valid_loader\n",
|
|
"\n",
|
|
" def test_dataloader(self):\n",
|
|
" test_loader = DataLoader(dataset=self.test, \n",
|
|
" batch_size=BATCH_SIZE, \n",
|
|
" drop_last=False,\n",
|
|
" shuffle=False,\n",
|
|
" num_workers=NUM_WORKERS)\n",
|
|
" return test_loader"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "fc7ad9eb",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Note that the `prepare_data` method is usually used for steps that only need to be executed once, for example, downloading the dataset; the `setup` method defines the the dataset loading -- if you run your code in a distributed setting, this will be called on each node / GPU. \n",
|
|
"- Next, lets initialize the `DataModule`; we use a random seed for reproducibility (so that the data set is shuffled the same way when we re-execute this code):"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"id": "e68b3400",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"torch.manual_seed(1) \n",
|
|
"data_module = DataModule(data_path='./data')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "f18335ff",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Training the model using the PyTorch Lightning Trainer class"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "ce6856d0",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Next, we initialize our model.\n",
|
|
"- Also, we define a call back so that we can obtain the model with the best validation set performance after training.\n",
|
|
"- PyTorch Lightning offers [many advanced logging services](https://pytorch-lightning.readthedocs.io/en/latest/extensions/logging.html) like Weights & Biases. Here, we will keep things simple and use the `CSVLogger`:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"id": "5b563683",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pytorch_lightning.callbacks import ModelCheckpoint\n",
|
|
"from pytorch_lightning.loggers import CSVLogger\n",
|
|
"\n",
|
|
"\n",
|
|
"pytorch_model = PyTorchMLP(\n",
|
|
" input_size=28*28,\n",
|
|
" hidden_units=(128, 256),\n",
|
|
" dropout_proba=0.5,\n",
|
|
" num_classes=10\n",
|
|
")\n",
|
|
"\n",
|
|
"lightning_model = LightningMLP(\n",
|
|
" pytorch_model, learning_rate=LEARNING_RATE)\n",
|
|
"\n",
|
|
"callbacks = [ModelCheckpoint(\n",
|
|
" save_top_k=1, mode='max', monitor=\"valid_acc\")] # save top 1 model \n",
|
|
"logger = CSVLogger(save_dir=\"logs/\", name=\"my-mlp\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "ce99925f",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Now it's time to train our model:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"id": "1f443b66",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"GPU available: True, used: True\n",
|
|
"TPU available: False, using: 0 TPU cores\n",
|
|
"IPU available: False, using: 0 IPUs\n",
|
|
"LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
|
|
"\n",
|
|
" | Name | Type | Params\n",
|
|
"-----------------------------------------\n",
|
|
"0 | model | PyTorchMLP | 136 K \n",
|
|
"1 | train_acc | Accuracy | 0 \n",
|
|
"2 | valid_acc | Accuracy | 0 \n",
|
|
"3 | test_acc | Accuracy | 0 \n",
|
|
"-----------------------------------------\n",
|
|
"136 K Trainable params\n",
|
|
"0 Non-trainable params\n",
|
|
"136 K Total params\n",
|
|
"0.544 Total estimated model params size (MB)\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validation sanity check: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "a728b85aca43481882dd0ad69645e474",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Training: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Validating: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Training took 1.36 min in total.\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import time\n",
|
|
"\n",
|
|
"\n",
|
|
"trainer = pl.Trainer(\n",
|
|
" max_epochs=NUM_EPOCHS,\n",
|
|
" callbacks=callbacks,\n",
|
|
" progress_bar_refresh_rate=50, # recommended for notebooks\n",
|
|
" accelerator=\"auto\", # Uses GPUs or TPUs if available\n",
|
|
" devices=\"auto\", # Uses all available GPUs/TPUs if applicable\n",
|
|
" logger=logger,\n",
|
|
" deterministic=True,\n",
|
|
" log_every_n_steps=100)\n",
|
|
"\n",
|
|
"start_time = time.time()\n",
|
|
"trainer.fit(model=lightning_model, datamodule=data_module)\n",
|
|
"\n",
|
|
"runtime = (time.time() - start_time)/60\n",
|
|
"print(f\"Training took {runtime:.2f} min in total.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "6805d8a1",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Evaluating the model"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "e4de47d0",
|
|
"metadata": {},
|
|
"source": [
|
|
"- After training, let's plot our training ACC and validation ACC using pandas, which, in turn, uses matplotlib for plotting (you may want to consider a [more advanced logger](https://pytorch-lightning.readthedocs.io/en/latest/extensions/logging.html) that does that for you):"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"id": "4a8892b9",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"<AxesSubplot:xlabel='Epoch', ylabel='ACC'>"
|
|
]
|
|
},
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
},
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABWZ0lEQVR4nO2dd3hUVdrAfyeT3hvpgRRqgBB6UwGxYMUurqLup+u6u666rq5uc3V1d93mNtvaXVdFV0ERsSKI9CYdAoEEkkBITyY9kznfH3cShzBJpt2ZSXJ+z3OfmXvuOfe+czO575zzNiGlRKFQKBQKe/HztgAKhUKh6F8oxaFQKBQKh1CKQ6FQKBQOoRSHQqFQKBxCKQ6FQqFQOIS/twXwBPHx8TIjI8OpsY2NjYSFhblXIDei5HMNJZ9rKPlcx5dl3L59e6WUcsgZB6SUA36bPHmydJbVq1c7PdYTKPlcQ8nnGko+1/FlGYFt0sYzVS1VKRQKhcIhlOJQKBQKhUMoxaFQKBQKhxgUxnGFQjGwaG9vp6SkhJaWll77RUVFceDAAQ9J5Ry+IGNwcDBpaWkEBATY1V8pDoVC0e8oKSkhIiKCjIwMhBA99jMajURERHhQMsfxtoxSSqqqqigpKSEzM9OuMWqpSqFQ9DtaWlqIi4vrVWko7EMIQVxcXJ+zN2uU4lAoFP0SpTTch6P3UikOhaIHahrb+GBnqbfFUCh8DqU4FIoe+O+mY9yzZCen6u2fwisUgwGlOBSKHjhYZgSgpKbZy5IofI3a2lqeeeYZh8ddfPHF1NbWOjzu1ltv5d1333V4nF4oxaFQ9MDBsnoATtQqxaE4nZ4UR0dHR6/jVq5cSXR0tE5SeQ7ljqtQ2KClvYOiqiZAKQ5f59EP97H/RL3NYx0dHRgMBofPmZMSyW8uG9vj8YceeogjR46Ql5dHQEAA4eHhJCcns3PnTvbv388VV1xBcXExLS0t3HPPPdxxxx0AZGRksG3bNhoaGrjooos466yzWLduHenp6XzwwQeEhIT0KduqVau4//77MZlMTJ06lWeffZagoCAeeughli9fjr+/PxdccAF/+ctf+N///sejjz6KwWAgKiqKtWvXOnwvbKEUh0Jhg4LyBjrMElCKQ3EmTzzxBHv37mXnzp2sWbOGSy65hL1793bFQbz88svExsbS3NzM1KlTufrqq4mLizvtHIcPH+att97iySef5LbbbuO9997jpptu6vW6LS0t3HrrraxatYqRI0dy88038+yzz3LzzTezbNkyDh48iBCiaznst7/9LZ9++impqalOLZH1hFIcCoUN8i32jdBAA6W1yjjuy/Q2M/BUcN20adNOC5775z//ybJlywAoLi7m8OHDZyiOzMxM8vLyMBqNTJ48maKioj6vk5+fT2ZmJiNHjgTglltu4emnn+auu+4iODiY22+/nUsuuYRLL70UgNmzZ3Prrbdy3XXXcdVVV7np0yobh0Jhk/xTRgL9/ZiSEatmHIo+sa6nsWbNGr744gs2btzIrl27mDhxos3guqCgoK73BoMBk8nU53W0TOdn4u/vz5YtW7j66qt5//33WbBgAQDPPfccjz/+OMXFxeTl5VFVVeXoR7N9PbecRaEYYBwsMzJ8SDhDY0PYXVLrbXEUPkZERARGo9Hmsbq6OmJiYggNDeXgwYNs2rTJbdcdPXo0RUVFFBQUMHz4cF5//XXmzJlDQ0MDTU1NXHzxxcyYMYPhw4cDcOTIEaZPn8706dP58MMPKS4uPmPm4wxKcSgUNsgvq2d2djwp0SHUNrXT1GYiNFD9uyg04uLimD17NuPGjSMkJITExMSuYwsWLOC5554jNzeXUaNGMWPGDLddNzg4mFdeeYVrr722yzh+5513Ul1dzcKFC2lpaUFKyd/+9jcAHnjgAQ4fPoyUkvnz5zNhwgS3yKH+ExSKbtQ2tXGqvpVRSREkRQUDcKK2heEJ4V6WTOFLvPnmmzbbg4KC+Pjjj20e67RjxMfHs3fv3q72+++/v9drvfrqq13v58+fzzfffHPa8eTkZLZs2XLGuKVLl/Z6XmdRNg6FohudgX+jkiJIjtLcI5WdQ6H4Fl0VhxBigRAiXwhRIIR4yMbxG4UQuy3bBiHEBEv7KCHETqutXghxr+XYI0KIUqtjF+v5GRSDj06PqtFJkaREd844lOJQ6M+PfvQj8vLyTtteeeUVb4t1BrotVQkhDMDTwPlACbBVCLFcSrnfqlshMEdKWSOEuAh4HpgupcwH8qzOUwossxr3NynlX/SSXTG4OVhmJCokgMTIIExmiZ9QikPhGZ5++mlvi2AXes44pgEFUsqjUso2YAmw0LqDlHKDlLLGsrsJSLNxnvnAESnlMR1lVSi6yC+rZ1RSBEIIAgx+JEYGq1gOhcIKPY3jqUCx1X4JML2X/rcBtixKi4C3urXdJYS4GdgG/NRK+XQhhLgDuAMgMTGRNWvW2C+5FQ0NDU6P9QRKPtfoLp+Ukv2lTcxO9e9qDxNt7C86wZo1Z3zNPC6fr+Et+aKionp0h7Wmo6PDrn7exFdkbGlpsf9vKaXUZQOuBV602l8M/KuHvvOAA0Bct/ZAoBJItGpLBAxos6XfAS/3JcvkyZOls6xevdrpsZ5Ayeca3eU7XtUohz24Qv53U1FX211v7pDn/OlLD0um0d/un6fYv3+/Xf3q6+t1lsR1fEVGW/cU2CZtPFP1XKoqAdKt9tOAE907CSFygReBhVLK7mGNFwE7pJSnOhuklKeklB1SSjPwAtqSmC5UNrTSbLIdqakYmBw61WkY/zZNRUp0MCdrWzCb1XdBoQB9bRxbgRFCiEwhRCDaktNy6w5CiKHAUmCxlPKQjXPcQLdlKiFEstXulcBedOKpLwu4+8sm7npzB1/sP0WbyazXpRQ+Qqcr7sjEbxVHanQIbR1mKhtbvSWWYgAQHq7FAZ04cYJrrrnGZp+5c+eybdu2Hs+RkZFBZWWlLvI5gm42DimlSQhxF/Ap2tLSy1LKfUKIOy3HnwMeBuKAZyw1b01SyikAQohQNI+s73c79Z+EEHmABIpsHHcbV01KpbikhA1Hqlix+yQxoQFckpvMFXmpTB4Wo2oeD0Dyy4ykRocQERzQ1ZbSFcvRQkJEsLdEUwwQUlJSfKookzPoGjkupVwJrOzW9pzV+9uB23sY24SmVLq3L3azmD2SmxbN4pwgnjv7HL4+XMH735zg3e0l/HfTcdJjQ1g4IZUrJqYwPEH/7JsKz5BfZjxtmQogJfrbIMC89GgvSKXolY8fgrI9Ng+FdJjA4MRjLmk8XPREr10efPBBhg0bxg9/+EMAHnnkEYQQrF27lpqaGtrb23n88cdZuPA0Z1KKioq49NJL2bt3L83Nzdx6660cPnyYMWPG0Nxsv9v3k08+ycsvvwzA7bffzr333ktjYyPXXXcdJSUldHR08Otf/5rrr7/eZq0OV1ApR+wgwODHuaMTOXd0Ig2tJj7bV8ayb0p5Zk0BT60uYFxqJFfkpXL5hBQSItUv0v5Km8nMkYoG5o9JOK09NVpFjyvOZNGiRdx7771diuOdd97hk08+4Sc/+QmRkZFUVlYyY8YMLr/88h5XJ5599llCQ0PZvXs3u3fvZtKkSXZde/v27bzyyits3rwZKSXTp09nzpw5HD16lJSUFD766CNAS7hYXV1ts1aHKyjF4SDhQf5cNSmNqyalUW5s4cNdJ/lgZymPf3SA3688wKzseK6YmMqFYxNPW+5Q+D5HKxswmSWjus04IkP8CQs0UKoUh2/Sy8ygWcd6HBMnTqS8vJwTJ05QUVFBTEwMycnJ/OQnP2Ht2rX4+flRWlrKqVOnSEpKsnmOtWvXcvvt2qJLbm4uubm5dl173bp1XHnllV3p3K+66iq+/vprFixYwP3338+DDz7IpZdeytlnn43JZLJZq8MVVK4qF0iICOa2szJZftdZrPrpHO6aN5xj1Y3c/79dTHn8C3705g62H/O877/COaxTjVgjhCAlOkTNOBRncM011/Duu+/y9ttvs2jRIt544w0qKirYvn07O3fuJDEx0WYtDmucsZXKHupyjBw5ku3btzN+/Hh+/vOf89vf/rbHWh2uoBSHm8geEs59F4xi7QPzeO8Hs7huSjrrCyq55eUtmDqUN1Z/4GCZkQCDIDM+7IxjmuJQ0eOK01m0aBFLlizh3Xff5ZprrqGuro6EhAQCAgJYvXo1x471nvDinHPO4Z133gFg79697N69267rnnPOObz//vs0NTXR2NjIsmXLOPvsszlx4gShoaHcdNNN3H///ezYsYOGhgbq6uq4+OKL+fvf/87OnTtd/dhqqcrdCCGYPCyGycNimJIRwz1LdnLoVAM5KZF9D1Z4lfwyI1nx4QT6n/l7KiU6hL2ldV6QSuHLjB07FqPRSGpqKsnJydx4441cdtllTJkyhby8PEaPHt3r+B/84AfcdNNN5ObmkpeXx7Rp9oWlTZo0iVtvvbWr/+23387EiRP59NNPeeCBB/Dz8yMgIIBnn30Wo9Fos1aHKyjFoSO5adEA7C6pVYqjH5BfZmTysBibx1Kjg6lqbKOlvYPgAIOHJVP4Mnv2fOvRFR8fz8aNG232a2hoALRYjM5aHCEhIbz66qt222Gs65Lfd9993Hfffacdv/DCC7nwwgvPGGerVocrqKUqHcmICyUy2J9dqvSoz1Pf0k5pbfMZhvFOUpRnlULRhZpx6IgQggnp0ewqVkscvs6hsjNTjVjzreJoIWuIqgSo0Jfp06fT2np6poLXX3+d8ePHe0mi01GKQ2dy06J47qujaonDx7Gu+mcLFcvhe0gpB2z2hs2bN3v0ej15afWEWqrSmdy0aDrMkn0n6r0tiqIX8suMRAT5dymI7iRGBiMEKpbDRwgODqaqqsrhB57iTKSUVFVVERxsf/CymnHoTGeKil3FtT0aXhXeJ7/MyEhL8SZbBPr7kRARxMk6pTh8gbS0NEpKSqioqOi1X0tLi0MPRG/gCzIGBweTlmarjp5tlOLQmcTIYBIjg9itDOQ+i5SSg2X1XDohpdd+KpbDdwgICCAzM7PPfmvWrGHixIkekMh5+oOM3VFLVR4gNy2a3SXKQO6rlNW3UN9i6tEw3klKlIoeVyhAKQ6PMCEtiqOVjdQ1t3tbFIUNugzjiX0ojuhgSmub1bq6YtCjFIcH6AwEVJHHvklPOaq6kxIdQqvJTHVjmyfEUih8FqU4PEBuWhQAO4trvSuIwiaHyowkRQYTFdp7NmPrWA6FYjCjFIcHiA4NJCMuVBnIfZSDZcYe4zes6XTVVS65isGOUhweQhnIfZMOs6SgoqFPwziotCMKRSdKcXiI3LQoTta1UG5Uyxy+xKkmSZvJbNeMIyY0gOAAP6U4FIMepTg8xARLIOBulbfKpygxarVS7FEcXQWdVBCgYpCjFIeHGJsSicFPqEy5PkZxgxmDn2B4gn2JC1OjQyhVxnHFIEdXxSGEWCCEyBdCFAghHrJx/EYhxG7LtkEIMcHqWJEQYo8QYqcQYptVe6wQ4nMhxGHLa7/I4xEa6M+IhHB2KTuHT1FiNJMZH0aQv30JKFUQoEKho+IQQhiAp4GLgBzgBiFETrduhcAcKWUu8BjwfLfj86SUeVLKKVZtDwGrpJQjgFWW/X7BhLRodpfUqgAyH6LEaJ99o5OU6BAqjK20mjp0lEqh8G30nHFMAwqklEellG3AEmChdQcp5QYpZY1ldxNgT5athcBrlvevAVe4R1z9yU2PorapneJq9YvVF2hsNVHRLBndR8S4NSnRWjK6sjq1XKUYvOiZ5DAVKLbaLwGm99L/NuBjq30JfCaEkMC/pZSds5FEKeVJACnlSSFEgq2TCSHuAO4ASExMZM2aNU59iIaGBqfHdsdUr/1KffOzDcxIds+td6d8euDL8h2p1f4e7ZXHWLOm1K4xFVXamJVrNjEmTv/6Kr58/0DJ5w76g4xnIKXUZQOuBV602l8M/KuHvvOAA0CcVVuK5TUB2AWcY9mv7Ta2pi9ZJk+eLJ1l9erVTo/tTpupQ4785Ur52If73HZOd8qnB74s31ubj8lhD66QRZUNdo8prGiQwx5cIf+3rVhHyb7Fl++flEo+d+DLMgLbpI1nqp5LVSVAutV+GnCieychRC7wIrBQSlnV2S6lPGF5LQeWoS19AZwSQiRbxiYD5bpIrwMBBj9yUiJVIKCPcLDMSJAB0mNC7R6TFKUtVSkDuWIwo6fi2AqMEEJkCiECgUXAcusOQoihwFJgsZTykFV7mBAiovM9cAGw13J4OXCL5f0twAc6fga3MyEtmr0n6ugwKwO5t8kvM5Ia7oefn/3lR4MDDMSHBynFoRjU6KY4pJQm4C7gU7RlqHeklPuEEHcKIe60dHsYiAOe6eZ2mwisE0LsArYAH0kpP7EcewI4XwhxGDjfst9vyE2Loqmtg4LyBm+LMqiRUpJ/ykhahOP/AqmW9OoKxWBF1wqAUsqVwMpubc9Zvb8duN3GuKPAhO7tlmNVwHz3Suo5JliVknXEDVThXioaWqlubCMtPdDhsSnRIRw6ZdRBKoWif6Aixz1MZlwYEUH+KoLcy3TW4HBmxtFZQlaqeBzFIEUpDg/j5ycYnxalDORexlXF0dzeQW2TquioGJwoxeEFctOiOVhWr6KPvcjBMiPx4UFEBtpvGO8k1RIEqOwcisGKUhxeIC89ivYOyYGTap3cWxw6ZbSrBoctOutynFTR44pBilIcXqCzBvkuVUrWK3SYJYdO2Vf1zxbJUaqgk2JwoxSHF0iOCiY+PEgZyL3E8eomWtodS25oTVxYIIH+qqCTYvCiFIcXEEIwQRnIvUZ+WT2A00tVfn6ClCgVy6EYvCjF4SVy06I5UtGAsUV55niag2VGhIARCc7H0WguuUpxKAYnSnF4iQnpUUgJe0rVrMPT5JcZyYgLIyTQ+ey2nbEcCsVgRCkOL9FpIFfLVZ4nv8zIKAdqcNgiJTqEU8YW2jvMbpJKoeg/KMXhJWLDAkmPDWG3MpB7lJb2DoqqGl1O95IaHYyUqqCTYnCiFIcXyU2LZlexmnF4ksOnGjBL5w3jnXTGcig7h2IwohSHF5mQFkVpbTOVDa3eFmXQcNDiUeXqjKNLcdQpxaEYfCjF4UUmdNk5ar0qx2Aiv8xIcIAfw+LCXDpPSlcQoFqqUgw+lOLwIuNSo/ATqOUqD5J/ysiIhAgMDhRvskVIoIHYsEAVy6EYlCjF4UXCgvwZnhCuZhwe5GCZ86lGupMSHaxsHIpBiVIcXiY3LZrdJXWqtoMHqG5so8LY6rIrbicpUSoIUDE4UYrDy0xIj6aqsU0teXgAdxnGO0mJDqG0plkpfcWgQykOLzMhLQpQdg5P0Fm8yVVX3E5So0NobOugvsXklvMpFP0FpTi8zOikSAINfgPWzuFLv8bzy4zEhAYwJCLILedTsRyKwYq/nicXQiwA/gEYgBellE90O34j8KBltwH4gZRylxAiHfgPkASYgeellP+wjHkE+B5QYRn3CynlSj0/h54E+vsxJjliQKZY/+tn+TyzuonodZ8zJCKoa0uICLa8Bp32Gh7kjxCueTv1Rqdh3F3XSLFUAjxR28yY5Ei3nFOh6A/opjiEEAbgaeB8oATYKoRYLqXcb9WtEJgjpawRQlwEPA9MB0zAT6WUO4QQEcB2IcTnVmP/JqX8i16ye5rctGiWfVNKh1m67CbqK7yx+Rj/+rKAvCEGxmYlUW5spcLYytGKRiqMrbTZyPEUEmCwUi7a67iUKK6dkubyw95sKd503ZR0l85jTaqacSgGKXrOOKYBBVLKowBCiCXAQqBLcUgpN1j13wSkWdpPAict741CiANAqvXYgcSE9Ghe33SMoxUNjHCTx483WZNfzsMf7GPeqCHcNKyR+eeOP+24lJK65nYqjK1dCqXc2EJF1/tWCsobWF9QyX82HqOqsY0fzM12SabS2maa2jrcZhgHiA8PIsAgKFVBgIpBhp6KIxUottovQZtN9MRtwMfdG4UQGcBEYLNV811CiJuBbWgzkxqXpfUiXQbykrp+rzj2n6jnR2/sYFRiBE99ZxJbN647o48QgujQQKJDA3v9vFJK7l6ykz99epDsIWFcMDbJabkOWgzj7lQcfn6CZOWSqxiECL2Ml0KIa4ELpZS3W/YXA9OklD+20Xce8AxwlpSyyqo9HPgK+J2UcqmlLRGoBCTwGJAspfw/G+e8A7gDIDExcfKSJUuc+hwNDQ2Eh4c7NdZezFLywy+amJ3qz+Icxwy3vcmXX93BjlMmrhoRSJC//ktgNS1mfrtR+/X98MxgYoL9XL5/bR2SP2xp4USDmV9OD2ZopHM1NJYfaWPp4XaePS+UEKt74ap8T2xppsMMv5wR4vQ5esMT3z9XUPK5ji/LOG/evO1SyilnHJBS6rIBM4FPrfZ/DvzcRr9c4Agwslt7APApcF8v18gA9vYly+TJk6WzrF692umxjnDdcxvk5U+tc3hcT/LVNbfJqY9/Loc9uEJe/tQ6WWlscVHC3jG2tMuL/r5W5vz6Y7mvtK5P+RzhVF2znP67L+SsP6yS5fXOfY4fvbFdnvXHVWe0uyrfT97+Rs76w5nndRee+v45i5LPdXxZRmCbtPFM1dMddyswQgiRKYQIBBYBy607CCGGAkuBxVLKQ1btAngJOCClfLLbmGSr3SuBvTrJ71EmpEdz4EQ9bSb3FAb68yf5VDa08tPzR3LwZD1XP7uBY1WNbjl3d0wdZn785g7yTxl5+sZJ5KS418MoITKYF2+ZQlVjK3f+dzst7R0On0Mr3uR+z6eUqBDK6lswqYJOikGEbopDSmkC7kKbNRwA3pFS7hNC3CmEuNPS7WEgDnhGCLFTCLHN0j4bWAyca2nfKYS42HLsT0KIPUKI3cA84Cd6fQZPMiEtmrYOc1d0sytsP1bDfzcf45ZZGfx4/gje/N50apvbueqZDewqrnVdWCuklDz64X5W51fw24VjmTsqwa3n72RcahRPXpfH9mM1/GLpHofiQ1pNHRytbHRb4J81KdEhdJgl5UaVGl8xeNA1AFBKuVJKOVJKmS2l/J2l7Tkp5XOW97dLKWOklHmWbYqlfZ2UUkgpc62OrbQcWyylHG85drnUPLD6PblWBnJXaDOZ+cXSPSRFBvPTC0YBMHlYLO/9YBYhgQYWPb+J1QfLXZa3k5fWFfL6pmN8/5wsbpw+zG3ntcXF45O57/yRLP2mlOe+Omr3uCPljXSYpVsN451Yx3IoFIMFFTnuI6TFhBAbFshuF2cEL3x9lPxTRh5bOI7woG+d5rKHhLP0h7PIGhLG7f/Zxjtbi3s5i318sreM3608wEXjknhwwWiXz2cPPz53OJdNSOFPnx7ks31ldo3JP6XN4vSYcXTGcqhcY4rBhFIcPoIQgty0KHa7MOMoqmzkH6sOc9G4JM7LSTzjeEJEMG9/fyazsuP42Xu7+ccXh51OCbKzuJZ73/6GCWnR/O36PPw8FLgohODP1+SSmxrFvW/vZP+Jvpf2DpYZCTT4kRHvWvEmWyRHq4JOisGHUhw+xIS0aA6XG2lsdTxpnpSSX76/hyCDH49cPrbHfuFB/rx861SumpTK3744xC+W7XHYsFtc3cTtr21lSEQQL94yheAA51xknSU4wMDzN08hMjiA7/1nGxV92Bfyy4xkJ4QTYHD/1z08yJ+okAC1VKUYVCjF4UNMSI/CLGFvqeOzjqU7SllfUMXPLhpNYmRwr30DDH789doJ3DVvOG9tKeb7r2+nqc0+ZVXX3M53X91Km8nMK7dOJT7cPQkDHSXRAU+r/DKjLstUnaREqyBAxeBCKQ4fIrerBrljiqO6sY3HP9rP5GEx3DhtqF1jhBDcf+EoHr9iHKvzy7nhhc1UNfT+y73NZOYH/93OsapG/r14CsMTvBvlbo+nVV1TOyfrWnQxjHeSGh2sbByKQYVSHD5EfHgQqdEhDmfKffyj/RhbTPz+yvEO2xpumjGM526a3Gesh5SSXyzbw4YjVTxxVS4zs+Mcuo5e9OVplX/K/alGuqNmHIrBhlIcPoajBvL9VR0s3VHK9+dkOf1wvGBsEm9+bwZ1vcR6PL26gHe3l3DP/BFcPTnNqevoRW+eVvll+nlUdZISHUJ9iwljS7tu11D0PwrKjcz6wyqKKvUJvPUmSnH4GBPSozle3UR1Y1uffVvaO3h1XysZcaH8+NwRLl138rAY3u0h1uODnaX85bNDXDUxlXvPc+06etCbp9XBMiORwf4k9WH3cYXOgk4n65RnleJbPtlbxom6FjYereq7cz9DKQ4fozMQ0J6KgP/68jDlTZLfXTneLZ5NtmI9thRW88D/djM9M5Y/XD1e10JLrtCTp1W+m4s32SLVEgSo7BwKa9YVVAJw4KTr2SB8DaU4fIzxqVEI0beBPL/MyL+/OsrsFH9mD4932/W7x3rc8vIW0mJDeH7xFIL8Pet26yi2PK3yTxl1tW+AKiGrOJOmNhM7jtUC2BVr1N9QisPHiAgOICs+rNcZh9ks+fnS3UQE+7NodKDbZeiM9bh+SjqxYYG8eus0okID3H4dPbD2tPr+69sxtpgYlaRvWdeEiGAMfkIpDkUXW4tqaOswkz0kjAMn6zGb9Slf4S2U4vBBJqRHs7O4rseo7je2HGfH8Vp+fWkOEYH6LMEEGPz44zW5rHtwHkPjQnW5hl5cPD6Zn5w3kq8OaWXp9TSMAxj8BEmRwSp6vBtVDa38+dODtJocz2bc31lfUEmgwY+bZ2bQ2NbB8eomb4vkVuxSHEKIMCGEn+X9SCHE5UKI/vETtB8yIS2ayoZWm8bWU/Ut/Onjg8weHseVE1N1l8VXbRp9cfd8zdMq0N9P96Uq0HJWKRvH6SzdUcrTq4+w3rLWP5hYd7iSScOimTQ0BoD9A8zOYe+MYy0QLIRIBVYB3wVe1UuowU5vBvJHlu+jrcPM767wXUO1LyCE4O/X57H6/rlEBuv/GyclOlgtVXVjS1E1ABsKBp5XUW9UNbSy/2Q9Zw2PZ0RiOAY/MeDsHPYqDiGlbAKuAv4lpbwSyNFPrMHNmORI/P3EGSnWP99/io/3lnH3/BG6JOwbaBj8RFf2Wr1JiQ6hrK6FjgG2lu0sZrNka6fiODK4FEfn5509PJ7gAAPDh4QPOM8quxWHEGImcCPwkaXNv5f+ChcIDjAwOjnitBlHQ6uJhz/Yy6jECO44J8t7wilskhIdgsks+0y4OFgoqGigtqmdrPgw9p+stysuaaCwvqCSiGB/xqdqKwc5KZGDdqnqXrSa4cssVfyygNW6SaVgQlo0u0vqurwx/vpZPmX1Lfz+qvG6ZHlVuIaqy3E6Wwq12caP5w8HYNMADIKzhZSSrw9XMjMrDn/L/2lOciQn61oGlPK06wkkpfzKUm3vjxYjeaWU8m6dZRvUTEiLxthiorCqkV3Ftby6oYibpg9j8rAYb4umsIGK5TidrUXVJEQEcWluCmGBBjYcGRwG8uPVTZTWNnPWiG9jq8Yka+7gA2m5yl6vqjeFEJFCiDBgP5AvhHhAX9F8gMOfM/TYu165dG66Ns3dcayGh5buISEiiAcWjPKKLIq+SVYlZLuQUrKlsJqpmbEEGPyYlhk7aOwc6wu+tW90MiZZ8+obSAZye9c8cqSU9cAVwEpgKLBYL6F8hsKvyCh6C9o874M9fEg4IQEG/vhJPgdO1vPo5WM94h2kcI7I4AAigvxVviqgpKaZk3UtTMuIBWBWdjxHKxopGwT3Zn1BJclRwWRZOa/EhQeRFBk8oOwc9iqOAEvcxhXAB1LKdmDgu49kzsVPmuD4Ro9f2t/gx7jUSCobWjk/J5ELxyZ5XAaFY6SoWA6ALm+qqZ2KY7iWgn+gL1eZzZL1RyqZPTz+DFf5nJTIwbdUBfwbKALCgLVCiGFAn3dBCLFACJEvhCgQQjxk4/iNQojdlm2DEGJCX2OFELFCiM+FEIctr/ot+g+biVn4Q+FXul2iN2ZkxRER5M+jl49VMRv9ABXLobG1qJqIYP+uwMsxSZFEhwYM+OWq/SfrqW1q5ywbueNykiMpKG/otVJlf8Je4/g/pZSpUsqLpcYxYF5vY4QQBuBp4CK0mI8bhBDdYz8KgTlSylzgMeB5O8Y+BKySUo5AC0Y8QyG5jcAw6iNHwlHvKI4fnzuCr342r8vwqvBtVEEnjS2F1UzNiMVgKSrm5yeYmRXHxiNVPabRGQh0ZsPtnGFZk5MSicksKShv8LRYumCvcTxKCPGkEGKbZfsr2uyjN6YBBVLKo1LKNmAJsNC6g5Ryg5SyxrK7CUizY+xC4DXL+9fQls90oyZmApzcBU3Vel7GJoH+fsSGuT+JoUIfUqJDqGlqt7t++0CkqqGVIxWNXctUnczKjqO0tpljVQMrZ5M16wsqGZUYQULEmbVfOj2rBoqB3N4gvpeBvcB1lv3FwCtokeQ9kQoUW+2XANN76X8b8LEdYxOllCcBpJQnhRAJtk4mhLgDuAMgMTGRNWvW9HLpngkIGkEmkr0f/ZvKITOdOoeeNDQ0OP3ZPMFgkq/upKYw3v9sLSnh7om16W/3b/sp7R741xSxZs23/8L+DWYAXv14A3PTPefk4an719Yh2XSkiXnp/javZ5aSIAN8tnU/CY1HvCKjO7FXcWRLKa+22n9UCLGzjzG2FuVtzlOFEPPQFMdZjo7tCSnl81iWvqZMmSLnzp3ryPAuvvqyHY6GMS60Epw8h56sWbMGZz+bJxhM8oUWVvP87o2kjRzPOSOHuOWc/e3+rf1wP0H+x7jlsnkE+n+rPKWU/H33KioNscydO8lr8unFhoJK2s2bWTQvj7mjE232GXdwA/V+grlzT/8B6ut/Y1vY+7OoWQjR+VBHCDEb6GsxtwRIt9pPA0507ySEyAVeBBZKKavsGHtKCJFsGZsMlKMj0i8Ahs3ymp1D0X9IUbEcbC2qJi89+jSlAVrSyVnZ8Ww8UjXgalOAZt/w9xNMyzzTvtFJTnIkB07UDwg7j72K407gaSFEkRCiCHgK+H4fY7YCI4QQmUKIQGARsNy6gxBiKLAUWCylPGTn2OXALZb3twAf2PkZnCdrDlQdhvoz9J5C0UViZDB+YvAqjoZWE/tO1DEtM9bm8ZnZcVQ1tnGo3OhhyfRnfUElE4dGEx7U8yJOTkokxlYTJTX9//thr1fVLinlBCAXyJVSTgTO7WOMCbgL+BQ4ALxjyXN1pxDiTku3h4E44BkhxE4hxLbexlrGPAGcL4Q4DJxv2deXzDnaq5p1KHohwOBHYmQwpYO0oNOOYzWYJT0qjlnZlniOAZZmva6pnd2ldX2WcO40kO8bAAZyhzLcWqLHO7kP+Hsf/VeiRZpbtz1n9f524HZ7x1raq4D5dgvtDhLHQWicFs+Rd4NHL63oXwxml9ytRdUY/ERX8aLupMWEMiwulA1Hqvi/szI9LJ1+bDxaiZTYjN+wZlRiBH5Ci/dYMK5/B/S64voxeCLS/Pwg42xtxjEA1icV+pESHcKJusGpOLYUVjM2JZKwXpZrZmXHsfloFaYOswcl05d1BZWEBRqYkB7da7+QQANZQ8IHhEuuK4pjcD1Bs+aA8QRUFXhbEoUPkxIdzMnalgFpAO6NVlMH3xTXnhG/0Z2Z2fEYW03sHQAPz07WF1QxIyvOrnIHOcmeSz1SYWzlzte3s7e0ru/ODtLrJxVCGIUQ9TY2I5Didml8mS47xxqviqHwbVKjQ2jrMFPZOLgKOu0pqaPNZO5bcWQNrLxVJTVNFFY29mnf6CQnJZLS2mbqmtp1lky7x5/sK8OswypJr4pDShkhpYy0sUVIKQdXBcDYLIhK91reKkX/ICWqsy7H4DKQb+lKbNh76rghEUGMSoxg4wDJW9Vp6Leuv9EbOZ0R5B6YdawvqCQqJICxKVFuP7cqJWcvQmizjsKvwTwwEpUp3M9gLei0tbCa4QnhxIUH9dl3ZnYcW4uqaTXp+3/U3NbB6uPttJn0s6esK6hkSEQQIxLC7eo/xkOKQ0rJusOVzMqO68oZ5k6U4nCErDnQUgtlu70ticJHSR2EiqPDLNl2rKbPZapOZg+Pp6XdzDfHa3WV69UNRby2v423thzX5fxms2R9QSVn2Uij3hNDIoIYEhGku4G8qKqJE3Utdi+hOYpSHI6g4jkUfRAZ4k9YoGFQ1eXILzNibDExLdO+CgfTMmPxE+iaZr3DLHlzyzEAnllToEs68/xTRqoa2xx+OOckR+o+4+jM1NuXi7CzKMXhCBGJMGSMsnMoekQIMehiObYUagrA3hlHVEgA41Oj2KijgXztoQqKq5uZl+7PqfpWlugw61hveTjPtpFGvTdyUiIpKDfquoS2/nAlqdEhDIsL1eX8SnE4StYcOLYRTIPLa0ZhP5riGDzG8a1FNaREBZMWY/9DamZ2PN8cr6WxVZ8U9K9vOsaQiCBuHBPI9MxYnllzxO2zjnUFlWQNCSM5yrF6OTnJkbR36Febo8Ms2Xi0itnD43QrAKcUh6NkzgFTM5Rs9bYkCh9lMM04pJRsKapmag9pRnpi9vA4TGbZVWbWnRRXN7E6v5wbpqbj7ye497yRlBtbeXOz+2YdbSYzm49WO7UUlJOir4F834k66prbdbNvgFIcjpMxG4SfsnMoeiQlKpiqxrYBUya0N8qbJBXG1h7zU/XElGGxBBiELm65b2w+jgAWTRsKaF5cM7JiefYr9806vjleQ3N7h1MP54y4MIID/HQzkHdVIsxWisN3CI6ClEnKzqHokU6X3JN1A3+56lCN9iCeZqd9o5OQQAMTh8a43UDeaurgnW3FnDcm8bSSyz85byQVxlbecNOsY31BJX4CZmQ5Zt8AMPgJRidFsv+k+yO6QZNtdFIEQyL6do12FqU4nCFrDpRuh9aBlx5a4TqDKZbjUI2ZmNAAhtsZx2DNrOw49p6oo7apzW3yfLynjOrGNhbPHHZa+/SsOGZlx/HsmiM0t7k+61h/pIrctGiiQpyrZpiTEsmBk0a31+Zoae9ga1GNbt5UnSjF4QyZc8BsgmMbvC2JwgfpjOUYDC65+TUdTMmIdcoIO3t4PFLCpqPus3O8vukYmfFhzLaxTHPveSOpbGjljc3HXLqGsaWdncW1Lj2cc5IjqWtu54SbZ6XbimpoM5mZbWcku7MoxeEM6dPBP1jZORQ2SYwKQgyCgk7l9S2UN0mHl6k6mZAWTUiAwW1uuftO1LH9WA03Th+Kn41o6WmZscweHsdzXx2hqc15b67NR6vpMEuXjM9dBnI32znWFVQSYBBO/03sRSkOZwgI1pSHSniosEGQv4Eh4UEDXnF05ady0DDeSaC/H1MzY91m5/jvpuMEB/hx7eT0Hvtos4423tjkvK1jXUElwQF+TBoW7fQ5RidFIIT7FYdWiTCm19T27kApDmfJmgPl+6ChwtuSKHyQwRDLsbWwmiADjLX8enaGWdlxHC5voLzetXtV39LOBztLuSw3hajQnu0OUzNiOWt4vEuzjvUFlUzLjCPI3+CsuIQG+pMZF+ZWA3lNYxt7T9TZXKZzN0pxOEvmXO1VeVcpbJA6CGI5thTVMDzaz646FD3R+ZDbeNS1WceyHaU0tXWcYRS3xU/OH0FVYxuvb3Tc1nGqvoXD5Q2c5WC0uC3GpLg39cjGo1VaJcIRrsvWF0pxOEtKHgRFKcWhsElKdDCltc1u95rxFeqa2zlYVs/IGOd/dYO21h8Z7O9SHXIpJa9vOsaEtChy06L77D95WCxnj4jn32uPOhy5/m2aEdd/1eckR1Jc3UxTu3u+I+sLKgkP8rfrHriKUhzO4meAjLOUgVxhk5ToEFpNZqob3edq6ktsP1aNlLisOAx+ghlZcWw46ryBfNPRagrKG7hpRt+zjU7uPW8k1Y1tvL7JsVnHuoJKYsMCGZPk/PJcJ50G8mKje3JWrS+oZEZWrEszQHtRisMVsuZA7TGoKfK2JAof49tYjoFp59hSWEOAQZAV7fojZFZ2HMXVzRRXNzk1/r+bjhEVEsBlE+wvSjp5WAznjBzC8w7MOqTU0qjPyo6z6bXlKGMttTmOu0FxFFc3UVTVpGuaEWt0VRxCiAVCiHwhRIEQ4iEbx0cLITYKIVqFEPdbtY8SQuy02uqFEPdajj0ihCi1Onaxnp+hV1SadUUPDPRYjq1F1YxLjSLI4PoDtPNh50w52fL6Fj7dV8Z1U9IIDnBs9vOT80ZQ3djGaxuL7Op/pKKBU/WtbguuGxIRRFxYIMfrXVccnfdO78C/TnRTHEIIA/A0cBGQA9wghMjp1q0auBv4i3WjlDJfSpknpcwDJgNNwDKrLn/rPC6lXKnXZ+iTIaMgPEnZORRnMJCjx1vaO9hdUuu2WIHhCeHEhwc55Za7ZGsxJrPkO9PtX6bqZOLQGOaO0mYdDXbMOtYddp99A7QU/DkpkW6ZcawrqCIhIsipCH5n0HPGMQ0okFIelVK2AUuAhdYdpJTlUsqtQG+V2+cDR6SUroV76oEQkHkOFK6FAWoEVThHTGgAwQF+A1Jx7Cyupb1DOpzYsCeEEMzKjmPDkSqHnAlMHWbe3Hycs0fEkxkf5tS17z1vJLVN7by2oajPvusKqhgWF0p6rPtqXOQkR1JqNNPe4bzyMJslGxysROgqekaJpALFVvslwHQnzrMIeKtb211CiJuBbcBPpZQ13QcJIe4A7gBITExkzZo1TlwaGhoaeh2b1JbI6MYKtn70Go3hGU5dwxX6ks/bDGb5ogMlOw8fZ82acqfP4Yv3b/mRNgTQUrIf2droFvniOtqpMLbx5kerSQ237/fs9lMmyupbuS5b9iiDPfcvd4iBZ77MJ6ujmBB/2w/eDrNk3aEmZqT4u/fvUWvCJOHtlWtIi3Dud/zx+g6qGtuIM1V47rsipdRlA64FXrTaXwz8q4e+jwD322gPBCqBRKu2RMCANlv6HfByX7JMnjxZOsvq1at771BzXMrfREq54Wmnr+EKfcrnZQazfDe9uEle/tQ6l87hi/fvphc3yQv/9pWU0n3yHa9qlMMeXCFfXV9o95gbX9gkZ/7+C9lu6uixjz3y7TxeI4c9uEL+a9WhHvtsK6qWwx5cIT/afcJu+ezhUFm9HPbgCrl0R7HT53j+qyNy2IMr5MnaZjdKpgFskzaeqXouVZUA1rH/acAJB89xEbBDSnmqs0FKeUpK2SGlNAMvoC2JeY/odIjNUnYOxRmkRA28IEBTh5kdx2rsLhNrL+mxoaTFhNhtID9a0cC6gkq+M30o/i66n05Ij2b+6ARe+LqQ+hbbq+brCyoRAmY6kUa9NzLjwwjwcy31yLqCSrKHhJEUFexGyXpHT8WxFRghhMgUQgSiLTktd/AcN9BtmUoIkWy1eyWw1yUp3UHmHChaDx36lMFU9E9SokOoMLby2b4y3Uqkepr9J+tpbOtwOj9Vb8zKjmOTJYFgX7yx+TgBBsF1U3vOS+UI95w3grrmdl5bX2Tz+LqCSsalRBETFuiW63Xib/AjLdzP6QjyNpOZLYXOVSJ0Bd0Uh5TSBNwFfAocAN6RUu4TQtwphLgTQAiRJIQoAe4DfiWEKBFCRFqOhQLnA0u7nfpPQog9QojdwDzgJ3p9BrvJmgNtRjixw9uSKHyImdlxRAT5c8fr25n4289Z/NJmXlpXyNEKfWpNe4IthVpiQz2yr84eHk9dc3ufv76b2zr437ZiLhybREKEe35l56ZFc96YBF74+ugZs47GVhPfHK/RLUYiPVKrBiidcLBxpRKhK+iaQlFqrrIru7U9Z/W+DG0Jy9bYJuCMeaGUcrGbxXSdjHO016NfQbp3V84UvsO0zFi2//p8th2rZvXBclbnV/DYiv08tgIy4kKZOyqBeaMTmJ4Z63AMgrfYWlTN0NhQXZZFOpeBNhypZHxaVI/9Ptx1gvoWE4sdiBS3h3vPG8ml/1rHK+uKuOe8EV3tW4qqae+Quv2qHxrhx9qSNsrqW0iOCul7gBVdlQiz9c9PZY2+uXcHC2FxkDRes3PMecDb0ih8iEB/P2ZlxzMrO55fXqJF+K7JL+fLg+W8teU4r24oIiTAwKzsOOaOTmDeqCGkxbjP3dOdSCnZWlTDvFEJupw/ITKY4QnhbDhSxffnZPfY7/VNxxiZGO42d+BOxqVGcX5OIi+uO8qtszO6qvutP1xJoL8fUzJi3Hq9ToZFags/B07WO6w41hVUMiE9mshg5yoROotSHO4icw5seR7amiDQN//xFd4nPTaUxTMzWDwzg5b2DjYerWLNwXK+zC9n1UHNbXdkYjjzRiUwd1QCJjvW+z3FkYoGqhvbmJapzwMUNDvHu9tLaDOZCfQ/cyV9V3Ete0rreGzhWF1iFu6ZP4LP95/ilfWF3HveSEB7OE/NiNFtVtjphrv/RD3njk60e1x9Szu7Sur44dyelaxeqFxV7iJrLnS0QfEmb0ui6CcEBxiYNyqBRxeOY+0D8/jivjn86pIxDIkI4uX1hdzwwiZ+traZGh9JlLilUAuXcrdHlTWzsuNpautgV0mtzeOvbzpGWKCBKyam6nL9calRXJCTyEvrCqlrbqfC2MrBMiOzdKxxEeIvGBYX6rCB3B2VCJ1FKQ53MXQm+PmrvFUKpxBCMDwhnNvPzuKN22fwzcMX8OdrcqlukXy421Evdn3YWlRNfHig01Ha9jAjKxYhsJlmvaaxjQ93neDKSalE6Lg0c+95IzG2mHhpXaHHckDlJEc67JK7vqCSkAADE4dG6yNULyjF4S6CwiFtqornULiF8CB/rp2STnqEH+/tKPW2OIDmUTU1I1bXtBbRoYGMTYm0Gc/x7vYSWk1mh9KnO0NOSiQLxibxyrpCVu45SWSwP+NSezbWu+WayZEUVTXZlTOrk3UFlUzNjHWpEqGzKMXhTjLnwImd0HxGBhSFwilmpfizq7iWI1524T1R20xpbbPbDdK2mJUdzzfHa2lu6+hqM5sl/918jKkZMYx2Qy2MvrjnvBEYW018uu8Us7LjMbghjXpvdNbmyC+zb9ZRVtdCgZsqETqDUhzuJGsOIKFonbclUQwQZiYb8BNaaVRvsrVIi9/Q077RyazsONo6zGw7Vt3Vtq6gkmNVTbrPNjoZkxzJReOSAJg9Qn8bQqfisHe5yp2VCJ1BKQ53kjoFAkKVnUPhNqKD/Th7xBCWfVOK2YseVpsLq4kI8mdMsv6/9qdmxOLvJ05Ls/76pmPEhweywPIw9wT3XziK6ZmxXJhjv6eTsyRFBhMTGmC3gXz9EfdVInQGpTjciX8gDJul7BwKt3LVpFRKa5vZXFjdd2ed2FpYzaRhMbov2QCEBfmTlx7dpThKa5tZdeAU109N9+h6fvaQcN7+/kwSIvXPASWEYIydBnLp5kqEzqAUh7vJnAOVh6DeNzxhFP2fC3KSCA/yZ9k3JV65fk1jG4fLGzxi3+hkVnYce0pqqWtu563NxwG4YdpQj13fG+QkR3KwzIipj9oc7q5E6AxKcbibLEs52cK13pVDMWAICTRw0bgkVu4pO81g7Ck8ad/oZNbweMxSW8tfsvU4545O8NmIeneRkxJJq8lMYWVjr/3cXYnQGZTicDeJ4yEkVtk5FG7lqklpNLSa+Gx/mcevvbWomkB/P3J7yR/lbiYOjSbI348nPj5IZUObx4zi3qTLQN6HnUOPSoSOohSHu/Hzg8yzNTuHKiercBPTM2NJjQ5hqRe8q7YU1ZCXFu3RRIxB/gamZsRyvLqJobGhnDNiiMeu7S2yh4QTaOg9xbqpw8ymo1VenW2AUhz6kDkH6kuh6oi3JVEMEPz8BFdOTOXrwxWUG1s8dt3GVhN7S+uYqmN+qp6YZYlRuGnGUK8ZgT1JgMGPEYnhvRrId5XU0dBqYraOKVDsQSkOPciaq70WrvGmFIoBxpWTUjFLWL7Tc44X3xyvpcMsPWrf6OSKvFQW5qVw/dSBbRS3pjP1SE+1OboqEXo4jXp3lOLQg9gsiExTdg6FW8keEs6E9GiPpiDZUlSNn4DJwzw/40iJDuEfiyZ2pTcfDOSkRFLV2EaFsdXm8XUFlYxNiSTWzZUIHUUpDj0QQvOuKvoazL271ikUjnD1pFQOnKzngJOlRh1la2E1Y5IjdU0qqPiWHEuA5T4bf9+mNn0rETqCUhx6kTlHy1lVttvbkigGEJfmphBgECz7Rv9ZR5vJzDfFNR6N3xjsjLF4Vtn6YbClUN9KhI6gFIdeZFrKyaoocoUbiQ0LZO6oBJZ9U9pnoJir7Cmto6XdrEt9cYVtIoMDSI8NsWkgX1+gVSL0hr2pO0px6EVkMsSPgsOfQ0e7t6VRDCCunpRKhbGV9UfOrFnhTl5Ye5SQAAMzsrxriB1sjEmKtOmSu66giinD9KtE6Ai6Kg4hxAIhRL4QokAI8ZCN46OFEBuFEK1CiPu7HSsSQuwRQuwUQmyzao8VQnwuhDhsefW81c5exlyq2Tn+MgLe/yHkfwIm20YvhcJe5o1OICokgKU79EtBsu5wJZ/sK+Ouc4cT42VD7GAjJyWSwspGmtq+rc1R2dDKgZP1PmHfAB0VhxDCADwNXATkADcIIXK6dasG7gb+0sNp5kkp86SUU6zaHgJWSSlHAKss+77J3F/A9W/AiAvgwAp463r4Uza8exvs/wDaek8toFDYIsjfwGUTkvl0X5lDhX/spb3DzKMf7mNobCi3nZXp9vMreicnORIp4WCZsautM+GjL9g3QN8ZxzSgQEp5VErZBiwBFlp3kFKWSym3Ao6s5SwEXrO8fw24wg2y6oPBX5t1XPU8PFAAN74LY6+AI1/COzdrSuTtm2D3O9BS521pFf2IKyem0dJu5uM9J91+7tc3HuNweQO/umSMTyyLDDZybBjI1x+u9EglQnvx1/HcqUCx1X4JMN2B8RL4TAghgX9LKZ+3tCdKKU8CSClPCiES3CKt3vgHwojzte3Sv8PxDbB/ORz4UNsMgVrg4JjLYfQlEOp9A5jCd5k0NJqMuFCW7ijl2inpbjtvVUMrf/viEGePiOd8D9ShUJxJanQIkcH+XQZyKSXrCiqZmR3nkbT29iB6ilB0+cRCXAtcKKW83bK/GJgmpfyxjb6PAA1Syr9YtaVIKU9YFMPnwI+llGuFELVSymirfjVSyjPsHEKIO4A7ABITEycvWbLEqc/R0NBAeHi4U2PtQpqJrM9nSMVGhlRsJLi1HIkftdHjqBgyk4ohs2gPjPaefC6i5HON3uT7oKCNZQXt/GVOCPEh7lk8eGVvK+tKTTw2O4SU8L7P2Z/vn69gS8YntjTT3gG/nhnCqUYzD37dzOKcQOYP9Ww8zbx587Z3MxVoSCl12YCZwKdW+z8Hft5D30eA+3s5V9dxIB9ItrxPBvL7kmXy5MnSWVavXu30WIcxm6Us3SHlF49K+c/JUv4mUsrfp0lZvNU35HMCJZ9r9Cbf8apGOezBFfKpLw+75Vq7i2tlxkMr5GMf7rN7TH++f76CLRkfWb5Xjv7Vx9LUYZb/3VQkhz24Qh4pN3pcNmCbtPFM1dPGsRUYIYTIFEIEAouA5fYMFEKECSEiOt8DFwB7LYeXA7dY3t8CfOBWqb2JEJAyEeY/DHdthTvXaUtWr18FpTu8LZ3Cx0iPDWVaRixLd5T0mNvIXqSU/Gb5XuLCArn7vBFuklDhLDnJkTS3d1BU1cj6gkpSooLJjA/ztlhd6KY4pJQm4C7gU+AA8I6Ucp8Q4k4hxJ0AQogkIUQJcB/wKyFEiRAiEkgE1gkhdgFbgI+klJ9YTv0EcL4Q4jBwvmV/4CEEJI2HW1ZASBS8fgWc2OltqRQ+xlWTUjlS0cjuEtecK97fWcqO47X87MLRRKr0Il6n00C+t7SODUe0NOpC+IZ9A/Q1jiOlXAms7Nb2nNX7MiDNxtB6YEIP56wC5rtRTN8mOl1THq9eoimPWz7UFIpCAVycm8zDy/exdEcJE9KjnTpHQ6uJP6w8SG5aFNdMtvXvqPA0IxIiCDAI3t1eQm1TO2eN8A033E5U5Hh/IGaYpjACQuG1y+HUPm9LpPARIoMDuCAnkeW7TtBmci4FydOrCyg3tvLI5WMHRd2L/kCgvx/DEyL42lImdpaX6290RymO/kJspqY8/IM05VF+0NsSKXyEqyalUtPUzleHKhweW1jZyEtfF3L1pDQmDfXdJAyDkTHJEQCMTopgSESQl6U5HaU4+hNx2dqylZ8BXrsMKg55WyKFD3D2iCHEhwc6lYLk8RX7CTAIHlwwSgfJFK7QmWLdV9KMWKMUR38jfrimPABeu4yQJs/XoFb4FgEGPy6fkMqqA+XUNrXZPW71wXJWHSzn7vkjSIgM1lFChTNMtMwA547yvXrrSnH0R4aMhFuWg9lE3s5fqdrmCq6alEpbh5kVu+1LQdJmMvPbFfvJig/ju7NVPipfZPKwGL64bw5nj1CKQ+EuEsbALcvxM7dry1bVhd6WSOFFxqZEMioxwu4CT6+sL6SwspFfX5ZDoL96DPgqwxN8M+pdfWP6M4lj2Zn3GLQ3acqj5pi3JVJ4CSEEV05KZfuxGooqe8+6XF7fwj9XHWb+6ATmjeofqd4UvoVSHP2cxvBMWPw+tNZryqO2uM8xioHJFXmpCAFL+5h1/PGTfNo6zPzq0u5VDhQK+1CKYyCQkqcpj+ZaTXnUKYP5YCQpKpizhsez7JueU5DsOF7DeztKuO2sLJ9KYaELjVUMPfau9n+hcCtKcQwUUifB4qXQWKkpj3r312lQ+D5XTUqluLqZbcdqzjhmNkseWb6PhIgg7jp3uBek8yCmNnj7RrIKX4dXLgZjmbclGlAoxTGQSJsCN70HDac05WE85W2JFB7mwrFJhAYabMZ0vLu9hN0ldfz84tGEB+mabci7SAkr74fjGzmefiXUFMFLFyjvQzeiFMdAY+h0rdJg/QlNeTQ4Hk2s6L+EBvqzYFwSK3afpKW9o6u9vqWdP316kElDo7kiL9WLEnqArS/Cjtfg7J9yNPtWuPVDaDXCywvg5G5vSzcgUIpjIDJsJtz4DtQeh5fOh20vQ2uDt6Wyj1YjFK2HjU/De9+DN66F7a+p0roOcPWkNIwtJr448O2M859fHKaqsY1HLx/nU1lW3U7hWvj4QRh5Ecz7ldaWOhn+71Otyuarl0DROu/KOABQimOgknGWtmwVFA4rfgJPjoGVP/OtNCXdlcRTU+EP6fDqxfDpL7R/8MpD8OHd8OcR8M4tkP8xdDhSon7wMSMrjuSoYJbt0JwkCsqNvLqhiOunpDM+zTdqVutCdaH2HYkfAVc9D35Wj7chI+G2TyEiWatvc3Blz+dR9MkAXuhUkDEbvv81FG+BrS9oM48t/4bMOTDte9qvMoOHvgKtRm2Z4OROra7IyZ1QeRittDwQkaJ5h427RitmlZIH4QnaenXpDti9BPa+B/vfh9A4GHc15C7SnAIG8i9oJzD4CRbmpfLC10epbGjl0Q/3ExJo4P4LB3A+qlYjLPkOSDMsehOCI8/sE5UG//cJvHENvH0TXP4vmHij52UdACjFMdARQrN7DJ0OF/5BW/vd9or2jxOZCpO/C5Nv0R7S7qK9Bcr2QOl2xuxfCXsf6EVJ5EFyHkQk9ix/2mRtu/D3UPAF7FqiLV9teR7ihmsKJPc6Lf28AtC8q5776gj3vbOLrw9X8vClOcSH+1aGVbdhNsOyO6EiX5tlx2X33Dc0Fm5eDu8shg9+CE2VMPsez8k6QFCKYzARPgTOuR9m3wuHPtGMiKsfh6/+CDkLYertMHSGY7/gzR2aUijd/u12ai+YTQBEB8ZC5gz7lERfGAJg1EXa1lwLB5bDrre1z7D6cRg6E3Kvh7FXQMjgThE+MjGC8alRrD1UwYiEcBbPHMBKdc0f4OAKWPBHyJ7Xd/+gcLjhbVj2ffj8Yc2F/fzfqpmrAyjFMRgx+MOYS7Wt8jBsfQl2vgl734XEcZoCyb0OArsFiEmpeWtZK4kTO6HNqB0PitSWmWbdrRkkUyexccch5s6d6/7PEBINk27WttrjsPsd2P02rLgXPv4ZjFwAExbB8PPBP9D91+8HXDsljT2ldfzmsrEEGAaoOXPfMlj7J5h4E0z/vv3j/APh6he1Hxgb/glN1XDZPzy3dNvPUXdpsBM/Ai56Aub/Gvb8D7a8qD18P38Y8r4DWfO0GUTpDk1RNFgCqfwCIGmc9nBOnaxtccNPN0gC4AFjfPRQbSZ19k/hxDeaAtnzrjYjCY2HaXdoNp3QWP1l6aSxSpvRHVuvxddkzYW0aRDgufTlN04fxvTMOEYlRXjsmqdh7oCGcohM1uf8J3fD+z+E9OlwyZOOzxj8DHDJXyFsCHz1BDTXwDUvQUCIPvIOIJTiUGgEhsHkW2HSLVC8WXvobX0JNltKxMcNh6w53yqJxHEefQjahRCasTx1ElzwOBz5Uvsca34P6/+ufbaZP9QUjV5UHdG8xHa+AaYWGDIa1v0dvv4r+Adry2lZczRFkpSrPbx0wuAnvKM02pq0z7/pGag+qjlhXPi73m0PjtJQoRnDQ2Lgute1ypjOIATM+7nmcPHxz+C/V8MNb0HwAPY+cwNKcShORwjNzjF0hmaMrsjXZhb9zWZgCICRF2rbqX2w/p+aZ9mW52H8NdpyWtI491xLSk3ZbvgXHPxIu/aERTDzLhgyClrq4dgGOLpG2754RBsXEgOZ52hebllzITarf6+zG09p93jri9qv99TJmuF560vw9HRtKWnOz1x/KJva4J2bobFC85Jy1mZmzfQ7tBnpsu9rsR43LXWvw8gAQ1fFIYRYAPwDMAAvSimf6HZ8NPAKMAn4pZTyL5b2dOA/QBJgBp6XUv7DcuwR4HtAZ0j0L6SUyilbD8ITBsY/T+JYuOrfcO6vtF/B21/TlrOGn09U+FyQc5x7YJs7NKPshn9ByVZNEZxzP0z93ukPs+BIGLVA20DLm1S4Fo5+pSmS/R9o7VFDIescbXkw85z+c+/LD8DGpzQ7U0c7jL5EU5qdjhYzfgRf/labie1aoi2LTlzs3GxLSvj4ATi+Aa5+SbOpuYvx12i2s7cXw8sXwuJlEJPhvvMPIHRTHEIIA/A0cD5QAmwVQiyXUu636lYN3A1c0W24CfiplHKHECIC2C6E+Nxq7N86lYxCYTfR6bDgD3DOA13LcBObPoeKpZqn2ehL7HuYtTVqzgQbn9LyIMVkwMV/0WxC3R0KbBGRpDkf5F6nPQirj8LR1ZoSObACvvmv1i9hLNmBwyEzUFvH13FZy2GkhMKvGL/7UVizA/xDNEeFGT88c0kqIhEWPq05XXzyc/jwHm1WsuAJLVDVEba+CNtfhbPu0x707mb4eZq77pvXwksXaolDE8e6/zr9HD1nHNOAAinlUQAhxBJgIdClOKSU5UC5EOIS64FSypPASct7oxDiAJBqPVahcJrQWJjzAMy6i0PvPMLIyk80v/644TDrx1pciC37jfGUttS17SVtKSZtKpz/mP0KxxZCaA/auGztwWrugJO7oFCbjaQWroRXPoCwBO06Yy7TZiOGANfugbOY2mDfUtjwFJzaQ0RAtJbaY+ptfTsfpEyE736sjf/8N9qS0JjL4YLH7Ptl35VOZAGc+2t3fBrbpE+F734Cr1+p5bea8n+Qd6MWfa4AQPSUt9/lEwtxDbBASnm7ZX8xMF1KeZeNvo8ADbZmEUKIDGAtME5KWW/peytQD2xDm5mckUNaCHEHcAdAYmLi5CVLljj1ORoaGggP983yjaDkc5WGhgbCw0IYUrGRoceXEtFwhNbAGEpTL+VEygJMAeGENh4nvfgDEk+tQcgOKuOnU5x+BfVRY3SXr7mugqGtBxlSsZG4qu0YzC20+4dRFTeNiiEzqYnJw2zQP7DPv72B5JOfklaygqC2ahpD0ylOv4KjYZMIjXTcW82vo5X04vcZevw9hDRTnL6Q40OvocPftkdTcHMZk7ffT1tgFDsm/ZkO/1C7ruPK9y+opZwRh18grmobAjN1kaMoSzqXiiFnYQpw33fal/9H5s2bt11KOaV7u56K41rgwm6KY5qU8sc2+j6CDcUhhAgHvgJ+J6VcamlLBCrRwpAfA5KllP/XmyxTpkyR27Ztc+pzrFmzRp84BDeh5HON0+SzLL+w7u/a0lFgBCTnai61/sHar86ZP3Kvd5Aj8rU3w5HVcOBDyF8JLbUQEAYjL9BmIiMugCA3e1HVFMGm52DHf6C9UTPkz7obhs8HIVz/+9aVwqpHNZtTeBKc9xttxmft1t1q1NKi15+A733p0P13y/evoVyT75s3oOKA9l0YfanFXX2uy0uIvvw/IoSwqTj0XKoqAdKt9tOAE/YOFkIEAO8Bb3QqDQAp5SmrPi8AK1wXVaFAWzbKmqttJ3fB+n9ocSHzfglTboOwOO/KFxACoy/Wto52KPoa9i/XDPT7loEhCLLP1ZTIqIt6XzoytWl1W4xlWmyOsczqvVV7YwX4+WuR/zN/pClSdxKVqiUknPo9+ORBeP8HsOUFuOiPkD7NsXQiehGeoC1hzrxLy7H2zRtazNPed7W0PRMWwYTvQLwOxbGk9ElPOz0Vx1ZghBAiEygFFgHfsWeg0PI+vwQckFI+2e1YssUGAnAlsNd9IisUFpInwDUve1uKnjEEaEoi+1wtiK14szYTOfAhHPoYhAEyz9ZmCK3GbkrhJDRXn3lO4QfhidoWlarlB4seqs0AonSu4ZE+FW77Ava8o7krv3Q+jL9WC847uEIzpNuTTkRPhLAk4JyoxaXkf6zFq6z7mxankz5Dm4WMvdJ2ksUe8G9v0AJsawq1DL81hVBzTHtvPAGB4ZrHXmgshMR2e42xvI85/VhQhK4KRzfFIaU0CSHuAj5Fc8d9WUq5Twhxp+X4c0KIJDQ7RSRgFkLcC+QAucBiYI8QYqfllJ1ut38SQuShLVUVAQ7kGVAoBiB+Bhg2S9su/L02SzqwXJuNrHpUmzF0KoSYDC3hZXiS5t0VkaS1RyRDWLx3Pbf8/LRf76Mv1QI2N/xLC6LMuwmm3+k9uWzhH6TlRBt7haaUdy3RPO0+vFsz4OdcrimRjHO0/vWlFoVQ9K1ysLye1VIH663OHZag/Z0yZmszmvYmLSVKc7X2Wn0EmmqgtZcaNX4B3yqbS57UzuXOj+/Ws3XD8qBf2a3tOav3ZWhLWN1ZB9hUl1LKxe6UUaEYUFhHz8//jVYAKyjSRioYHyYoXIu5mXQzHPpUe/XB5ZouIpLgrHu1YMfSHbDzv7DnPc0uEhILbQ3Q0fZtfz9/bSYXkwGpkymoMTN8ynyIydTaguw0lHeYNO++ToXS9dqtzYHZj72oyHGFYqAihBbQ1l+JHqrlGOsvnFYC4A+Q/xEUrNJmcjEZmmKIzYTItNOSKZasWcPwMXMdv57BX8t4HT7EbR/BXpTiUCgUCncTEKwVGxt3tbcl0YV+NH9VKBQKhS+gFIdCoVAoHEIpDoVCoVA4hFIcCoVCoXAIpTgUCoVC4RBKcSgUCoXCIZTiUCgUCoVDKMWhUCgUCofQLa26LyGEqACOOTk8Hi2Nu6+i5HMNJZ9rKPlcx5dlHCalPCM0fVAoDlcQQmyzlY/eV1DyuYaSzzWUfK7TH2TsjlqqUigUCoVDKMWhUCgUCodQiqNvnve2AH2g5HMNJZ9rKPlcpz/IeBrKxqFQKBQKh1AzDoVCoVA4hFIcCoVCoXAIpTgsCCEWCCHyhRAFQoiHbBwXQoh/Wo7vFkJM8qBs6UKI1UKIA0KIfUKIe2z0mSuEqBNC7LRsD3tKPsv1i4QQeyzX3mbjuDfv3yir+7JTCFFvqW9v3cej908I8bIQolwIsdeqLVYI8bkQ4rDlNaaHsb1+V3WU789CiIOWv98yIUR0D2N7/S7oKN8jQohSq7/hxT2M9db9e9tKtiIhxM4exup+/1xGSjnoN8AAHAGygEBgF5DTrc/FwMdotdBnAJs9KF8yMMnyPgI4ZEO+ucAKL97DIiC+l+Neu382/tZlaIFNXrt/wDnAJGCvVdufgIcs7x8C/tiD/L1+V3WU7wLA3/L+j7bks+e7oKN8jwD32/H398r963b8r8DD3rp/rm5qxqExDSiQUh6VUrYBS4CF3fosBP4jNTYB0UKIZE8IJ6U8KaXcYXlvBA4AqZ64thvx2v3rxnzgiJTS2UwCbkFKuRao7ta8EHjN8v414AobQ+35ruoin5TyMymlybK7CUhz93XtpYf7Zw9eu3+dCCEEcB3wlruv6ymU4tBIBYqt9ks488FsTx/dEUJkABOBzTYOzxRC7BJCfCyEGOtZyZDAZ0KI7UKIO2wc94n7Byyi539Yb94/gEQp5UnQfiwACTb6+Mp9/D+0GaQt+vou6MldlqW0l3tY6vOF+3c2cEpKebiH4968f3ahFIeGsNHW3U/Znj66IoQIB94D7pVS1nc7vANt+WUC8C/gfU/KBsyWUk4CLgJ+JIQ4p9txX7h/gcDlwP9sHPb2/bMXX7iPvwRMwBs9dOnru6AXzwLZQB5wEm05qDtev3/ADfQ+2/DW/bMbpTg0SoB0q/004IQTfXRDCBGApjTekFIu7X5cSlkvpWywvF8JBAgh4j0ln5TyhOW1HFiGtiRgjVfvn4WLgB1SylPdD3j7/lk41bl8Z3ktt9HH29/DW4BLgRulZUG+O3Z8F3RBSnlKStkhpTQDL/RwXW/fP3/gKuDtnvp46/45glIcGluBEUKITMuv0kXA8m59lgM3W7yDZgB1ncsKemNZE30JOCClfLKHPkmWfgghpqH9bas8JF+YECKi8z2aEXVvt25eu39W9PhLz5v3z4rlwC2W97cAH9joY893VReEEAuAB4HLpZRNPfSx57ugl3zWNrMre7iu1+6fhfOAg1LKElsHvXn/HMLb1nlf2dC8fg6heVz80tJ2J3Cn5b0AnrYc3wNM8aBsZ6FNp3cDOy3bxd3kuwvYh+YlsgmY5UH5sizX3WWRwafun+X6oWiKIMqqzWv3D02BnQTa0X4F3wbEAauAw5bXWEvfFGBlb99VD8lXgGYf6PwOPtddvp6+Cx6S73XLd2s3mjJI9qX7Z2l/tfM7Z9XX4/fP1U2lHFEoFAqFQ6ilKoVCoVA4hFIcCoVCoXAIpTgUCoVC4RBKcSgUCoXCIZTiUCgUCoVDKMWhULgBIUSHOD0Dr9uyrgohMqyzrCoU3sbf2wIoFAOEZillnreFUCg8gZpxKBQ6Yqmt8EchxBbLNtzSPkwIscqSkG+VEGKopT3RUutil2WbZTmVQQjxgtDqsXwmhAjx2odSDHqU4lAo3ENIt6Wq662O1UsppwFPAX+3tD2FlmY+Fy1Z4D8t7f8EvpJassVJaNHDACOAp6WUY4Fa4GpdP41C0QsqclyhcANCiAYpZbiN9iLgXCnlUUuiyjIpZZwQohItJUa7pf2klDJeCFEBpEkpW63OkQF8LqUcYdl/EAiQUj7ugY+mUJyBmnEoFPoje3jfUx9btFq970DZJxVeRCkOhUJ/rrd63Wh5vwEtMyvAjcA6y/tVwA8AhBAGIUSkp4RUKOxF/WpRKNxDiBBip9X+J1LKTpfcICHEZrQfajdY2u4GXhZCPABUAN+1tN8DPC+EuA1tZvEDtCyrCoXPoGwcCoWOWGwcU6SUld6WRaFwF2qpSqFQKBQOoWYcCoVCoXAINeNQKBQKhUMoxaFQKBQKh1CKQ6FQKBQOoRSHQqFQKBxCKQ6FQqFQOMT/A5SaqfjeVFDBAAAAAElFTkSuQmCC\n",
|
|
"text/plain": [
|
|
"<Figure size 432x288 with 1 Axes>"
|
|
]
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA80ElEQVR4nO3deXxU1fn48c+TfV9IIAQiJCCyFBAIAu5QrOJScUHFVqUqpdq61H77/Wnt/u3yta3tt7alUqzUYlVqtVRqsVgtSFVkU0BWRUBJwhKWLJNMyPb8/rgXGMIkZJK5SSY879drXjNz77l3nrmE+8w959xzRFUxxhhjmorq7ACMMcZ0TZYgjDHGBGUJwhhjTFCWIIwxxgRlCcIYY0xQMZ0dQDhlZ2drfn5+m7atqqoiOTk5vAGFgcUVGosrNBZXaLpjXGvXrj2gqj2DrlTVbvMoLCzUtlq6dGmbt/WSxRUaiys0FldoumNcwBpt5pxqVUzGGGOCsgRhjDEmKEsQxhhjgupWjdTB1NXVUVRURE1NTYvl0tPT2bJlSwdF1XpdIa6EhATy8vKIjY3t1DiMMR3L0wQhIlOAx4Bo4Peq+kiT9ZnAPGAgUAPcoaob3XUPADMBBd4HblfVls/yQRQVFZGamkp+fj4i0my5yspKUlNTQ9295zo7LlXl4MGDFBUVUVBQ0GlxGGM6nmdVTCISDcwGLgeGATeLyLAmxR4G1qnqSOA2nGSCiPQF7gPGqupwnAQzvS1x1NTUkJWV1WJyMM0TEbKysk55BWaM6X68bIMYB2xX1R2qWgssAKY2KTMMeB1AVbcC+SKS466LARJFJAZIAkraGoglh/ax42fM6cnLKqa+wO6A90XA+CZl1gPXAW+KyDigP5CnqmtF5FHgE8APvKqqrwb7EBGZBcwCyMnJYdmyZSesT09Pp7Ky8pTBNjQ0tKpcR+sqcdXU1JxwbH0+30nHuiuwuEJjcYWmM+NqVMVfD5W1iq9Wqaxzn2uVI7W1QPjj8jJBBPvZ2XTyiUeAx0RkHU47w3tAvds2MRUoAMqAv4jILar6p5N2qDoXmAswduxYnThx4gnrt2zZ0qo6/M6u629OV4krISGB0aNHH3u/bNkymh7rrsDiCo3FFZpwxXWkvoFyfx0V/jr3uZ7D1bUcqqp1n+s4XFXLoepaDrvLDlfX0dAYfP6e9PgoHrur/XE15WWCKALOCHifR5NqIlWtAG4HEKceY6f7uAzYqaql7rq/AucBJyWIrq6srIxnn32WL3/5yyFtd8UVV/Dss88SHR3tUWTGmHBQVUrKa9i+38dB3xHK3ZN++bGTf91Jy2rqGpvdX3SUkJkUR4/kWDKT4jizVwqZyXH0SIpznt3lPZLjjj2vevs/nnw3LxPEamCQiBQAxTiNzJ8LLCAiGUC120YxE1iuqhUi8gkwQUSScKqYJgNrPIzVM2VlZfz2t789KUE0NDS0ePJfvHgxQJeoXjLGQGOjUlrdyL+37uPDfT4+3O/jw32VbN/vo6q24aTyqfExpCXGkpYYS3piDAXZyaQnxp7wSAt4nekmgNT4GKKiQmv386qd0LMEoar1InIPsASnF9I8Vd0kIne56+cAQ4H5ItIAbAbudNetFJEXgHeBepyqp7ntjen7f9/E5pKKoOtOdcJuzrA+aXz3s59qdv1DDz3ERx99xKhRo4iNjSUlJYXc3FzWrVvH5s2bueaaa9i9ezc1NTXcf//9zJo1C4D8/HzWrFnD3r17ueGGG7jgggt4++236du3Ly+99BKJiYlBP++JJ55g7ty51NbWcuaZZ/L000+TlJTEvn37uOuuu9ixYwcAjz/+OOeddx7z58/n0UcfRUQYOXIkTz/9dMjHwJjupLFRKTrs54N9lU4S2F/Jh/t8bN/vw1/XAMud36q9UuMZlJPCDWPPYFBOCmf2TCEnLYH0xFhSE2KIiY78+5A9vQ9CVRcDi5ssmxPwegUwqJltvwt818v4OsIjjzzCxo0bWbduHcuWLePKK69k48aNx+4pmDdvHj169MDv93POOedw/fXXk5WVdcI+PvzwQ5577jmeeOIJbrzxRl588UVuueWWoJ933XXX8cUvfhGAb33rWzz55JPce++93HfffVx88cUsXLiQhoYGfD4fmzZt4kc/+hFvvfUW2dnZHDp0yNuDYUwXoaqU+o7w8cFqdh2o4uOD1ew8WMWuA1V8VOo7oQooJy2es3JSmT7uDLSshKsuLOTMXilkJMV14jfoGN3+TupALf3S76jG4HHjxp1ww9mvfvUrFi5cCMDu3bv58MMPT0oQBQUFjBo1CoDCwkJ27drV7P43btzIt771LcrKyvD5fFx22WUA/Pvf/2b+/PkAREdHk56ezvz585k2bRrZ2dkA9OjRI1xf05hOp6rsrzzCzgNVfHywil0Hq53nA85zYLVQdJSQl5lI/6xkJgzIYlCvFAblpHJmrxTSE4+PILBsWSlj80+f/yenVYLoCgLHbF+2bBmvvfYaK1asICkpiYkTJwa9IS0+Pv7Y6+joaPx+f7P7/8IXvsDf/vY3zj77bJ566qkWu+Spqt3jYCJWTV0De8tr2FtRw76KmmOvS8r8zpXBwaoTrgRiooQzeiSRn5XEuIIe5Gcl0T87mYKsZPpmJhLbDaqEws0ShMdSU1ObbWguLy8nMzOTpKQktm7dyjvvvNPuz6usrCQ3N5e6ujqeeeYZ+vbtC8DkyZN5/PHH+epXv0pDQwNVVVVMnjyZa6+9lgceeICsrCwOHTpkVxGmzWrrG/mo1MfWvRVs2+vjSH0DCbHRJMREkxgXdex1fGwUibHRzvvYaBJinXWJsc66hNhoKo4oG4vLnRN/RQ373JP/3oojx16X++tOiiEpLprc9ATys5I5b2A2BdlJ9M9KJj8rmT4ZCd2iXaAjWYLwWFZWFueffz7Dhw8nMTGRnJycY+umTJnCnDlzGDlyJIMHD2bChAnt/rwf/OAHjB8/nv79+zNixIhjyemxxx5j1qxZPPnkk0RHR/P4449z7rnn8s1vfpOLL76Y6OhoRo8ezVNPPdXuGEznq6lroKTMz8GqWjKTYslOiSc9MTYsV4xHu3Vu21vB1r2VbN1Tyba9lXxU6qPe7acfFx1FfGwUNXUN1DUE77t/SkvfPPYySiA7JZ7e6Qn0c68AeqcnkJOWQO+0BHqnx5OTlkBqgg0oGU6WIDrAs88+G3R5fHw8r7zyStB1R9sZ4uPj2bhx47HlX//611v8rLvvvpu77777pOU5OTm89NJLJy2fMWMGM2bMaHGfpuuprW9kT7mfosN+dh+qdp4PO89Fh6vZV3HkpG1io4Ws5HiyU+PIToknOyWe6kO1bI/eQc9U5/3R54zEWKKihMqaOj7YV8kWNwlsdZNCZU39sf32zUhkSO9ULhnWi8G90xjaO5X87ORjVTYNjUpNXQM1dQ346xqoqWukpq6BI/UN+Gud1zX1znJ/XQNH6hrYteMjLigcQU6akxR6psTbr/9OYAnCmC6o6ki9U6VSXkNxmf/Yib/okPO8t6KGwJtqo6OE3PQE8jITuWhQT/Iyk8jLTCQ7NZ6y6loO+Go54DvCgcojzrOvlq17KimtrGPxzpOHk4+OEtISYjhcfbwaJzU+hsG9U7n67D4MyXUSwVm9U0k7xa/26CghOT6G5PjWn26WNXzCxOG9W13eeMMSRIT6yle+wltvvXXCsvvvv5/bb7+9kyIyraGqlPvr2HO0Tr28xnld7mdvxRH2lvvZU15zwi90ABHITUsgr0cSEwZmkZeZxBmZiccSQW562+rXly5dyujx53PAd4TSSjeJuI9DVXXkZSYyOCeVIbmp9M1ItE4NpxlLEBFq9uzZnR3CaeWA7wiHq2qd6pH6Bvy1DW7ViFtd0qT6JLC6pKa+gR3Ffr6/Zhl7yv0nDbMgAj1T4slNT6Ag22lczUlLIDc9gd7pznNueiJxMeGvYhERMpLiyEiK48xeYd+9iXCWIIxpQlXZeaCK1bsOsXrXYVbvOsTHB6tbvX1MlLg9co730JFG+FReGpcM7UXv9ES3YdU5+fdMjbculqZLsgRhTnv1DY1s3VvJqp2HjiWFAz6nkTczKZax+T34/Ph+5KYnHuuWmRjQRTM+JprEOPd9TFTQqh5nFNAxHf3VjGkXSxDmtFNT18C63WWs3nmIVbsO8d4nZfiOOHX+fTMSuXBQNufk92BcQSYDe6ZYvbs5bVmCMKeF4jI/z678mHd2HGJDUdmxvvmDc1K5ZnQfzsnvwTn5PeiTEXwQRGNOR5YgupiUlBR8Ph8lJSXcd999/OEPfzipzMSJE3n00UcZO3ZsJ0QYWT45WM1vl23nxXeLUIUReenccX4B5+T3YGx+5mkx4JoxbWUJoovq06cPL7zwgs0H0UY7Sn3MXvoRf1tXTHSUcPO4ftx18UC7QjAmBKdXgnjlIdj7ftBViQ31EN2Gw9F7BFz+SLOrH3zwQfr3739swqDvfe97iAjLly/n8OHD1NXV8cMf/pCpU6eesN2uXbu46qqrWLFiBX6/n9tvv53NmzczdOjQFgfrA+du6tWrV+P3+5k2bRrf//73AVi9ejX3338/VVVVxMfH8/rrr5OUlMSDDz7IkiVLEBG++MUvcu+994Z+HLqIEl8jX13wHovWlxAbHcWMc/P50sUDyElL6OzQjIk4p1eC6ATTp0/nq1/96rEE8fzzz/PPf/6TBx54gLS0NA4cOMCECRO4+uqrm20Mffzxx0lKSmLDhg1s2LCBMWNa7g3zox/9iB49etDQ0MDkyZPZsGEDQ4YM4aabbuLPf/4z55xzDhUVFSQmJjJ37lx27tzJe++9R0xMTMTOCbF1bwW//vd2Fm/wkxBby8wLBzDzwgJ6pVpiMKatPE0QIjIFeAxnRrnfq+ojTdZnAvOAgUANcIeqbhSRwcCfA4oOAL6jqr9sV0At/NL3ezQfxOjRo9m/fz8lJSWUlpaSmZlJbm4uDzzwAMuXLycqKori4mL27dtH797BhxZYvnw59913HwAjR45k5MiRLX7m888/z9y5c6mvr2fPnj1s3rwZESE3N5dzzjkHgLS0NABee+017rrrLmJinD+FSBvNdWNxOb/+94cs2bSPlPgYrhwQy/c/dzFZKfGn3tgY0yLPEoSIRAOzgc8ARcBqEVmkqpsDij0MrFPVa0VkiFt+sqpuA0YF7KcYWOhVrF6bNm0aL7zwAnv37mX69Ok888wzlJaWsnbtWmJjY8nPzw86D0Sg1na13LlzJ48++iirV68mMzOTL3zhC9TU1DQ790OkzgmxbncZv379Q17fup/UhBjumzyIO87PZ92qty05GBMmXt6+OQ7Yrqo7VLUWWABMbVJmGPA6gKpuBfJFJKdJmcnAR6r6sYexemr69OksWLCAF154gWnTplFeXk6vXr2IjY1l6dKlfPxxy1/toosu4plnngGcGeM2bNjQbNmKigqSk5NJT09n3759x0aLHTJkCCUlJaxevRpw5o2or6/n0ksvZc6cOdTXO/cBdOUqpsZGZe3Hh5gxbxXXzH6LNR8f5r8+cxZvPvhpvvaZs6xHkjFh5mUVU19gd8D7ImB8kzLrgeuAN0VkHNAfyAP2BZSZDjzX3IeIyCxgFjhDWjedQS09Pb1VPYEaGho86zHUr18/ysvL6d27NykpKUydOpUbb7yRMWPGMGLECM466yx8Pt+xz6+srMTn89HY2EhDQwO33HILd999N8OHD2fEiBEUFhZSVVUVNN4BAwYwfPhwhg4dSn5+PuPHj6empoYjR44wb948vvzlL1NTU0NCQgKLFi3ipptuYuPGjQwfPpzY2FhmzJjBl770pZP2W1NTc8Kx9fl8Lc5W1xqqir8eKmqVylo96fn4a6eMr1ZpUEiNhRvOiuXT/WJJjC7mvZXFYY3LCxZXaCyu0HgVl6i2cTKPU+1Y5AbgMlWd6b6/FRinqvcGlEnDaaMYDbwPDAFmqup6d30cUAJ8SlX3cQpjx47VNWvWnLBsy5YtDB069JTxdtSc1KHqKnE1PY7O0BETW9xGVSmtPMKOA1XsDHiUlPk56KvlUFUttQ2NQbdNiY8hKyWOrOQ4eiTHk50SR4/kOPr1SOLqUX1Iigv+26Y1cXUGiys0Fldo2hOXiKxV1aA3VXl5BVEEnBHwPg/nZH+MqlYAtwOIUxG+030cdTnwbmuSg+k85dV17DjgY+eBKnYdqDqWEHYdOHFi+LiYKPr3cIanHpabRo+UOLKT4+mRHEdWijOJTY9kJxEkxEZ34jcKszo/Cf49UFECMQnHH1E2QJ/p2rxMEKuBQSJSgNPIPB34XGABEckAqt02ipnAcjdpHHUzLVQvne7Gjx/PkSMnzhz29NNPM2LECM8+s76hkRffLWLRhiP8avNb7DxQdcKkMlECeZlJFGQnc05+Dwb0dOYDLshOpk9GItFRkdcg3i7F78KCzzGhcg+sbLIuOs5NFvEQk+g8xyacmERi4iG5J+SdA/0mQI8BzvjgxnQAzxKEqtaLyD3AEpxurvNUdZOI3OWunwMMBeaLSAOwGbjz6PYikoTTA+rkCvHQY4nInjqnsnJl0zOON45WQy7/oJQfvLyZD/f7yIgXhvSNYsrwXAqykyjITqEgO5l+PZI8mbcgIr3/Arz0FUjuybazvsLgwWdBXQ3UBz6OQJ3fea4/+lzjlKs+6Lze9R9Y6w65ktwTzhjvJIszJkDu2RBjjfPGG57eB6Gqi4HFTZbNCXi9AhjUzLbVQFZ7Y0hISODgwYNkZWV1yyThNVVl7/5StpbW8MDiVfTPSmLurYXE7t/CpEnndnZ4XVNjIyz9Ifzn59DvXLjxafas2cTgwolt39+BbfDJO7B7JXyyAra+7KyLSYA+Y6DfeCdhnDEOkiLrXhbTdXX7O6nz8vIoKiqitLS0xXJHe/Z0NZ0ZV2OjUlFTz+b9fp58t5yHrxjCjPPyiY+JZlnp1k6Jqcs7Ugl//RJs+weMvhWu/EX7f+FHRUGvoc5jrDulbOU+2P0OfLLSeX7719D4f8667MHHE0b++ZCZ377PjyQl78H6BZB1JuSOgt7DIdbD8bfq/LB3I+xZ5wzjU9fyMDgt6jMKzv1KuCILi26fIGJjYykoKDhluWXLljF69OgOiCg0nRFXfUMjz676hF/86wMq/HXcdE4/Ft0/imy7Aa1lh3fBczdD6VaY8hMY/yXv2gtSc2DYVOcBUFsNJe86VxefrITNL8G78511o26BS74HKT29iaWr2LMe/jgVan2gbucIiYaeg51k0WeUUyXXewTEJYe+/9oq0sq3wMptULLOSQql245/VmIPSEhve/xticlj3T5BmNAEtjOcOyCLb181jGF90jo7rK5v11vw/K3QWA+ffwHOnNyxnx+XBPkXOA9wqqVKt8L65+Cd38KWv8Okh+GcmW0blLKrK90GT18L8alw91tOYj56Et+zHra/BuufdcpKFGQNchPGKCdp5I50tj3qSCXs2eBse3QfBz5gjLrdspN7OdsPufJ48knr2+06EHTDvxTTFh+V+vjxP7bw+tb99M9K4ne3FnLpsBxrt2mNtU/BP/4LMgvg5gWQfWZnR+RUS+UMg0t/4FR1vfL/4J8POlcVV/z0eCLpDg7vgvnXOFcLMxZBhtu7Pj0Phl7lvFaFyj3Oib5knfO8czlsODrkmzjVUllnwsHtzgP3HrHUXCcJDLuG9w9GM+LSWyG1d7dLBsFYgjjNlVfX8djrHzJ/xS4SY6NPaGcwp9BQD0sehlW/g4GTYdo8SMzo7KhO1vMsuHWh07D9z4fhqSth+DQneUS6ihL449VQVw23L4asgcHLiUBaH+cx+PLjyyv3Hb9KKFkHhz6C7LNg5I3Hry5Sj4/+c3DZMkjL9fALdS2WIE5T9dVlPLfuEL94bTtl/jqmn9OP/7r0LGtnaK3qQ/DC7bBjGZx7D1zy/a5ddSMCQz/rJLK3fglv/hK2vcIZZ0yD+vMis6ts1QHnyqH6INy2CHI+Ffo+UnMg9VI469Kwh9cddOG/aOOJkvc49OrPSN/1CldqEmcmjqb/eVfQZ8xQSI7Ak0QwdTXg2+v8OqzcA759ULkX/Ifd3i1unXNbGxRLP4DnboKy3TB1Noy+JbzxeykuyWmLOPtmWPIwA7f9ER5/Cy7/ace3m7SHv8xpcyj7GG55EfIKOzuibskSxOlAFT56nfr//JKYj/9DjCbxl5gruSAvlgmHVyJvfgPe/Aak94MBF8OAiVBwEaT06uzIT6Tq1DdX7nFO+D43AVTucxPCXs4/XATLfCdvGxXjNEL6Dx9f1mOgkyyO9m7JPRsSM1uO4cN/wQt3OHc4f+Fl54a1SNSjAG5+jg0vPsrI4mfgT9fBkKvgsh9DZv+27bPqoNMwXrrVqcPvWwjDrw9/XX1tFTx7I+zf4rT5dKf2lC7GEkR31lAHm/4Gbz0G+97nMD14ov5zSOHt3HfFGJLjY5yT7sHtTlXJjmWwZRG897Szfc5wKHATRv/zID6lk75HPWxaeOx7nCAq1mkwTO0NWWeyP3YAfYeMgZTeTuNiao7zOinLabj1lbp1zu85dc5Fa2DTX4/vLzPfTRajjvdySerhHKcVv4F/fcepypj+3PHG0Ah2KGssTL3X+W7LH4XZ4+CCr8H59wW/f0DVqdo5mghKtzo9iEq3QlXAvUZRsdD4W1j9JFzxM+d+hHCoq4EFn4Oi1XDDUzDokvDs1wRlCaI7qq2Cd5+GFbOh/BP2xOXz87ovsTnrUn44rZAx/QJ+JYtA9iDnMe6L0NjgNNjteMNJGKt/D+/Mdn6B551zLGFIY33HfI/3/gRv/wbKP3FuALv8Z05DZKqbABIzT/iF+uGyZfS9aGLz+0zp6ZxUAk8sVQePd2U82li5+aXj69P7OYmmaDUMvRqundMl+6y3WUw8XPhfMPImePVbsOzHsO5P8JkfOI3upducX+tHE4E/YM6Q+DToOQTOmuI89xriPKfmOj80Xvs+/O5Cp3vtpIdPfYXWkoa64+0+1zx+/B4Q4xlLEN2JrxRWzYXVT4D/MAd6jOH78jlerRrJVz59Fj++eOCpx0mKinaqBvoWwoVfc+4M3b3SvcJ4A974CbzxCOfFpELlVBh2jZM0wtnIWXXA+R6r5jpVQv3OdbpmDrrMmxFQk7Oc+vfAOnj/4YAukeuck+Okbzkn0u46Cmt6nvOrfOwdsPi/4S8zjq9LyHDu5B52tZMAeg6GnkNb7u5Z+AUnoS79kfNDY+OLzg17o24J/Rg2NsDf7oZti+GKR2HU5069jWk3SxDdwaEdzq/sdc9A/RGqB1zGo77LmPdJDmP7Z/KP60dwZq82zikRm+hUMQ2Y6Lz3H4ad/+HgG0/Se/Mi5xd+QjoMvtL5RTdwkvOLtK3fY8VsZ5/1Nc4+z7/fGTaioyVmnvi9TycFF8Fdb8IHS5x2m55DnPaotrQlJPWAK38OY2Y4SWfRvc59I1f8zPkR0hqq8PID8P5fYPJ3nStd0yEsQUSykvecevnNL0FUDI0jp/Ni/DV85606oqOEH1wzhM+P60dUOIfYTsyEYVezdX8avS84Fz5a6nz+tn84d6rGpzn9zIdNdbpUxrZiHKkm34ORN8F59zn9903niI49fpNZOOSOhDv+CRueh399G56YDGNudU74ydnNb6fqVHu9+0fn6u3Cr4UvJnNKliAi1btPw6J7nBPyeffxQf7n+fqS/WwoKueSob34n6nD6ZPh4SBl4FwpDJ7iPOprYecbsPlvsOVl5w7VuBSnbvpT18CZl5zY6On2rOKtx5w7Wt3vwfi7TqsbkU4rInD2Tc4PiDd+AivnOD8KPv1tKLw9+H0kb/zEaUAf9yWnnOlQliAi0e7V8I+vwYBJ1Fz3B3715j5+94ftZCbF8pvPjebKEbkdP0RGTBwM+ozzuOqXzkn/aLLY+ALEJjs3Iw27Bhpq4a1fOT2SUnrDZ/7Hqa9uz0BnJnIkpMFlPzo+BMjir8PaPzrVTv0DhpB/+zew7H9h1OdhyiOnxdAWXY0liEhTuc8ZFC6tD6vP+Tn/b856dh6o4obCPL555VAykrrAzW7Rsccbfa/8P2fCm80vOQPGbVrolMk+y7nJbMQNbW+zMJGt1xC47SXnb2PJN+EPU5zqxc/8D7klS+CD3zpVlZ/9VfftGNDFeZogRGQK8BjOjHK/V9VHmqzPBOYBA4Ea4A5V3eiuywB+DwzHGTXrDneCodNXfa3Ts6SmnDcvfo5b/riVfj2S+NOd47lgUAv1uJ0pOsZpuB44yel98snbzoinBRPtP71xrgo+dY1z5fmfX8Dbv4Kt/+Cs2ioYdClc9/uuPYRJN+fZkReRaGA2zrShRcBqEVmkqpsDij0MrFPVa0VkiFv+aF/Dx4B/quo0EYkDkryKNWIs+QZ8sgLfZ+fy1VfqGN43jb986TwS4yJkYL3oGKeHjDFNxSXD5G873Vf/9R0OlpaSfeP8yBwjqhvx8ifcOGC7qu5Q1VpgAdD0zpZhwOsAqroVyBeRHBFJAy4CnnTX1apqmYexdn3vPu30JT/vPr6zfTBl1XX89PqzIyc5GNMaWQNh+jNsHPGwtzPBmVaRoxPSh33HItOAKao6031/KzBeVe8JKPNjIEFVvyYi44C3gfFAAzAX2AycDawF7lfVqiCfMwuYBZCTk1O4YMGCNsXr8/lISemkoSRa4PP5yG0sYfR736As41P8Kfeb/OLdeq4eGMt1gzrv11VXPl4WV+tZXKHpjnFNmjRpraqODbpSVT15ADfgtDscfX8r8OsmZdKAPwDrgKeB1TgJYSxQj5NQwKlu+sGpPrOwsFDbaunSpW3e1ktvLlmo+ugQ1f8brhWH9uqEH7+ml/x8mdbU1XdqXF31eFlcobG4QtMd4wLWaDPnVC9bf4qAwNHM8oCSwAKqWgHcDiBOv8yd7iMJKFLVlW7RF4CHPIy1a2qo41ObfuLcvXznqzyybD97K2p48e7zbEIfY4znvGyDWA0MEpECt5F5OrAosICIZLjrAGYCy1W1QlX3ArtFZLC7bjJOddPpZck3ySjfDFN/wzv+vjyz8hPuOL/gxMH2jDHGI55dQahqvYjcAyzB6eY6T1U3ichd7vo5wFBgvog04CSAOwN2cS/wjJtAduBeaZw23nsGVv2O3XnXkD34Wh56bDn9eiTx9UsHn3pbY4wJA087GKvqYmBxk2VzAl6vAAY1s+06nLaI00/xu87gZAUXseOM23j6tQ/YdbCaZ7843notGWM6jN2p1NX4SuHPt0JKDkx7io8q4ff/2cHN4/px3sAuejOcMaZbslsUu5KGOvjLF6D6ANyxhNr4TOa9f4ReqQl844ohnR2dMeY0YwkiHFTDM5DYq9+Gj9+Ea+dCn1H89rUPKPIpT84YTlpCbPv3b4wxIbAE0V7P3+bMtFZwoTO5TMFE527QUBPG+gWw8nGY8GU4+ya27a1k9tLtTMiNZvLQHA8CN8aYllmCaI/9W52RKPuOdaam3PJ3Z3laHgy42E0YFzvzGbekZB38/X7IvxA+8z/UNzTy/15YT1pCLJ8fav9ExpjOYWef9lj5OMQkwOf+DElZzpSZO99w5m/ettiZAhScuXuPJoz+5zvj4R9VdQD+fAskZTvzAUfHMm/5R6wvKufXN48m9fAHnfDFjDHGEkTbVR10qoVG3nh8ysSsgc5j7B3Q2Ah7NzjJYucbzoQoK+eARDtz8Q642Lm6eOMn4NsPdy6B5Gx2Haji569+wCVDc7hqZC5vvGEJwhjTOSxBtNXaP0B9jdNmEExUFPQZ5Twu+CrUH4Hdq44njP/8HJb/zCl7zRzoM5rGRuXBFzcQFxPFj64d3vGzwhljTABLEG1RXwurnoABk6DX0NZtExPvNGQXXAh8G2rKYddb0HAEPnUtAM+u+oSVOw/xk+tHkJOW4F38xhjTCpYg2mLz38C3F6b+pu37SEiHIVcce1tS5ueRV7ZywZnZ3Dj2jBY2NMaYjmF3UodKFVbMhqxBMHDyqcu3apfKwwvfp6FR+d/rRljVkjGmS7AEEapP3oE962DC3WGbU/lv64pZtq2U/75sMGf0sJlVjTFdgyWIUL0zGxIy4OzpYdldaeURvv/3zYzpl8GM8/LDsk9jjAkHSxChOLwLtv4Dxt7uTLIeBt9btInqIw38dNpIoqOsaskY03VYggjFyrkgUXDOF8Oyu9e37OMf7+/h/ksGcWav1LDs0xhjwsUSRGvVVMC782HYNZDeNyy7XLatlNSEGGZdNCAs+zPGmHDyNEGIyBQR2SYi20XkpDmlRSRTRBaKyAYRWSUiwwPW7RKR90VknYis8TLOVln3DNRWNn9jXBsUl/k5IzOJ2GjL08aYrsez+yBEJBqYDXwGKAJWi8giVQ2cW/phYJ2qXisiQ9zygX1HJ6nqAa9ibLXGBmeYjDPGQ15h2HZbUuYnL9N6LRljuiYvf7qOA7ar6g5VrQUWAFOblBkGvA6gqluBfBHpemNbb3vFaaCecHdYd1t82E9eZmJY92mMMeEiqurNjkWmAVNUdab7/lZgvKreE1Dmx0CCqn5NRMYBb7tl1orITuAwoMDvVHVuM58zC5gFkJOTU7hgwYI2xevz+UhJSQm6btR73yShZh8rx/8OjQrPnNDVdcqXX6/mpsFxXF7Q/GRALcXVmSyu0FhcobG4QtOeuCZNmrRWVccGXamqnjyAG4DfB7y/Ffh1kzJpwB+AdcDTwGrgbHddH/e5F7AeuOhUn1lYWKhttXTp0uArStapfjdN9c3H2rzvYDaXlGv/B1/Wl9eXtC2uTmZxhcbiCo3FFZr2xAWs0WbOqV6OxVQEBA4qlAeUBBZQ1QrgdgBxxpfY6T5Q1RL3eb+ILMSpslruYbzBvfM4xCbDmNvCutuSMj8AfTJsUD5jTNfkZRvEamCQiBSISBwwHVgUWEBEMtx1ADOB5apaISLJIpLqlkkGLgU2ehhrcJV74f0XYPTnITEjrLsudhNE3wxrgzDGdE2eXUGoar2I3AMsAaKBeaq6SUTuctfPAYYC80WkAdgM3OlungMsdAetiwGeVdV/ehVrs1Y/CY31MP6usO+6uMxPXHQU2SnxYd+3McaEg6fDfavqYmBxk2VzAl6vAAYF2W4HcLaXsZ1SnR/WPAlnTXFmiQuz4sN+cjMSiLLhNYwxXZTdodWcDc9D9UE4N3w3xgUqKfNb9ZIxpkuzBBGMqtM4nTMc8i/05CNKymroYwnCGNOFWYIIZsdSKN3iDKvhweQ9tfWN7KussSsIY0yXZgkimHceh+SeMPx6T3a/r6IGVevBZIzp2ixBNFX6AXz4KpwzE2K9uUeh6LDbxdWG2TDGdGGWIJpaOQei42DsHZ59xPGb5CxBGGO6LksQgaoPwfrnYMSNkNLLs485epNcbrrdRW2M6bosQQR6949QVw0Twn9jXKCSMj/ZKfEkxIZn4D9jjPGCJQiXNNY7U4oWXAS9R3j6WcVlfmt/MMZ0eZYgXD1L34bKkrDOGNec4jI/fW2QPmNMF2cJAkCVvKJF0GMgDLrM449Su4vaGBMRLEEAFK0mrfJDZ8a4KG8PyaGqWmrqGq0HkzGmy2v2bCgiPz068mqT5Q+IyE+8DauDrZhNXUwynH2z5x9VbF1cjTERoqWfy1cBwab5fAy40ptwOkFNBXz0b/bkXgrx3k8lWGLzQBhjIkRLCUJVtTHIwkag+4xRnZAG96/nk37XdcjHFZfVAJYgjDFdX0sJolpETpqrwV3m9y6kTpDUg/rYtA75qOLDfpLioslIiu2QzzPGmLZqKUF8B3hFRL4gIiPcx+3AP9x1pyQiU0Rkm4hsF5GHgqzPFJGFIrJBRFaJyPAm66NF5D0ReTmUL9WVlZT56ZORiHgwSqwxxoRTswlCVV8BrgEmAU+5j4nA9e5McS0SkWhgNnA5MAy4WUSGNSn2MLBOVUcCt+G0bwS6H9hy6q8ROYqti6sxJkK01IspAdinqjNUtdB9zAD2uetOZRywXVV3qGotsACY2qTMMOB1AFXdCuSLSI77+Xk4jeG/D/lbdWFHryCMMaarE1UNvkJkLvBPVf1rk+WfBy5Q1btb3LHINGCKqs50398KjFfVewLK/BhIUNWvicg44G23zFoReQH4XyAV+LqqXtXM58wCZgHk5OQULliwoDXf+yQ+n4+UFG97MR1pUL70r2quHxTLZwfGdZm42sLiCo3FFRqLKzTtiWvSpElrVXVs0JWqGvQBbG5h3abm1gWUuQH4fcD7W4FfNymTBvwBWAc8DawGzsbpYvtbt8xE4OVTfZ6qUlhYqG21dOnSNm/bWtv3V2r/B1/Wv767u9XbdERcbWFxhcbiCo3FFZr2xAWs0WbOqTEtJJaWWlFbc7txEXBGwPs8oCSwgKpWALcDiNNqu9N9TAeuFpErgAQgTUT+pKq3tOJzu6zioxMFZSR1ciTGGHNqLZ3o97vVPidwl5W2Yt+rgUEiUiAicTgn/UVN9pXhrgOYCSxX1QpV/Yaq5qlqvrvdvyM9OUDgREE2UJ8xputr6Qriv4HnReQpYK27bCxOb6Ppp9qxqtaLyD3AEiAamKeqm44O36Gqc4ChwHwRaQA2A3e29YtEguIyP1ECvdMsQRhjur5mE4SqrhKR8cCXgS8ACmwCZuAkiZWn2rk63WEXN1k2J+D1CuCkm/GalF8GLDvVZ0WC4jI/vdMSiIm2MRKNMV1fS1cQqOo+4LsiMhq4GSc5XAS82AGxdTvFh62LqzEmcjSbIETkLJyqpJuBg8CfcbrFTuqg2LqdknI/Y/pldnYYxhjTKi1dQWwF/gN8VlW3gzPUd4dE1Q01NCp7y2vsCsIYEzFaqgy/HtgLLBWRJ0RkMt1pFNcOVlp5hLoGtWE2jDERo6WxmBaq6k3AEJxG4geAHBF5XEQu7aD4uo1imwfCGBNhTtmdRlWrVPUZdYa6yMO56/mkkVlNy44liExLEMaYyBBSf0tVPaSqv1PVT3sVUHdVYlONGmMijHXI7yDFh/2kJ8aSEt9iz2JjjOkyLEF0EBvm2xgTaSxBdBCbKMgYE2ksQXQQJ0HYGEzGmMhhCaIDVNTUUVlTbz2YjDERxRJEB7AeTMaYSGQJogMcnyjIEoQxJnJYgugAJXYXtTEmAlmC6ADFZTXERUeRnRLf2aEYY0yreZogRGSKiGwTke0ictLwHCKSKSILRWSDiKwSkeHu8gT3/XoR2SQi3/cyTq8Vl/nJzUggKsrGOjTGRA7PEoSIRAOzgcuBYcDNIjKsSbGHgXWqOhJnlrrH3OVHgE+r6tnAKGCKiEzwKlavlZT56ZNu1UvGmMji5RXEOGC7qu5Q1VpgATC1SZlhwOsAqroVyBeRHHX43DKx7kM9jNVTxYf91sXVGBNxRNWb866ITAOmqOpM9/2twHhVvSegzI+BBFX9moiMA952y6x1r0DWAmcCs1X1wWY+ZxYwCyAnJ6dwwYIFbYrX5/ORkpLSpm1bUt+ofPHVaq4eGMu1g+K6TFztZXGFxuIKjcUVmvbENWnSpLWqOjboSlX15AHcAPw+4P2twK+blEkD/oAzhPjTwGrg7CZlMoClwPBTfWZhYaG21dKlS9u8bUs+OVil/R98Wf+86pM2be9VXO1lcYXG4gqNxRWa9sQFrNFmzqleDi1aBJwR8D4PKAksoKoVwO0AIiLATvcRWKZMRJYBU4CNHsbriWK7Sc4YE6G8bINYDQwSkQIRiQOmA4sCC4hIhrsOYCawXFUrRKSniGS4ZRKBS3DmyI44x26SszYIY0yE8ewKQlXrReQeYAkQDcxT1U0icpe7fg4wFJgvIg3AZuBOd/Nc4I9uO0QU8LyqvuxVrF46epNcbroN1GeMiSyezl6jqouBxU2WzQl4vQIYFGS7DcBoL2PrKMVlfrJT4kmIje7sUIwxJiR2J7XHbJhvY0yksgThseIyuwfCGBOZLEF4SFXtLmpjTMSyBOGhw9V11NQ1WhdXY0xEsgThIeviaoyJZJYgPFRs80AYYyKYJQgPWYIwxkQySxAeKinzkxgbTUZSbGeHYowxIbME4aGjw3w7w0wZY0xksQThoZJyv/VgMsZELEsQHio+7Lf2B2NMxLIE4ZGaugYOVtXaMBvGmIhlCcIjx3ow2T0QxpgIZQnCI0eH+bZhNowxkcoShEfsLmpjTKSzBOGRkjI/UQI5adYGYYyJTJ4mCBGZIiLbRGS7iDwUZH2miCwUkQ0iskpEhrvLzxCRpSKyRUQ2icj9XsbpheKyGnqnJRAbbTnYGBOZPDt7udOFzgYuB4YBN4vIsCbFHgbWqepI4DbgMXd5PfBfqjoUmAB8Jci2XVpxWbXdA2GMiWhe/rwdB2xX1R2qWgssAKY2KTMMeB1AVbcC+SKSo6p7VPVdd3klsAXo62GsYVdSVmMJwhgT0bxMEH2B3QHvizj5JL8euA5ARMYB/YG8wAIiko8zP/VKrwINt8ZGZU+5zSRnjIlsoqre7FjkBuAyVZ3pvr8VGKeq9waUScOpVhoNvA8MAWaq6np3fQrwBvAjVf1rM58zC5gFkJOTU7hgwYI2xevz+UhJSWnTtk0drmnkgWV+bhsWx6f7tW+gvnDGFU4WV2gsrtBYXKFpT1yTJk1aq6pjg65UVU8ewLnAkoD33wC+0UJ5AXYBae77WGAJ8LXWfmZhYaG21dKlS9u8bVNrdh3S/g++rP/esq/d+wpnXOFkcYXG4gqNxRWa9sQFrNFmzqleVjGtBgaJSIGIxAHTgUWBBUQkw10HMBNYrqoV4gx/+iSwRVV/4WGMnjh2k5y1QRhjIliMVztW1XoRuQfnKiAamKeqm0TkLnf9HGAoMF9EGoDNwJ3u5ucDtwLvi8g6d9nDqrrYq3jDqfhYgrB7IIwxkcuzBAHgntAXN1k2J+D1CmBQkO3exKlyikglZX7SEmJITbCJgowxkcvu4vKAM1FQUmeHYYwx7WIJwgPFZX4b5tsYE/EsQXjASRDWQG2MiWyWIMKsoqaOypp668FkjIl4liDCbE9ZDWDDfBtjIp8liDArLqsG7B4IY0zkswQRZsVHryAsQRhjIpwliDArPuwnNlromRLf2aEYY0y7WIIIs5IyP7npiURFRex9fsYYA1iCCDvr4mqM6S4sQYRZSZnfGqiNMd2CJYgwqmtoZF9FjXVxNcZ0C5YgwmhveQ2Nig2zYYzpFixBhNHRYb77ZthAfcaYyGcJIoxKbB4IY0w3YgkijIoP20xyxpjuw9MEISJTRGSbiGwXkYeCrM8UkYUiskFEVonI8IB180Rkv4hs9DLGcCop95OdEkdCbHRnh2KMMe3mWYIQkWhgNnA5MAy4WUSGNSn2MLBOVUcCtwGPBax7CpjiVXxeKC6rsXsgjDHdhpdXEOOA7aq6Q1VrgQXA1CZlhgGvA6jqViBfRHLc98uBQx7GF3bFh6uteskY0214mSD6ArsD3he5ywKtB64DEJFxQH8gz8OYPKOqlNgVhDGmGxFV9WbHIjcAl6nqTPf9rcA4Vb03oEwaTrXSaOB9YAgwU1XXu+vzgZdVdTjNEJFZwCyAnJycwgULFrQpXp/PR0pKSpu2BaisVe79dzWfGxLHpfmxbd5PuOPyisUVGosrNBZXaNoT16RJk9aq6tigK1XVkwdwLrAk4P03gG+0UF6AXUBawLJ8YGNrP7OwsFDbaunSpW3eVlX1/aIy7f/gy/rK+3vatZ+m2huXVyyu0FhcobG4QtOeuIA12sw51csqptXAIBEpEJE4YDqwKLCAiGS46wBmAstVtcLDmDxT5HZxzbNhNowx3YRnCUJV64F7gCXAFuB5Vd0kIneJyF1usaHAJhHZitPb6f6j24vIc8AKYLCIFInInV7FGg7Hb5KzBGGM6R5ivNy5qi4GFjdZNifg9QpgUDPb3uxlbOFWXOYnMTaazKTwtT8YY0xnsjupw8QZ5jsBEZsoyBjTPViCCJPiMj99M22QPmNM92EJIkxKyvw2zLcxpluxBBEGNXUNHPDV2k1yxphuxRJEGFgPJmNMd2QJIgxKymoA7ArCGNOtWIIIg+KyasCuIIwx3YsliDAoLqshSqB3ujVSG2O6D0sQYVB82E9OWgKx0XY4jTHdh53RwsC5Sc6ql4wx3YsliDAoLvNbA7UxptuxBNFOjY3KnnK7gjDGdD+WINqp1HeEugalrw3zbYzpZixBtFOxe5OcDbNhjOluLEG0U/HhownCBuozxnQvliDa6fgwG3YFYYzpXixBtFNxmZ+0hBhSE2yiIGNM9+JpghCRKSKyTUS2i8hDQdZnishCEdkgIqtEZHhrt+0q7B4IY0x35VmCEJFoYDbOXNPDgJtFZFiTYg8D61R1JHAb8FgI23YJI/pmcOmwnM4Owxhjws7LOanHAdtVdQeAiCwApgKbA8oMA/4XQFW3iki+iOQAA1qxbZdw/yVBp9Q2xpiIJ6rqzY5FpgFTVHWm+/5WYLyq3hNQ5sdAgqp+TUTGAW8D44GCU20bsI9ZwCyAnJycwgULFrQpXp/PR0pKSpu29ZLFFRqLKzQWV2i6Y1yTJk1aq6pjg63z8gpCgixrmo0eAR4TkXXA+8B7QH0rt3UWqs4F5gKMHTtWJ06c2KZgly1bRlu39ZLFFRqLKzQWV2hOt7i8TBBFwBkB7/OAksACqloB3A4gIgLsdB9Jp9rWGGOMt7zsxbQaGCQiBSISB0wHFgUWEJEMdx3ATGC5mzROua0xxhhveXYFoar1InIPsASIBuap6iYRuctdPwcYCswXkQacBug7W9rWq1iNMcaczMsqJlR1MbC4ybI5Aa9XAEG7AQXb1hhjTMexO6mNMcYEZQnCGGNMUJ7dB9EZRKQU+LiNm2cDB8IYTrhYXKGxuEJjcYWmO8bVX1V7BlvRrRJEe4jImuZuFulMFldoLK7QWFyhOd3isiomY4wxQVmCMMYYE5QliOPmdnYAzbC4QmNxhcbiCs1pFZe1QRhjjAnKriCMMcYEZQnCGGNMUKdVgmjFFKgiIr9y128QkTEdFNcZIrJURLaIyCYRuT9ImYkiUi4i69zHdzootl0i8r77mWuCrO/wYyYigwOOwzoRqRCRrzYp0yHHS0Tmich+EdkYsKyHiPxLRD50nzOb2dazaXWbietnIrLV/XdaKCIZzWzb4r+5B3F9T0SKA/6trmhm244+Xn8OiGmXOy1BsG29PF5Bzw0d9jemqqfFA2fQv49wZquLA9YDw5qUuQJ4BWc+ignAyg6KLRcY475OBT4IEttE4OVOOG67gOwW1nfKMWvy77oX52afDj9ewEXAGGBjwLKfAg+5rx8CftKWv0cP4roUiHFf/yRYXK35N/cgru8BX2/Fv3OHHq8m638OfKcTjlfQc0NH/Y2dTlcQx6ZAVdVa4Og0poGmAvPV8Q6QISK5XgemqntU9V33dSWwBejr9eeGSaccswCTgY9Uta130LeLqi4HDjVZPBX4o/v6j8A1QTZtzd9jWONS1VdVtd59+w7OPCsdqpnj1RodfryOEhEBbgSeC9fntVYL54YO+Rs7nRJEX2B3wPsiTj4Jt6aMp0QkHxgNrAyy+lwRWS8ir4jIpzooJAVeFZG14kzv2lRnH7PpNP8ftzOOF0COqu4B5z840CtImc4+bnfgXPkFc6p/cy/c41Z9zWumuqQzj9eFwD5V/bCZ9R1yvJqcGzrkb+x0ShCtmca01VOdekFEUoAXga+qM3FSoHdxqlHOBn4N/K2DwjpfVccAlwNfEZGLmqzvtGMmzmRSVwN/CbK6s45Xa3XmcfsmztS+zzRT5FT/5uH2ODAQGAXswanOaaoz/2/eTMtXD54fr1OcG5rdLMiykI7Z6ZQgTjkFaivLeEJEYnH+AJ5R1b82Xa+qFarqc18vBmJFJNvruFS1xH3eDyzEuWwN1GnHDOc/5Luquq/pis46Xq59R6vZ3Of9Qcp0ynETkRnAVcDn1a2obqoV/+Zhpar7VLVBVRuBJ5r5vM46XjHAdcCfmyvj9fFq5tzQIX9jp1OCaM00pouA29yeOROA8qOXcV5y6zifBLao6i+aKdPbLYeIjMP5tzvocVzJIpJ69DVOI+fGJsU65Zi5mv1l1xnHK8AiYIb7egbwUpAyHT6trohMAR4ErlbV6mbKtObfPNxxBbZZXdvM53XWNMSXAFtVtSjYSq+PVwvnho75G/Oi5b2rPnB63HyA07L/TXfZXcBd7msBZrvr3wfGdlBcF+Bc+m0A1rmPK5rEdg+wCacnwjvAeR0Q1wD389a7n92VjlkSzgk/PWBZhx8vnAS1B6jD+cV2J5AFvA586D73cMv2ARa39PfocVzbceqkj/6NzWkaV3P/5h7H9bT7t7MB5wSW2xWOl7v8qaN/UwFlO/J4NXdu6JC/MRtqwxhjTFCnUxWTMcaYEFiCMMYYE5QlCGOMMUFZgjDGGBOUJQhjjDFBWYIwJgQi0iAnjiQbtlFFRSQ/cDRRYzpbTGcHYEyE8avqqM4OwpiOYFcQxoSBOyfAT0Rklfs4013eX0Redweie11E+rnLc8SZk2G9+zjP3VW0iDzhjv3/qogkdtqXMqc9SxDGhCaxSRXTTQHrKlR1HPAb4Jfust/gDIc+EmdwvF+5y38FvKHOYIJjcO7CBRgEzFbVTwFlwPWefhtjWmB3UhsTAhHxqWpKkOW7gE+r6g53cLW9qpolIgdwho6oc5fvUdVsESkF8lT1SMA+8oF/qeog9/2DQKyq/rADvpoxJ7ErCGPCR5t53VyZYI4EvG7A2glNJ7IEYUz43BTwvMJ9/TbOKJoAnwfedF+/DtwNICLRIpLWUUEa01r268SY0CTKiZPX/1NVj3Z1jReRlTg/vG52l90HzBOR/wZKgdvd5fcDc0XkTpwrhbtxRhM1psuwNghjwsBtgxirqgc6OxZjwsWqmIwxxgRlVxDGGGOCsisIY4wxQVmCMMYYE5QlCGOMMUFZgjDGGBOUJQhjjDFB/X/7KU2Q+UE4RgAAAABJRU5ErkJggg==\n",
|
|
"text/plain": [
|
|
"<Figure size 432x288 with 1 Axes>"
|
|
]
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"\n",
|
|
"metrics = pd.read_csv(f\"{trainer.logger.log_dir}/metrics.csv\")\n",
|
|
"\n",
|
|
"aggreg_metrics = []\n",
|
|
"agg_col = \"epoch\"\n",
|
|
"for i, dfg in metrics.groupby(agg_col):\n",
|
|
" agg = dict(dfg.mean())\n",
|
|
" agg[agg_col] = i\n",
|
|
" aggreg_metrics.append(agg)\n",
|
|
"\n",
|
|
"df_metrics = pd.DataFrame(aggreg_metrics)\n",
|
|
"df_metrics[[\"train_loss\", \"valid_loss\"]].plot(\n",
|
|
" grid=True, legend=True, xlabel='Epoch', ylabel='Loss')\n",
|
|
"df_metrics[[\"train_acc\", \"valid_acc\"]].plot(\n",
|
|
" grid=True, legend=True, xlabel='Epoch', ylabel='ACC')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "6aa6ff75",
|
|
"metadata": {},
|
|
"source": [
|
|
"- The `trainer` automatically saves the model with the best validation accuracy automatically for us, we which we can load from the checkpoint via the `ckpt_path='best'` argument; below we use the `trainer` instance to evaluate the best model on the test set:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 14,
|
|
"id": "51c48cb0",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Restoring states from the checkpoint path at logs/my-mlp/version_17/checkpoints/epoch=10-step=2353.ckpt\n",
|
|
"LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
|
|
"Loaded model weights from checkpoint at logs/my-mlp/version_17/checkpoints/epoch=10-step=2353.ckpt\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/vnd.jupyter.widget-view+json": {
|
|
"model_id": "99b4b63be6cb49d99728a56d629db7b4",
|
|
"version_major": 2,
|
|
"version_minor": 0
|
|
},
|
|
"text/plain": [
|
|
"Testing: 0it [00:00, ?it/s]"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
|
|
"┃<span style=\"font-weight: bold\"> Test metric </span>┃<span style=\"font-weight: bold\"> DataLoader 0 </span>┃\n",
|
|
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
|
|
"│<span style=\"color: #008080; text-decoration-color: #008080\"> test_acc </span>│<span style=\"color: #800080; text-decoration-color: #800080\"> 0.9706000089645386 </span>│\n",
|
|
"└───────────────────────────┴───────────────────────────┘\n",
|
|
"</pre>\n"
|
|
],
|
|
"text/plain": [
|
|
"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
|
|
"┃\u001b[1m \u001b[0m\u001b[1m Test metric \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m DataLoader 0 \u001b[0m\u001b[1m \u001b[0m┃\n",
|
|
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
|
|
"│\u001b[36m \u001b[0m\u001b[36m test_acc \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.9706000089645386 \u001b[0m\u001b[35m \u001b[0m│\n",
|
|
"└───────────────────────────┴───────────────────────────┘\n"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"[{'test_acc': 0.9706000089645386}]"
|
|
]
|
|
},
|
|
"execution_count": 14,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"trainer.test(model=lightning_model, datamodule=data_module, ckpt_path='best')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "0cbde19d",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Predicting labels of new data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "e8552150",
|
|
"metadata": {},
|
|
"source": [
|
|
"- You can use the `trainer.predict` method on a new `DataLoader` or `DataModule` to apply the model to new data.\n",
|
|
"- Alternatively, you can also manually load the best model from a checkpoint as shown below:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 15,
|
|
"id": "05e58080",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"logs/my-mlp/version_17/checkpoints/epoch=10-step=2353.ckpt\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"path = trainer.checkpoint_callback.best_model_path\n",
|
|
"print(path)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 16,
|
|
"id": "fcd730e7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"lightning_model = LightningMLP.load_from_checkpoint(path, model=pytorch_model)\n",
|
|
"lightning_model.eval();"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "67c85b70",
|
|
"metadata": {},
|
|
"source": [
|
|
"- Note that our PyTorch model, which is passed to the Lightning model requires input arguments. However, this is automatically being taken care of since we used `self.save_hyperparameters()` in our PyTorch model's `__init__` method.\n",
|
|
"- Now, below is an example applying the model manually. Here, pretend that the `test_dataloader` is a new data loader."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 17,
|
|
"id": "119b7461",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"tensor([7, 2, 1, 0, 4])"
|
|
]
|
|
},
|
|
"execution_count": 17,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"test_dataloader = data_module.test_dataloader()\n",
|
|
"\n",
|
|
"all_true_labels = []\n",
|
|
"all_predicted_labels = []\n",
|
|
"for batch in test_dataloader:\n",
|
|
" features, labels = batch\n",
|
|
" with torch.no_grad(): # since we don't need to backprop\n",
|
|
" logits = lightning_model(features)\n",
|
|
" predicted_labels = torch.argmax(logits, dim=1)\n",
|
|
" all_predicted_labels.append(predicted_labels)\n",
|
|
" all_true_labels.append(labels)\n",
|
|
" \n",
|
|
"all_predicted_labels = torch.cat(all_predicted_labels)\n",
|
|
"all_true_labels = torch.cat(all_true_labels)\n",
|
|
"all_predicted_labels[:5]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "af07da91",
|
|
"metadata": {},
|
|
"source": [
|
|
"Just as an internal check, if the model was loaded correctly, the test accuracy below should be identical to the test accuracy we saw earlier in the previous section."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 18,
|
|
"id": "d24d1c85",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Test accuracy: 0.9706 (97.06%)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"test_acc = torch.mean((all_predicted_labels == all_true_labels).float())\n",
|
|
"print(f'Test accuracy: {test_acc:.4f} ({test_acc*100:.2f}%)')"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|