transformer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\ProgramData\\Anaconda3\\envs\\py38\\lib\\site-packages\\scipy\\__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.3\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from os.path import exists\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn.functional import log_softmax, pad\n",
    "import math\n",
    "import copy\n",
    "import time\n",
    "from torch.optim.lr_scheduler import LambdaLR\n",
    "import pandas as pd\n",
    "from torchtext.data.functional import to_map_style_dataset\n",
    "from torch.utils.data import DataLoader\n",
    "from torchtext.vocab import build_vocab_from_iterator\n",
    "import torchtext.datasets as datasets\n",
    "import spacy\n",
    "import warnings\n",
    "from torch.utils.data.distributed import DistributedSampler\n",
    "import torch.distributed as dist\n",
    "import torch.multiprocessing as mp\n",
    "from torch.nn.parallel import DistributedDataParallel as DDP\n",
    "\n",
    "\n",
    "# Set to False to skip notebook execution (e.g. for debugging)\n",
    "os.environ[\"KMP_DUPLICATE_LIB_OK\"]=\"TRUE\"\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "绝大多数序列处理模型都是 **encoder-decoder** 的结构。首先,encoder部分将输入序列$(x_1,...,x_n)$映射到一系列对输入的表征序列 $z = (z_1,...,z_n)$中。Decoder部分是使用输入的表征序列$z$，来生成输出序列$(y_1,...,y_m)$.在每一步，该模型都是自回归模型，在生成下一个符号时，将之前生成的符号作为额外的输入。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EncoderDecoder(nn.Module):\n",
    "    \"\"\"\n",
    "    标准的encoder-decoder结构\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, encoder, decoder, src_embed, tgt_embed, generator):\n",
    "        super(EncoderDecoder, self).__init__()\n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.src_embed = src_embed\n",
    "        self.tgt_embed = tgt_embed\n",
    "        self.generator = generator\n",
    "\n",
    "    def forward(self, src, tgt, src_mask, tgt_mask):\n",
    "        \"Take in and process masked src and target sequences.\"\n",
    "        return self.decode(self.encode(src, src_mask), src_mask, tgt, tgt_mask)\n",
    "\n",
    "    def encode(self, src, src_mask):\n",
    "        return self.encoder(self.src_embed(src), src_mask)\n",
    "\n",
    "    def decode(self, memory, src_mask, tgt, tgt_mask):\n",
    "        return self.decoder(self.tgt_embed(tgt), memory, src_mask, tgt_mask)\n",
    "\n",
    "class Generator(nn.Module):\n",
    "    \"Decoder后的生成输出序列的模块\"\n",
    "\n",
    "    def __init__(self, d_model, vocab):\n",
    "        super(Generator, self).__init__()\n",
    "        self.proj = nn.Linear(d_model, vocab)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return log_softmax(self.proj(x), dim=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![def](./transformer.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transformer中encoder, decoder都是由相同结构的layer 组成的，所以这个函数用来重复相同的layer\n",
    "def clones(module, N):\n",
    "    \"Produce N identical layers.\"\n",
    "    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attention "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "注意力机制可以被描述为将一个query和一组key-value对映射到一个输出，其中query、key、value和输出都是向量。输出是数值的加权和，其中分配给每个数值的权重是由query与相应的key的相关性函数计算的。\n",
    "$$\n",
    "Attention(Q,K,V) = softmax(\\frac{QK^T}{\\sqrt{d_k}})V\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def attention(query, key, value, mask = None, dropout = None):\n",
    "    \"Compute Scalued Dot Product Attention\"\n",
    "    d_k = query.size(-1)  # 64\n",
    "    scores = torch.matmul(query, key.transpose(-2,-1)) / math.sqrt(d_k)\n",
    "    # print(scores.shape)\n",
    "    if mask is not None:\n",
    "        # 对于encoder的self attention以及decoder的cross attention只考虑sequence有内容的，不考虑是padding\n",
    "        # 对于decoder的self attention 还需要mask掉当前step之后的词\n",
    "        scores = scores.masked_fill(mask == 0, -1e-9)  \n",
    "    \n",
    "    p_attn = scores.softmax(dim = -1)\n",
    "    # print(p_attn.shape)\n",
    "    if dropout is not None:\n",
    "        p_attn = dropout(p_attn)\n",
    "    return torch.matmul(p_attn, value), p_attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 127, 512])\n"
     ]
    }
   ],
   "source": [
    "# 例子\n",
    "q = torch.randn(2,127,512)\n",
    "k = torch.randn(2,127,512)\n",
    "v = torch.randn(2,127,512)\n",
    "output, _ = attention(q,k,v)\n",
    "# 其中attention 中的scores 维度为[2, 127, 127], 表示的是127个词互相之间的attenton scores\n",
    "print(output.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MultiHead Attention\n",
    "$$\n",
    "\\begin{array}{c}\n",
    "\\text { MultiHead }(Q, K, V)=\\operatorname{Concat}\\left(\\operatorname{head}_{1}, \\ldots, \\operatorname{head}_{\\mathrm{h}}\\right) W^{O} \\\\\n",
    "\\text { where head }{ }_{\\mathrm{i}}=\\operatorname{Attention}\\left(Q W_{i}^{Q}, K W_{i}^{K}, V W_{i}^{V}\\right)\n",
    "\\end{array}\n",
    "$$\n",
    "其中多头的个数必须要能够整除模型的hidden states size。比如，使用$h=8$，每个头的$d_k=d_v=d_{model}/h = 64$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadedAttention(nn.Module):\n",
    "    def __init__(self, h, d_model, dropout = 0.1):\n",
    "        super().__init__()\n",
    "        assert d_model % h == 0\n",
    "        self.d_k = d_model // h\n",
    "        self.h = h\n",
    "        self.linears = clones(nn.Linear(d_model, d_model), 4)\n",
    "        self.attn = None\n",
    "        self.dropout = nn.Dropout(p = dropout)\n",
    "    \n",
    "    def forward(self, query, key, value, mask = None):\n",
    "        if mask is not None:\n",
    "            mask = mask.unsqueeze(1)  # 这里是为了多头注意力机制，新增一维度 （bs, num_head, ...）\n",
    "        # print(mask.shape) # 一般的src mask: torch.Size([bs, 1, 1, squence_length])  tgt mask: torch.Size([bs, 1, squence_length, squence_length])\n",
    "\n",
    "        nbatches = query.size(0)\n",
    "\n",
    "        # 1) 首先对Q,K,V进行一次线性变换,然后将d_model --> h x d_k\n",
    "        query = self.linears[0](query).view(nbatches, -1, self.h, self.d_k).transpose(1,2)  # torch.Size([bs, 1, squence_length, d_model // h])  d_model // h 一般为 512 // 8 = 64\n",
    "        key = self.linears[1](key).view(nbatches, -1, self.h, self.d_k).transpose(1,2)  \n",
    "        value = self.linears[2](value).view(nbatches, -1, self.h, self.d_k).transpose(1,2)  \n",
    "\n",
    "        # 2) apply attention\n",
    "        x, self.attn = attention(\n",
    "            query, key, value, mask=mask, dropout=self.dropout\n",
    "        )\n",
    "\n",
    "        # 3) h x d_k --> d_model\n",
    "        x = x.transpose(1,2).contiguous().view(nbatches, -1, self.h * self.d_k)\n",
    "        del query\n",
    "        del key\n",
    "        del value\n",
    "        return self.linears[-1](x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 127, 512])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test example\n",
    "MHattention = MultiHeadedAttention(8, 512)\n",
    "q = torch.randn(2,127,512)\n",
    "k = torch.randn(2,127,512)\n",
    "v = torch.randn(2,127,512)\n",
    "MHattention(q,k,v).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Multi-Head Attention在Transformer中有三处使用了 \n",
    "1) \"encoder-decoder attention\"，queries来自前一个decoder layer，而keys和values来自encoder的输出。这使得decoder中的每个输出都能关注到输入序列中的所有位置。 \n",
    "\n",
    "2) encoder包含self-attention layers。在self-attention layers，所有的queries, keys, values都来自encoder中前一层的输出。encoder中的每个位置都可以关注到encoder前一层的所有位置。 \n",
    "\n",
    "3) decoder中的self-attention layers允许decoder关注直到并包括该位置的所有信息。我们需要防止decoder中的信息向左流动，以保持自动回归的特性。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Position-wise Feed-Forward Networks\n",
    "在encoder和decoder中，每个attention based 层后都接有一个fully connected feed-forward 网络。它由两层线性层和ReLU激活函数组成。\n",
    "$$\n",
    "FFN(x) = ReLU(xW_1+b_1)W_2+b_2\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PositionwiseFeedForward(nn.Module):\n",
    "    \"Implements FFN equation.\"\n",
    "\n",
    "    def __init__(self, d_model, d_ff, dropout=0.1):\n",
    "        super(PositionwiseFeedForward, self).__init__()\n",
    "        self.w_1 = nn.Linear(d_model, d_ff)\n",
    "        self.w_2 = nn.Linear(d_ff, d_model)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.relu = nn.ReLU()\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.w_2(self.relu(self.dropout(self.w_1(x))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Residue Connection\n",
    "每一层的输出都采用了residule connection的形式，同时也使用了Layer Normalization。\n",
    "$$\n",
    "Opt = LayerNorm(x+Sublayer(x))\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LayerNorm(nn.Module):\n",
    "    \"Construct a layernorm module (See citation for details). Now, we can use nn.LayerNorm to do this job\"\n",
    "\n",
    "    def __init__(self, features, eps=1e-6):\n",
    "        super().__init__()\n",
    "        self.a_2 = nn.Parameter(torch.ones(features))\n",
    "        self.b_2 = nn.Parameter(torch.zeros(features))\n",
    "        self.eps = eps\n",
    "    \n",
    "    def forward(self, x):\n",
    "        mean = x.mean(-1, keepdim = True)\n",
    "        std = x.std(-1, keepdim = True)\n",
    "        return self.a_2 * (x - mean) / (std + self.eps) + self.b_2 \n",
    "\n",
    "class SublayerConnection(nn.Module):\n",
    "    \"\"\"\n",
    "    A residual connection followed by a layer norm.\n",
    "    \"\"\"\n",
    "    def __init__(self, size, dropout):\n",
    "        super().__init__()\n",
    "        self.norm = nn.LayerNorm(size)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "    \n",
    "    def forward(self, x, sublayer):\n",
    "        return self.norm(x + self.dropout(sublayer(x)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoder Layer\n",
    "每一个EncoderLayer都包含两层layer，第一个是multi-head self-attention mechanism,第二个是positional-wise fully connected feed-forward network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EncoderLayer(nn.Module):\n",
    "\n",
    "    def __init__(self, size, self_attn, feed_forward, dropout):\n",
    "        super().__init__()\n",
    "        self.self_attn = self_attn\n",
    "        self.feed_forward = feed_forward\n",
    "        self.sublayer = clones(SublayerConnection(size, dropout), 2)\n",
    "        self.size = size\n",
    "    \n",
    "    def forward(self, x, mask):\n",
    "\n",
    "        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, mask))\n",
    "        return self.sublayer[1](x, self.feed_forward)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Encoder\n",
    "Encoder是由几个encoder layer构成的。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "\n",
    "    def __init__(self, layer, N):\n",
    "        super().__init__()\n",
    "        self.layers = clones(layer, N)\n",
    "        self.norm = nn.LayerNorm(layer.size)\n",
    "    \n",
    "    def forward(self, x, mask):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x, mask)\n",
    "        \n",
    "        return self.norm(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decoder Layer\n",
    "Decoder layer 比 encoder layer 多一层与encoder layer交互的cross attention。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DecoderLayer(nn.Module):\n",
    "\n",
    "    def __init__(self, size, self_attn, src_attn, feed_forward, dropout):\n",
    "        super().__init__()\n",
    "        self.size = size\n",
    "        self.self_attn = self_attn\n",
    "        self.src_attn = src_attn\n",
    "        self.feed_forward = feed_forward\n",
    "        self.sublayer = clones(SublayerConnection(size, dropout), 3)\n",
    "\n",
    "    def forward(self, x, memory, src_mask, tgt_mask):\n",
    "        # print(f'x: {x.shape}')  # [batch, seq_len-1, 512] \n",
    "        # print(f'memory: {memory.shape}') # [batch, seq_len, 512]\n",
    "        # print(f'src_mask: {src_mask.shape}')  # [batch, 1, seq_len]\n",
    "        # print(f'tgt_mask: {tgt_mask.shape}')  # [batch, seq_len-1, seq_len]\n",
    "        m = memory # 这是从encoder来的key, value\n",
    "        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, tgt_mask))\n",
    "        x = self.sublayer[1](x, lambda x: self.src_attn(x, m, m, src_mask))\n",
    "        return self.sublayer[2](x, self.feed_forward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "\n",
    "    def __init__(self, layer, N):\n",
    "        super().__init__()\n",
    "        self.layers = clones(layer, N)\n",
    "        self.norm = nn.LayerNorm(layer.size)\n",
    "\n",
    "    def forward(self, x, memory, src_mask, tgt_mask):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x, memory, src_mask, tgt_mask)\n",
    "        return self.norm(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tgt mask\n",
    "我们还修改了decoder中的self attention，在其中加入了mask，为了防止当前模型能够获得当前step之后的序列信息。这种屏蔽，确保对位置i的预测只取决于小于i的位置的已知输出。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def subsequent_mask(size):\n",
    "    \"Mask out subsequent positions.\"\n",
    "    attn_shape = (1, size, size)\n",
    "    subsequent_mask = torch.triu(torch.ones(attn_shape), diagonal=1).type(\n",
    "        torch.uint8\n",
    "    )\n",
    "    return subsequent_mask == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ True, False, False, False, False, False, False, False, False, False],\n",
       "         [ True,  True, False, False, False, False, False, False, False, False],\n",
       "         [ True,  True,  True, False, False, False, False, False, False, False],\n",
       "         [ True,  True,  True,  True, False, False, False, False, False, False],\n",
       "         [ True,  True,  True,  True,  True, False, False, False, False, False],\n",
       "         [ True,  True,  True,  True,  True,  True, False, False, False, False],\n",
       "         [ True,  True,  True,  True,  True,  True,  True, False, False, False],\n",
       "         [ True,  True,  True,  True,  True,  True,  True,  True, False, False],\n",
       "         [ True,  True,  True,  True,  True,  True,  True,  True,  True, False],\n",
       "         [ True,  True,  True,  True,  True,  True,  True,  True,  True,  True]]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subsequent_mask(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mask in Transformer\n",
    "这里着重讲解一下transformer中的几个mask。\n",
    "\n",
    "1. src_mask \\\n",
    "    src_mask是用来标记input sequence中是padding的部分。举个例子，input sequence为[1,2,3,4], 但是在组成batch时, 它被padding成了[1,2,3,4,0,0,0], 因此此时的src_mask就位[1,1,1,1,0,0,0]. 它的作用就是标记当前有效的输入序列长度。\n",
    "2. tgt_mask \\\n",
    "    tgt_mask同样也会用来标记target sequence中是padding的部分, 同时它也需要mask掉当前step之后的序列信息. 如同上面例子所示."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Word embedding\n",
    "word embedding 使用pytorch自带`nn.Embedding`但是要乘$\\sqrt{d_k}$，第一个原因是防止被positional encoding淹没。[原因链接](https://zhuanlan.zhihu.com/p/442509602)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Embeddings(nn.Module):\n",
    "    def __init__(self, d_model, vocab):\n",
    "        super(Embeddings, self).__init__()\n",
    "        self.lut = nn.Embedding(vocab, d_model)\n",
    "        self.d_model = d_model\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.lut(x) * math.sqrt(self.d_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Positional Encoding\n",
    "\n",
    "为了使模型能够利用序列的顺序，我们必须注入一些关于序列中标记的相对或绝对位置的信息。为此，我们在encoder和decoder的底部为输入添加 \"位置编码\"。位置编码的公式如下:\n",
    "$$\n",
    "\\begin{array}{c}\n",
    "P E_{(p o s, 2 i)}=\\sin \\left(p o s / 10000^{2 i / d_{\\text {model }}}\\right) \\\\\n",
    "P E_{(p o s, 2 i+1)}=\\cos \\left(p o s / 10000^{2 i / d_{\\mathrm{model}}}\\right)\n",
    "\\end{array}\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PositionalEncoding(nn.Module):\n",
    "    \"Implement the PE function.\"\n",
    "\n",
    "    def __init__(self, d_model, dropout, max_len=5000):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "\n",
    "        # Compute the positional encodings once in log space.\n",
    "        pe = torch.zeros(max_len, d_model)\n",
    "        position = torch.arange(0, max_len).unsqueeze(1)\n",
    "        div_term = torch.exp(\n",
    "            torch.arange(0, d_model, 2) * -(math.log(10000.0) / d_model)\n",
    "        )\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        pe = pe.unsqueeze(0)\n",
    "        self.register_buffer(\"pe\", pe) # 注册成不更新的参数\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x + self.pe[:, : x.size(1)].requires_grad_(False)\n",
    "        return self.dropout(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqoAAAHkCAYAAAD2Pw3rAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABHDElEQVR4nO3de5zcZX33/9dnN9kcgJADEJDzGa0IIqKVaq2gorWCSgVrFa2W23rWesDb/tTSehetrS2tt5IqFQ83arlr5fZEEaFaFRA0gJwk4gEo4ZBwTrJJdj+/P+YbmYbdZEN2P/udyeuZxzx2Dt/Z9zWT2ZnPXN/rur6RmUiSJEltMzDdDZAkSZLGYqEqSZKkVrJQlSRJUitZqEqSJKmVLFQlSZLUShaqkiRJaqUpK1Qj4uyIuDMiftJ13cKIuDAibmp+Lmiuj4g4MyKWRcTVEXHEVLVLkiRJYxurftvo9nFrtog4panxboqIUyajPVPZo/pp4LiNrjsNuCgzDwQuai4DPA84sDmdCnx8CtslSZKksX2aR9Zv3cas2SJiIfB+4CnAUcD7N3RIbo0pK1Qz8zvAyo2uPh44pzl/DnBC1/WfyY5LgfkRsdtUtU2SJEmPNE791m28mu25wIWZuTIz7wEuZNMF74RUj1FdnJm3N+eXA4ub87sDt3Rtd2tznSRJktpjvJptSmq5GVv7Cx6tzMyI2OLjt0bEqXS6mhlk8ElzmTfpbRvL8F7bleQAPH7+XSU51961c0kOQBa+0mbd8lBZ1vAeda+LQxfWvC7+a/2ckhyAh342qyzrMQfdU5a1Yv32JTmrR2aW5ADsN3tTHSyT64YHdinLevwOd5dl/eTemvfcQ4s+QwCuuafuc+TQBXWP68qrh+/OzLoHN4bn/s52uWLlyKT/3iuvHr4WWNN11ZLMXDLpQZOkulC9IyJ2y8zbm27iO5vrbwP27Npuj+a6R2iezCUA82JhPiWOmcr2/tpN735KSQ7AZS/+REnO4z/xhpIcgOFFo2VZB7zl0rKsm9/6m2VZl7+85nXx/rt+oyQH4IqXHFSWdfpXv1iW9Zm7jy7J+cm9u5bkAJx78LllWb95cd1702XHfqos64B/e11JzuUn1NUc+5/3P8qyLj/xrLKswd1u+mVZ2DhWrBzh8gv2mvTfO7jbTWsy88it+BXj1Wy3Ac/c6PpLtiIHqN/1fz6wYRbYKcBXuq5/ZTOT7KnAfV1DBCRJkrYpCYxOwb9JMF7NdgHwnIhY0Eyiek5z3VaZsh7ViDiXTmW9U0TcSmcm2BnAlyLiNcAvgZc2m38deD6wDFgFvHqq2iVJktR+yUjW7Y3cYJz6bSZAZn6CcWq2zFwZEX8B/LD5Vadn5laPGZqyQjUzXzbOTY/YV5+ZCWzxvp6cN5d1v7k1vdcT99iPLC/JARh+0bqSnDjs/pIcgFhdN5aOqNtRMLgmyrKqzB6oef0BMFj3fzWadf9XA1Hz4ZKFj6lSvz4uqRdson7bcPu4NVtmng2cPZntmbbJVJIkSRpbZ9f/Fs857zseQlWSJEmtZI+qJElSC03S5KeeZqEqSZLUMkkyku767+1Cddf18M6aBYDzxXWLXX9/zdySnBMPWFqSA3DBbY8tyxqYM7ssa3C4LKrsm/XMmPwFpseTA3Wjj0aonEzlh4skTYbeLlQlSZL6lJOpnEwlSZKklrJHVZIkqWUSGLFHtbcL1YNm38MFj/vXkqwnvvFNJTkAn7mzpqP7PY/5RkkOwE8fXFyWdf/225VlDa4pi2Jd1owdnVW64H/duNGRLDwQRNEY1dHCcbeSNB16ulCVJEnqV45RtVCVJElqnQSXp8LJVJIkSWqpnu5RvXtkiE/dv3dJ1p+9+gslOQD/8z9fXJJzzt7fKckBOGj7O8qyrph3UFlW5TqqVWNUK9dRpXAd1UoDHk1G0iTwncQeVUmSJLVUT/eoSpIk9aMkXZ4KC1VJkqT2SRixTnXXvyRJktqpp3tU77x7Ph9f8sKSrMvfcWZJDsCHlg6V5Kx+bt1MoINm316Wddn8J5ZllS74XzSsfnbULfifM+q+K48Wfi8fqFrwP+sW/B+MwoML2IskkTiZCuxRlSRJUkv1dI+qJElSfwpGPEyyhaokSVLbJDDqMJjeLlSH7lrFYz7x45Ks3zr2D0pyAHZaurok5z/W7FCSA7D/0J1lWWsXzS7LmjFc9y4ynDWjlWbG+pIcgCxc8L+yZ2LQkWXaVlhIaYr1dKEqSZLUr9z172QqSZIktZQ9qpIkSS2T2KMKvV6oDg0xsNfuJVHzP7RdSQ7A0A23luR89o6nleQA/NUe/68sa/VOdS/rwcIxqg8VRQ3FSE0QwGDdm/BI9t86qlm4jqqkepVrJbeVu/4lSZLUSr3doypJktSH3PXfYY+qJEmSWskeVUmSpJZJghH7E3u7UF2zeJDr3rmgJOug1/6wJAegarn171/75KIkWLjXzLKsNQvrdpXM+1Xdwu6rRgdLckoX/B/szzfhwXDBf0maDD1dqEqSJPUrZ/1bqEqSJLWOk6k6+nO/myRJknpeT/eoHjTvDr7ynI+WZL385HeU5ADM+/LSkpz5V9WNG537u0NlWWsWlUWx4Kd1YxEfypr/r5mFC/5n4YL/o4U9EwPULPjvbkGpn0XpgUraymdAkiRJrdTTPaqSJEn9KIFR+xMtVCVJktrIyVQ9XqiOEjyUNWtLHvK2n5TkACxfumdJzs5LV5XkAAwUfiscXlQ3bnRwTd14zgdGZ5fkDJWuo1r3Jly5cHY/rqM6UPqB6YezpI6eLlQlSZL6UaaTqcDJVJIkSWope1QlSZJaqHJZvbayUJUkSWqZzpGp3PHd04XqsnsX83vnv7Uk6+aXnFWSA3D0Yf+jJGf+t5eV5ACsy7oJOoOLhsuyBobrJs3cPzqnJGfR4IMlOVA8mapwrNdA0WSqdMF/SX2upwtVSZKk/uRkKnAylSRJklrKHlVJkqSWmc4jU0XEccDfA4PAJzPzjI1u/yjwO83FucAumTm/uW0EuKa57VeZ+cKtaUtPF6qz71jLYz9yW0nWnx59REkOwN2H1bwwt//i3SU5AA/m2rKsnRc8UJY1MDxUlnXfyNySnF0H7yvJARgtHKNa+YY/SJZlSdJkiohB4GPAs4FbgR9GxPmZed2GbTLzbV3bvwl4YtevWJ2Zh09We3q6UJUkSepXI9MzYfIoYFlm3gwQEV8AjgeuG2f7lwHvn6rGWKhKkiS1TBJTtTzVThFxRdflJZm5pOvy7sAtXZdvBZ4y1i+KiL2BfYFvd109u/n964EzMvPftqaxFqqSJEnbjrsz88hJ+l0nA+dl5kjXdXtn5m0RsR/w7Yi4JjN/9mgDertQXT9Crry3JOo/zzyqJAdg3u/XjB0dmFOzLifAHSN1Y/b2n1839nbF2l3Ksu4tGqM6FCOb32iS5EB/rgNatY7qqOuo9haHLmsLjU7P8lS3AXt2Xd6juW4sJwNv6L4iM29rft4cEZfQGb/6qAtVl6eSJEnSBj8EDoyIfSNiiE4xev7GG0XEIcAC4Add1y2IiFnN+Z2Aoxl/bOuE9HaPqiRJUh+arkOoZub6iHgjcAGd5anOzsxrI+J04IrM3FC0ngx8ITO79xU8FjgrIkbpdIae0b1awKNhoSpJktQySUzXrH8y8+vA1ze67n0bXf7AGPf7PnDoZLbFXf+SJElqpZ7uUV27yxx+9ZpJLdzHtftfXVqSA/D8d95TknPBfk8uyQG4Zf2OZVkHb3dHWdYPhheWZd23vmYy1czKyVQz6noLKnsmXPBf0mSYriNTtYnPgCRJklqpp3tUJUmS+lEmjEzP8lStYqEqSZLUOsEorpXc04Xqbjut5M9e9YWSrE9/63dLcgBetMPHS3K++ITnluQA3DS8a1nWAbMLx6iu3b8s6951NWNUK8dX5mDhGNXCkU5VC/73LYf4Smr0dKEqSZLUjxJ3/YOTqSRJktRS9qhKkiS10HQcmaptfAYkSZLUSj3dozp/YD3Hb7e8JOv0d5XEALDXjB1Kcu4+rO57yk2rF5dl/cHCuoMzsG59WdT962eX5MwsnAg0OlgWxWjhWK9Bap7DLJx0NOjsY6lUEoxO0yFU26SnC1VJkqR+5a5/d/1LkiSppexRlSRJapmkdshSW/V0oXrjqoU8Y+kflmRd+tR/KskBuGe0ZuDZwifcVZID8NMHdinLeswua8qyWLuuLOretXNKcmZG4YL/A/264L8r1kvSZOjpQlWSJKk/BSNOYrRQlSRJaht3/Xf4DEiSJKmVerpHdXD5APM/tF1J1vf/eceSHIAHRmrGIr5yn7r1Rj9+4zPKshYODJVl5frCdVTX1rzWBykco1r4DjRSuB5h1Tqqo+4WlPqau/7tUZUkSVJL9XSPqiRJUj/KDMeoYqEqSZLUSiMWqu76lyRJUjtNS49qRLwNeC2d1ReuAV4N7AZ8AVgEXAm8IjPXbvIXPbiage/8eGob23jb515TkgOw32//oiRnyf5fKskB+MjK48qy5sSssqxcV7fg/0Nrax5X5bfX0cG6iQKjhY9sMGomU0nqX4kTJmEaelQjYnfgzcCRmfl4YBA4GfgQ8NHMPAC4B6irDCVJktQ607XrfwYwJyJmAHOB24FnAec1t58DnDA9TZMkSZpuwUgOTPqp15S3ODNvAz4C/IpOgXofnV3992bmhkUpbwV2r26bJEmS2qN8jGpELACOB/YF7gX+BZjwAMaIOBU4FWBo7gIeOOGpU9DKR9rvzBtLcgCuX3hgSc7uB+1QkgMwuKJPF5goXPD/oeGaAxkMRd2YqBwsiyrtSRgoOmhCFh7EoFTdMSek1uocQrVP/8a3wHRUD8cCP8/MuwAi4l+Bo4H5ETGj6VXdA7htrDtn5hJgCcD2i/b07UySJPWlERdnmpZn4FfAUyNibkQEcAxwHXAxcGKzzSnAV6ahbZIkSWqJ8h7VzLwsIs4DfgSsB35Mp4f0a8AXIuIvm+s+Vd02SZKkNkjCXf9M0zqqmfl+4P0bXX0zcNSW/J7Zu6zmsW/9yaS1a1OW/0fdeM5FV9V0dK940UMlOQCzV9T9sa3LunGjOTJSljU8XPPnWruOamGYNIYBd61KrdanM1wkSZJ6W+WBStrKQlWSJKllMmHEXf+W6pIkSWone1QlSZJayMlUPV6o7jP0IJ/a83slWQe+/XUlOQD7f+GBkpxvPrRnSQ7A7BVlUdyfw2VZOVq3lO/6NTV/roOlC/7XZY1SlzUzaibZVX6IDRQ+f5K0QU8XqpIkSf2oszyVIzQtVCVJklpoxD0ZTqaSJElSO/V0j+ot6+byp8uPKMn67Ev+sSQH4PS/eFZJzmdu+82SHIA5K+oWxr+rLgpytC5ruGZ1/MqxiFm44P9I4S60WbGuLEtSf0qcTAX2qEqSJKmlerpHVZIkqT85mQrsUZUkSVJL9XSP6oN3bsd/nnlUSdY7/+I7JTkAI/fdV5Kz7JrHluQA7Leibm3TX6xfUJZVKdbUfK8cLByjOlo5RrXwe/lg1Kyvm3XL+EqaBpXrP7eVPaqSJEktkwkjGZN+moiIOC4iboyIZRFx2hi3vyoi7oqIpc3ptV23nRIRNzWnU7b2eejpHlVJkiRNnogYBD4GPBu4FfhhRJyfmddttOkXM/ONG913IfB+4Eg6Cxdc2dz3nkfbHntUJUmSWmg0Byb9NAFHAcsy8+bMXAt8ATh+gk1+LnBhZq5sitMLgeMe1YNvWKhKkiRpg92BW7ou39pct7GXRMTVEXFeROy5hfedsJ7e9T+44iHmn3NZSdbRx7ypJAfg4B1uKslZtLRukPbMlavKsm4a3rUsq9LAcM3/V78u+F+5cPYghQeC6Ef9usi5k9+0BZKYqvetnSLiiq7LSzJzyRb+jv8HnJuZwxHxP4BzgCk5WlFPF6qSJEn9aopm/d+dmUdu4vbbgD27Lu/RXPdrmbmi6+IngQ933feZG933kkfbUHDXvyRJkh72Q+DAiNg3IoaAk4HzuzeIiN26Lr4QuL45fwHwnIhYEBELgOc01z1q9qhKkiS1TFI7ZOnXuZnrI+KNdArMQeDszLw2Ik4HrsjM84E3R8QLgfXASuBVzX1XRsRf0Cl2AU7PzJVb057eLlS3m0Mc+viSqIM/XDfGMg/cqyRn0VX3l+QAxL0PlmXduKpyjGrdgQwG19S8YQ1Gf45RrVzwfyBqxqhmv47llDStMvPrwNc3uu59XeffA7xnnPueDZw9WW3p7UJVkiSpT01wOam+ZqEqSZLUNjlls/57iqW6JEmSWqmne1TX7Zrc8q6asWB7vezmkhyAFS9/UknOTuf9pCQHIAfqvhPddP/OZVnEf5VFDRYNhx0o/P5auVerchea66hK2lrJlC1P1VPsUZUkSVIr9XSPqiRJUr9yjKo9qpIkSWope1QlSZJaZroW/G+bni5UD9nubi5+yidLsp752reX5ADcd3DNRIwF//xASQ7AwNBQWdat9+y5+Y0myZ6Dd5RlDa6pyRkoHLw/Wrrgf93jGowsyXHBf6m/Wai661+SJEkt1dM9qpIkSf0occF/sEdVkiRJLdXTPaoPjg7y/eEdS7JOev2FJTkA37rjkJKcwR1rnjuAkfvuK8tatXJOWVYM1g2yrFrwv1IWvgN5zOytMxj27EjVXPC/xwtVSZKkvpROpgJ3/UuSJKml7FGVJElqGddR7ejpQvWWlTvxts+9piTr+j/+eEkOwB4zV5bkfPbA55fkAHDFNWVRM1fMLMuKmXV/QlVjVAejbkdL9uk6qgPUrIXcrx9iWbMMraQe0NOFqiRJUr/q1y+jW8JCVZIkqWVcR7XDyVSSJElqJXtUJUmSWijtUe3tQnXWnWvY78wbS7Je9DvPKckB+OR+Xy7J+bvDdijJAVi0tG6C06wVZVEwo+5PaMaa/pthUjmZqnLB/8GomUwlSf2upwtVSZKkfuWRqRyjKkmSpJayR1WSJKll0kOoAr1eqMYAzBoqiVr5kT1LcgAWfWK7kpyVh9WNo9t59qyyrDkr6sZyVi74XzVGdaBwR0vhsFFGKseoFi3478L4Un9zMpW7/iVJktRSvd2jKkmS1Jdc8B/sUZUkSVJL9XSP6prFQ/z0bfuUZO33jh+U5ADc9rEHSnIOecKvSnIA2GH7sqg5KwrXsJxZtz7s4HD/rc1Zuo5q4TIvAzh4VNLWc4xqjxeqkiRJ/Shx1j+461+SJEktZY+qJElS26RL0IE9qpIkSWqpnu5R3XfBnZz9kn8syXrPt15XkgPwlQdWlOS86jHfL8kB+PSOzy7LmrVybVkWQzUHnAAYXNN/k6lGC9+BKhf8r+JEC6m/VU4CbaueLlQlSZL6UeKXUXDXvyRJklrKHlVJkqTW8chU0OOF6qxI9pmxpiRrxjuXl+QAfO6XTynJ+dqhny3JAfjkgrllWTNWPlSWxSwX/N8aOVg3pbXyDX8w+u//qpQznSU1erpQlSRJ6lcuT+UYVUmSJLWUPaqSJEkt5Kz/Hi9Ub3hgF46+5A0lWTcd86mSHIBDPvv6kpwFh9WNGx3eaVZZ1na/uqssKxfOK8saXLO+LKtKDtZljRTuQBosGmTph5jUvzL9Gwd3/UuSJKmlerpHVZIkqV+5PJU9qpIkSeoSEcdFxI0RsSwiThvj9rdHxHURcXVEXBQRe3fdNhIRS5vT+VvbFntUJUmSWmg6lqeKiEHgY8CzgVuBH0bE+Zl5XddmPwaOzMxVEfEnwIeBk5rbVmfm4ZPVnp4uVGcvH+XgD68qyfrEk/YqyQHY6aqanF+97IGaIGD1wrpZM3MfrFvwP3ddWJYVa/txMlV/Lvg/QP8t+D/gDjip3DRNpjoKWJaZNwNExBeA44FfF6qZeXHX9pcCfzhVjfGdR5IkSRvsDtzSdfnW5rrxvAb4Rtfl2RFxRURcGhEnbG1jerpHVZIkqR8lMVU9qjtFxBVdl5dk5pJH84si4g+BI4Hf7rp678y8LSL2A74dEddk5s8ebWMtVCVJkrYdd2fmkZu4/TZgz67LezTX/TcRcSzwXuC3M3N4w/WZeVvz8+aIuAR4IrCNFqprhskbby6JWrLk90pyAB5z1d0lOV9+4NCSHIA1i+rG2Yw+VDNuGSBn1f0JDd5T97iqZOHgo9E+HOnkYcB7jEsNaQtN09/4D4EDI2JfOgXqycAfdG8QEU8EzgKOy8w7u65fAKzKzOGI2Ak4ms5Eq0ettwtVSZKkfjRNR6bKzPUR8UbgAmAQODszr42I04ErMvN84K+B7YF/iQiAX2XmC4HHAmdFxCideVBnbLRawBazUJUkSdKvZebXga9vdN37us4fO879vg9M6u5aC1VJkqQ2cnxPHw7akiRJUl+Ylh7ViJgPfBJ4PJ3vC38E3Ah8EdgH+AXw0sy8Z1O/Z91Oc7njpZuauDZ5djvrxyU5UPcF6nO/eHJREqxZVBZFrl9XljUyp+5ABjPuqHtcZWbUdReMFI71Goz+W/BfUr1pWvC/VaarR/XvgW9m5iHAYcD1wGnARZl5IHBRc1mSJGmblDn5p15TXqhGxI7AM4BPAWTm2sy8l87huc5pNjsHOKG6bZIkSWqP6dj1vy9wF/DPEXEYcCXwFmBxZt7ebLMcWDwNbZMkSZp2ibv+YXoK1RnAEcCbMvOyiPh7NtrNn5kZEWN2UEfEqcCpAPN3m81Jr79wqtsLwHe+fXhJDsD6G24qybn/qp1KcgDWLVpfllVpZKhwp8S6/hujmoN1+6FGK8eoFo0079sPsX59XJK22HSMUb0VuDUzL2sun0encL0jInYDaH7eOdadM3NJZh6ZmUdut3CopMGSJEmlks6Xtsk+9ZjyQjUzlwO3RMTBzVXHANcB5wOnNNedAnylum2SJElqj+la8P9NwOcjYgi4GXg1naL5SxHxGuCXwEunqW2SJEnTrhdn6U+2aSlUM3MpMNYCqMdsye/ZdXCYdy9cNilt2pyz3vXbJTkAj31nzaKjO11Vt9bjihPXlGXFjJllWSOzC3dKDK+tyyoSpWNU6/6vBsYeYi9JW8a3Eo9MJUmSpHaarl3/kiRJGlf078oeW8AeVUmSJLWSPaqSJElt5BjV3i5Ulw3P4/ibnluSdcmxf1eSA/Dax76xJGfHq1eW5ADMeu1wWdbAnNllWetnF+6WWT9SEjNK3SQ7CidTjdCHu9D8EJP6V/bxQT22wIQK1YjYHdi7e/vM/M5UNUqSJEnabKEaER8CTqKzKP+GLp0ELFQlSZKmintNJtSjegJwcGbW7buVJEnSNm8iherNwEygdYXqyPIh7v2bvUqy7juzbjjv3YfNLclZ/MkbSnIADthxsCzrth3mlWVVjlHNtX244P9A3XjY9aN1r8HBynG+kvqYY1QnUn2tApZGxEV0FauZ+eYpa5UkSdK2zl3/EypUz29OkiRJUpnNFqqZeU5EDAEHNVfdmJnrprZZkiRJ2zh7VCc06/+ZwDnAL+gMltgzIk5pw/JUcd8qZp9/eUnWCce8tSQHYPTwmu8BOw+vKckBOGS7+8uybt1x97KskVmFY1TXry/JGcm68ZUDM+rehUcLx3oNFn26uMaipH43kV3/fwM8JzNvBIiIg4BzgSdNZcMkSZK2WQn4ZZSBCWwzc0ORCpCZP6WzCoAkSZI0ZSbSo3pFRHwS+Fxz+eXAFVPXJEmSJKVjVCdUqP4J8AZgw3JU3wX+95S1SJIkSU6mYmKz/oeBv21OrZLz5rL26CeXZB3yN7eW5ADMO3d1Sc79u+xckgNw4KybyrK+uajmgAkAI7PLomBkZPPb9JiBwbqJWyOO9doqA4WT0cIPZ0mNcQvViPhSZr40Iq5hjJo+M58wpS2TJEnalvkFe5M9qm9pfr6goiGSJElSt3Fn/Wfm7c3Z12fmL7tPwOtrmidJkrRtipz8U6+ZyGSqZwPv3ui6541xXbnYdR0z3rm8JCtf/EBJDsApu15akvM3h/xBSQ7APjNWlGUNLxwqyxqZVRZFFo1RHS0cvV85RnU0J7Ia3+QYKPo0cEaw1McSJ1Ox6TGqf0Kn53S/iLi666YdgO9NdcMkSZK0bdtUj+r/Ab4B/BVwWtf1D2TmyiltlSRJ0jYtnEzFpgvVzMxfRMQbNr4hIhZarEqSJGkqba5H9QXAlXRGSXSX9QnsN4XtmpADZ9/HBY89vyTr0Dc9ol6fMr81699Lct5z+JySHIDFg3VrgK5eNFiWVbmOao7WDFYapW7c6OBA5RjVup6JQQeWSZoMvpWMX6hm5guan/vWNUeSJEmAhSqbWJ5qg4g4OiK2a87/YUT8bUTsNfVNkyRJ0rZsIuu1fBxYFRGHAX8K/Az47JS2SpIkaVuXU3DqMRMpVNdnZgLHA/+YmR+js0SVJEmSNGUmsuD/AxHxHuAVwNMjYgCYObXNmpi7Rob42L37lGR94JX/pyQHYPuBmhk69x22riQHYMeBupXx1yysmzRTueA/WTPxaKRwFfkZlQv+03/LvKRL10j9K3F5KibWo3oSMAz8UWYuB/YA/npKWyVJkqRt3mYL1aY4/TywY0S8AFiTmZ+Z8pZJkiRtwyIn/9RrJjLr/6XA5cDvAy8FLouIE6e6YZIkSds0J1NNaIzqe4EnZ+adABGxM/At4LypbNhE3H3Xjnxqye+WZP3gnX9XkgNww7r1JTlPf9xPS3IAZkXdsObhRWVRjMzuwb/6zRgtfCcrHaPqgv+S1HMmMkZ1YEOR2lgxwftJkiSpx0TEcRFxY0Qsi4jTxrh9VkR8sbn9sojYp+u29zTX3xgRz93atkykR/WbEXEBcG5z+STg61sbLEmSpHaJiEHgY8CzgVuBH0bE+Zl5XddmrwHuycwDIuJk4EPASRHxOOBk4DeAxwDfioiDMvNRH0d9IpOp3gmcBTyhOS3JzHc/2kBJkiRt3jRNpjoKWJaZN2fmWuALdNbS73Y8cE5z/jzgmIiI5vovZOZwZv4cWNb8vkdt3B7ViDgQ+AiwP3AN8I7MvG1rwibbzLtWsdvHf1SSdfQxryzJAThh76tLcl65y/dKcqqt3almjC9ArOu/Ne5GKseoDlSOUe3DEUv9usZinw7x7cUZ15pmU/M3vlNEXNF1eUlmLum6vDtwS9flW4GnbPQ7fr1NZq6PiPuARc31l2503923prGb2vV/NvAZ4DvA7wH/ALx4a8IkSZI0re7OzCOnuxETtalCdYfM/Kfm/I0RUdN1KUmStK2bvuWkbgP27Lq8R3PdWNvcGhEzgB3pTLafyH23yKb2hc2OiCdGxBERcQQwZ6PLkiRJ6i8/BA6MiH0jYojO5KjzN9rmfOCU5vyJwLczM5vrT25WBdgXOJDOWvyP2qZ6VG8H/rbr8vKuywk8a2uCJUmStAnT0KPajDl9I3ABMAicnZnXRsTpwBWZeT7wKeCzEbEMWEmnmKXZ7kvAdcB64A1bM+MfNlGoZubvbM0vLjFriIF999z8dpNgpw/PKckB+OJbazqs3/nUpSU5AMOFE1nmLFxdlrXmvtllWVUqF/yfObhV719bpHLB/4E+neMkqdZ0TcDLzK+z0VKkmfm+rvNr6ByxdKz7fhD44GS1pQ+nwUqSJKkfTGTBf0mSJFVzSTN7VCVJktROE+pRjYjdgb27t8/M70xVoyZqza6DXP+uHUuyDvyjKza/0STJZz2tJGfWb84syQG4c2RVWdbuC+4ty/r5mp3Ksoia75UjWThGdaBujOr6wnHSg0XdIIX/VZKmg3/jmy9UI+JDwEl0ZnBt+FRJOgcCkCRJkqbERHpUTwAOzszhKW6LJEmS6Mz497C7EytUbwZmAhaqkiRJVQqX1WuriRSqq4ClEXERXcVqZr55ylo1QQfvcAfnH/t3JVkn/8E7S3IAdl5aM27v+nVrSnIA1mXdAhMHzrurLOvWe+aXZcXgYEnOaElKR7+uoypJmhwTqR7O55GHzpIkSdJUctf/5gvVzDwnIuYAe2XmjQVtkiRJkja/jmpE/B6wFPhmc/nwiLCHVZIkaQptmFA1madeM5GFBT8AHAXcC5CZS4H9pqxFkiRJ6uz6n+xTj5nIGNV1mXlfxH+biFA532Jc6wlWjtYsWv+Et15VkgNwy8sWl+R84Z6jSnIAnrTdz8uyDp57R1nWpbP2Kcuqmky1tnAV+aHCBf8rJ1OVHVqgBz90JGlLTKRQvTYi/gAYjIgDgTcD35/aZkmSJG3DenRX/WSbyBf/NwG/QWdpqv8D3A+8ZSobJUmSJE2kR/Vlmfle4L0broiIM4DTpqxVkiRJ2zp7VCdUqL4kItZk5ucBIuIfgTlT26yJ+dm9i3nJV2o6d3/2+2eV5AAc98sjS3LOW3Z4SQ7A9ofUHVzg8bNvLcvabmhtWVbMrDloQuUA9KHCBf/XjtSM8e1Xg1E28taj8UgbWKhOrFAFzo+IUeA44N7MfM3UNkuSJEnbunEL1YhY2HXxtcC/Ad8D/jwiFmbmyilumyRJ0jbLyVSb7lG9kk6nc3T9/N3mlLiWqiRJkqbQuIVqZu5b2ZBHY/Ydaznkb2rGI77tt55UkgOQIzXj9gaW7lCSA3DDHruVZb1gh6vLsubNqht7mzNqxqiuKxwfWLmO6pqRmucPYLDoKUzHckrqc5t9546ImcCfAM9orroEOCsz101huyRJkrSNm0gXw8eBmcD/bi6/ornutVPVKEmSpG2eY1Q3OZlqRmauB56cmYd13fTtiKg7nqgkSdK2xiNTAZs+MtXlzc+RiNh/w5URsR9QN7BMkiRJ26RN7frfMEr/HcDFEXFzc3kf4NVT2agJGxkl73ugJOoHf//kkhyARY/5ZUnOzletL8kB+NkzFpVl7bpH3VfQeTOHy7LuH5pZkjNC5WSqutfgaM4uy5KkSWGP6iYL1Z0j4u3N+bOADYd1GQGeCFw8lQ2TJEnStm1TheogsD08ontlBlC3rpEkSdK2yB7VTRaqt2fm6WUtkSRJEtDpJXQy1cTGqLbW2l1m88vX/kZJ1h4fvLQkB2D1s48oydnumjtKcgCW3TO/LGtezCrL2nHm6rKs+2fOL8lZl5uaYzm5ZhQu+D9auDj+YNXbpx9ikvrcpgrVY8paIUmSpP/OL6PjL0+VmSsrGyJJkiR1qzv4tSRJkibGBf+BHi9UH7NoJaef8rmSrCXfelFJDsBdhw2V5Ox+0a0lOQDrVuxaljUz6l7W82euKsu6ZebOJTmV66jOKlxHdf1o3dhbSZoUFqqbPDLVlIqIwYj4cUR8tbm8b0RcFhHLIuKLEVFTrUmSJKmVprOL4S3A9V2XPwR8NDMPAO4BXjMtrZIkSWqDnIJTj5mWQjUi9gB+F/hkczmAZwHnNZucA5wwHW2TJElSO0zXGNW/A97Fw0e4WgTcm5kbBqzdCuw+De2SJElqBSdTTUOhGhEvAO7MzCsj4pmP4v6nAqcC7Ln7IM+fe/fkNnAcH3h33aSP1avqFpGvMnT34HQ3YUrsOKPw/2rWzJKYdVn3f1U5mSoLF/yv2lVV+ZhK+eEsqTEdPapHAy+MiOcDs4F5wN8D8yNiRtOrugdw21h3zswlwBKAIw6b5duZJEnqT1Y59WNUM/M9mblHZu4DnAx8OzNfDlwMnNhsdgrwleq2SZIktcJUTKTqwcK3TQsLvht4e0QsozNm9VPT3B5JkiRNo2ld8D8zLwEuac7fDBy1Jfe/YdUinnblKye/YWP4wZPPLskBeO/yp5fkLNutbhH+WYUH5F2dw2VZOw7WLfifQzV/rmsLx6jOHBgtyxotPJCBJE0GJ1O1q0dVkiRJ+rWePoSqJElS37JH1R5VSZKkNoqc/NNWtSdiYURcGBE3NT8XjLHN4RHxg4i4NiKujoiTum77dET8PCKWNqfDN5dpoSpJkqSJOA24KDMPBC5qLm9sFfDKzPwN4Djg7yJiftft78zMw5vT0s0F9vSu/xnLg50/PLsk6+JPLyzJAfjDRd8vyXnX4/+kJAdgzoq6/RcrR9eWZVVOphqtWvCfwgX/Y11Z1mjh4viDUZTlbsHe4v+XtlT7XjPHA89szp9DZ0L8u7s3yMyfdp3/r4i4E9gZuPfRBNqjKkmSpIlYnJm3N+eXA4s3tXFEHAUMAT/ruvqDzZCAj0bErM0F9nSPqiRJUl+augX6d4qIK7ouL2mO+glARHwLGGv9yvf+t+ZlZsT4o14jYjfgs8ApmblhLcL30Clwh+gcZfTdwOmbaqyFqiRJUstEc5oCd2fmkePdmJnHjndbRNwREbtl5u1NIXrnONvNA74GvDczL+363Rt6Y4cj4p+Bd2yusb1dqD64mvje1SVR7/rcKSU5AFf/8T+U5Nx1eM2YR4Cdr1lflrV8ZLN7EibNvME1ZVmjs2vGjq4rXPB/1kDd6yILx6hKUp86n85h7s9gnMPdR8QQ8GXgM5l53ka3bShyAzgB+MnmAh2jKkmS1EY5Baetcwbw7Ii4CTi2uUxEHBkRn2y2eSnwDOBVYyxD9fmIuAa4BtgJ+MvNBfZ2j6okSZJKZOYK4Jgxrr8CeG1z/nPA58a5/7O2NNNCVZIkqYW2doH+ftDTherIwu24//lPKcna98zrS3IA+OOamOHD6tYAHbqk7q/t5rU7l2XtPOP+sqzRWTVjR9dm3dvCzMIxqiOjdSOdBqZqCsQjOO5WUn/r6UJVkiSpb9mjaqEqSZLUShaqzvqXJElSO9mjKkmS1DbpZCro8UJ17uLVHPaWq0qybrlkfkkOwJVra3JeePA1NUHADffsW5a1bHiThx6eVPsN3VWWNdKHC/7PjrrJVKMu+N87/HCW1OjpQlWSJKlv+aXNQlWSJKmN3PXvZCpJkiS1VE/3qO4980E+sccPSrIO+NPXleQAfG7F00pyXr3Tf5bkALzv3kVlWTc8uGtZ1ovm/bgsa/2smjGW6yoX/C8co5qFPRODVQvx29si9Tf/xu1RlSRJUjv1dI+qJElSv3KMqoWqJElS+yTu+qfHC9VfrduOt/zXk0uyzn3RP5TkALzse39cknPmsy4tyQHIh1aVZd18/y5lWXN3Hy3LGpldM1JnbeE6qjNjpCxrtGrcqCRp0vR0oSpJktS37FF1MpUkSZLayR5VSZKklgmcTAX2qEqSJKmlerpH9aE753LZmU8qyXrPBy8uyQGYu3ROSc663ymcyLJ6dVnW8pXzyrJmR90EnfWzanKGc2ZNEDB7YF1ZVmYfTqYq7G0ZsF9DY7DHb4r5/PZ2oSpJktSvovKQei3lV2RJkiS1kj2qkiRJbeOC/0CPF6qDKx5ix89eVpL1tGPeXJIDsO9Va0tyLh+uG4uYI3XjYUdWFA3mBGZH3eL4I7Nrxliuzbq3hbkDNa91gJHRujGqAx5cQJImRU8XqpIkSf3KyWoWqpIkSe1koepkKkmSJLVTb/eobjcHDju0JOqQv36wJAcgVt5fkvPZu59WktPxUFnSrBV140ZnRt13vap1VNdl5fO3viyrP9dR7cPHBPYiSQ13/dujKkmSpJbq7R5VSZKkfmWPqoWqJElS66S7/sFd/5IkSWqpnu5RXbdr8l/vqpmMscdJPy/JARhZV/OYvnX9E0tyAA4euqYsa9bKsihmULngf03O8GjdgSCGou5AEJWTqQajTyc5Saplj6o9qpIkSWqnnu5RlSRJ6keBY1TBQlWSJKmd0kq1pwvVQ7a7m/948tklWU8/9W0lOQCLP/HDkpwdlhatIA/E3LllWXNW1P1hDxYu+F81RrVfF/wf7cPF8f0Mk9TverpQlSRJ6lfu+ncylSRJklrKHlVJkqS2SVyeih4vVB8YncHFaxaWZL38df9ekgNw8TcOLcnZ+erhkhyAmLd9Wdbsu+vGPQ4U7pQYKRpSvKZwHdWZ1K2jOjraf2NUJanf9XShKkmS1K9idLpbMP0sVCVJktrIXf9OppIkSVI72aMqSZLUQi5P1eOF6q0rF/Guz51SknXtqR8ryQE4/9BjSnJ2uOyXJTkAufOCsqxZK9eUZVUanV3zjlW54P9Q1E2mysIF/8sm2fkhJqlQRCwEvgjsA/wCeGlm3jPGdiPANc3FX2XmC5vr9wW+ACwCrgRekZlrN5Xprn9JkqS2STqHn5vs09Y5DbgoMw8ELmouj2V1Zh7enF7Ydf2HgI9m5gHAPcBrNhdooSpJktRCkZN/2krHA+c0588BTpjwY4kI4FnAeVtyfwtVSZIkTcTizLy9Ob8cWDzOdrMj4oqIuDQiTmiuWwTcm5kbFju/Fdh9c4E9PUZ11h1r2PfM60uyjn/m80tyAO46vGaM4Jyv3FmSAzB68GPKsoZuu68sq9LorJoF9YZH694WZkbdwRkcztlDCscTS602NW9cO0XEFV2Xl2Tmkg0XIuJbwK5j3O+9/61pmRkxbh/t3pl5W0TsB3w7Iq4BHtWHc08XqpIkSdoid2fmkePdmJnHjndbRNwREbtl5u0RsRswZo9XZt7W/Lw5Ii4Bngj8X2B+RMxoelX3AG7bXGPd9S9JktQyQSvHqJ4PbFhu6RTgK49od8SCiJjVnN8JOBq4LjMTuBg4cVP335iFqiRJUttMxYz/rZ/1fwbw7Ii4CTi2uUxEHBkRn2y2eSxwRURcRacwPSMzr2tuezfw9ohYRmfM6qc2F9jbu/4HBog5c0qiHvjIHiU5AIN/VDPGcmBm3X//qkVDZVlD191fllVqds2ao7VjVOsOZF25jmqZfnxMklorM1cAj1jsPTOvAF7bnP8+cOg4978ZOGpLMnu7UJUkSepTHpnKXf+SJElqKXtUJUmS2sgeVXtUJUmS1E493aO6ZvEQN/7p3iVZ+7/9ByU5ACf/r+GSnB/s/biSHIDVi+q+E23/4ENlWeuybsH6gVk1k6nWj9YccAJgkMLJVKNOPJLUWxyj2uOFqiRJUl9KYNRK1V3/kiRJaiV7VCVJktrIDtXeLlT3W3Ann3nxmSVZ77jw9SU5ACfu+PclORcc+oySHIA1i8qiyOGaMb4Ao4XvIkOzasbD9u+C/2VRDFAzHtbxa5L6XU8XqpIkSf3KL6MWqpIkSe1UuSuopZxMJUmSpFbq6R7VmTHKroM14xFnv+u/SnIADpm5XUnOXYfXrZc5MrfuW2GO1Kw3CjBcuI7q7FnrSnLWFq6jOlS5jmq6jqqk3uKu/2noUY2IPSPi4oi4LiKujYi3NNcvjIgLI+Km5ueC6rZJkiSpPaZj1/964E8z83HAU4E3RMTjgNOAizLzQOCi5rIkSdK2J6fo1GPKd/1n5u3A7c35ByLiemB34Hjgmc1m5wCXAO+ubp8kSdJ0CyCcTDW9k6kiYh/gicBlwOKmiAVYDiyernZJkiRp+k3bZKqI2B74v8BbM/P+iIcnOmRmRow9hDgiTgVOBRhcNJ+nf/stFc1l2bM/WZID8ODo2pKc2YfdU5IDcP/KmgliAETd9691hZOBtp9VM3FwbeGC/wOV85v6cTJVn3a2OIFEatR9xLTWtPSoRsRMOkXq5zPzX5ur74iI3ZrbdwPuHOu+mbkkM4/MzCMHdygsfiRJklRqOmb9B/Ap4PrM/Nuum84HTmnOnwJ8pbptkiRJbRGZk37qNdOx6/9o4BXANRGxtLnufwJnAF+KiNcAvwReOg1tkyRJUktMx6z//6QzmW0sx2zJ75q9fIRDPvzA1jdqAv7hyP1KcgAOmLW8JOdl+19RkgNwLkeWZQ0MDZVlrcm6AUQ7DBWNUR2pW/B/ZuEgy8L/Kknaej26nNRk6+kjU0mSJPWnhB7cVT/ZpnV5KkmSJGk89qhKkiS1kEu12aMqSZKklurtHtXhteRNvyiJ+vRZzyvJATjopT8tyflfe9WtAPb9+fuXZa3bbk5Z1qrRukXk5w+tLsm5d23d8zdYuAZ/Fi74P1h10Il+PIiBpIc5RrXHC1VJkqR+lBCuVuKuf0mSJLWTPaqSJElt5K7/3i5U1+08l+Un1ywkv+tZPyrJAbj88U8oydl/v+1LcgAO2OGusqwbdnhMWdZDWfcntOPMmjGqd66ue10Mjnvsjyng+70k9ZyeLlQlSZL6ll+wLVQlSZLaKNz172QqSZIktVNP96gu2vl+XvG6b5ZkfetbNWNhARYsrflvuf/5NWMeAQ6Zc3tZ1nXzDy7LemB0dlnWvBlrSnLWjQ6W5ADMjLoxqpXrqJaxs0Xqb/ao2qMqSZKkdurpHlVJkqS+lIAL/tujKkmSpHayR1WSJKllgnTWPz1eqO4yOMxbF/ysJOsf3/2skhyAA5esKsn5jzWLSnIADpy1vCxr3cI5ZVn3F06mWjDzoZKcdSN1k6kGShf878PJVP3Kz2apw0LVXf+SJElqp57uUZUkSepb9qjaoypJkqR26uke1WXDO/J7P/3dkqzvHvN3JTkAp/7p8SU5n7796JIcgDP2+nJZ1ppFM8uyHhitGw+742DNARrWj9R9fx0sHaNaF1WmHx+TpA6XpwJ6vFCVJEnqV876d9e/JEmSWspCVZIkqY0yJ/+0FSJiYURcGBE3NT8XjLHN70TE0q7Tmog4obnt0xHx867bDt9cZk/v+h9dPpOHPrJHSdZd/1g37nH9ihUlOT/+yVNKcgAW71u3Nufqneq+f907Mrcsa/5gzfq660b7dR3VuqgB+wA0hnAvrnrfacBFmXlGRJzWXH539waZeTFwOHQKW2AZ8O9dm7wzM8+baKDvppIkSa0zBb2pWz/m9XjgnOb8OcAJm9n+ROAbmfmoe1osVCVJktommapCdaeIuKLrdOoWtGpxZt7enF8OLN7M9icD52503Qcj4uqI+GhEzNpcYE/v+pckSdIWuTszjxzvxoj4FrDrGDe9t/tCZmbE+ANaImI34FDggq6r30OnwB0CltAZNnD6phproSpJktRG07COamYeO95tEXFHROyWmbc3heidm/hVLwW+nJnrun73ht7Y4Yj4Z+Adm2tPTxeqcd8qZn318pKsE499S0kOwIFzry7JWXBV3aSZ7Y/fbO/+pFmzqCyK+0bqFvzfc2bNJLt16+teF4NROZmqMKuKk3Mk1TofOAU4o/n5lU1s+zI6Pai/1lXkBp3xrT/ZXGBPF6qSJEn9qoUL/p8BfCkiXgP8kk6vKRFxJPC6zHxtc3kfYE/gPza6/+cjYmcggKXA6zYXaKEqSZKkzcrMFcAxY1x/BfDarsu/AHYfY7tnbWmmhaokSVIbta9HtVxPF6q541zWPv2okqyDP3JLSQ5A7rdnSc5OSx8qyYHaBdCHF9X9Yd+3vm7B/3kDa0pyRkb6c9W6nIZJCZL0qCUwaqHan59IkiRJ6nk93aMqSZLUnyblSFI9zx5VSZIktVJP96gOLF7H7HfcVpKVL6obz3nPb+9VkrPgGzeU5ACMFq5aPLpwbVnWvevq1lGdO1DzuEZH69YbrRy73J/rqPbhY5L0MHtUe7tQlSRJ6lsWqu76lyRJUjvZoypJktQ2Lk8F2KMqSZKklurpHtUDZt/H1w7+aknW49/8hpIcgHXb13yDmvf5e0pyAFaN1k1wmr+obuLbfYWTqeZFzXNYueD/AHWTgcKOCUk9JT1SCT1eqEqSJPUtJ1O561+SJEntZI+qJElS2ziZCujxQvXOkSH+4Z79SrL+1ys+U5ID8Nc/e25JzsDcuSU5AHePrivL2nvHurG396+dXZY1q2iQZY7UjRsdjMKdOoUHMqjiuFtJ/a6nC1VJkqS+5RhVx6hKkiSpnexRlSRJaiN7VHu7UF1x1458+qznlWR9910fLckBWL73ZSU55x/w9JIcgNvWb1eWdfAOd5RlXXr3PmVZs6NmjOVo4TqqpXy/l9RT0kIVd/1LkiSppXq6R1WSJKkvJTDqkansUZUkSVIr2aMqSZLURo5R7e1CdeZdq9j1rB+VZP3WMa8qyQH42hH/VJLz6cN+ryQH4OZ1u5RlHTRneVnWt4YPLsuaHYM1QX24MD7Qn5Op+vExSXqYhaq7/iVJktROPd2jKkmS1J8SRu1RtUdVkiRJrdTbPaqzhogD9iqJWvyhWSU5ALuft0NJzt2Hl8QAcP3qx5RlPW/e1WVZq9YMlWXNjJrvlbm+bozqQOV3ZTsmekb4fyU16/27PFVvF6qSJEn9yl3/7vqXJElSO9mjKkmS1EYuT9XbheqaXQe54Z014zkPfPWPS3IAVow+VJKzxxNuL8kBuOnBunVUX7PwwbKsdWvr/oRmULOOaoz05zqq0a/rw0pSH+vpQlWSJKkvZcKok6kcoypJkqRWskdVkiSpjRyjaqEqSZLURumu/94uVA+edwdfO+bvS7JOfPk7S3IAvv7QLSU5r9zz0pIcgP+97LfLsnbep2bSEcDIcF3WYNGC//TpZCoX/Jek3tPThaokSVJ/Snf942QqSZIktZQ9qpIkSW2TeAhVerxQXZcDLB+ZVZL1pLf+qCQH4NO3/GZJzucPPrckB+CDK59flrX9QM1rAoA1dWNUB4p2gPTrgv/9KPwMk/pbOpnKXf+SJElqpZ7uUZUkSepHCaS7/u1RlSRJUjv1dI/qzffuwklffnNJ1rKTPlGSA3DAd19XkrPr47YvyQGIFXXjRqvGcgIMrOm/73r9OkY1RvvwcWUfPibwcfWSfnxMbZHpGFVa1qMaEcdFxI0RsSwiTpvu9kiSJE2XHM1JP/Wa1hSqETEIfAx4HvA44GUR8bjpbZUkSZKmS5t2/R8FLMvMmwEi4gvA8cB109oqSZKk6eCu//b0qAK7A90Hub+1uU6SJEnboMiWHEc2Ik4EjsvM1zaXXwE8JTPfuNF2pwKnNhcfD/yktKHttRNw93Q3oiV8Lh7mc/Ewn4uH+Vw8zOfiYT4XDzs4M3eYzgZExDfp/J9Mtrsz87gp+L1Tok27/m8D9uy6vEdz3X+TmUuAJQARcUVmHlnTvHbzuXiYz8XDfC4e5nPxMJ+Lh/lcPMzn4mERccV0t6GXismp1KZd/z8EDoyIfSNiCDgZOH+a2yRJkqRp0poe1cxcHxFvBC4ABoGzM/PaaW6WJEmSpklrClWAzPw68PUtuMuSqWpLD/K5eJjPxcN8Lh7mc/Ewn4uH+Vw8zOfiYT4XLdGayVSSJElStzaNUZUkSZJ+rScK1c0dWjUiZkXEF5vbL4uIfaahmVMuIvaMiIsj4rqIuDYi3jLGNs+MiPsiYmlzet90tLVCRPwiIq5pHucjZmhGx5nN6+LqiDhiOto51SLi4K7/76URcX9EvHWjbfr2dRERZ0fEnRHxk67rFkbEhRFxU/NzwTj3PaXZ5qaIOKWu1VNjnOfiryPihuZv4MsRMX+c+27y76nXjPNcfCAibuv6O3j+OPftq8N5j/NcfLHrefhFRCwd575987oY7zN0W32/6BmZ2eoTnYlVPwP2A4aAq4DHbbTN64FPNOdPBr443e2eoudiN+CI5vwOwE/HeC6eCXx1utta9Hz8AthpE7c/H/gGEMBTgcumu80Fz8kgsBzYe1t5XQDPAI4AftJ13YeB05rzpwEfGuN+C4Gbm58LmvMLpvvxTMFz8RxgRnP+Q2M9F81tm/x76rXTOM/FB4B3bOZ+m/3M6bXTWM/FRrf/DfC+fn9djPcZuq2+X/TKqRd6VH99aNXMXAtsOLRqt+OBc5rz5wHHREQUtrFEZt6emT9qzj8AXI9H79qU44HPZMelwPyI2G26GzXFjgF+lpm/nO6GVMnM7wArN7q6+z3hHOCEMe76XODCzFyZmfcAFwI9vW7hWM9FZv57Zq5vLl5KZ43qvjfO62IiJvKZ01M29Vw0n5UvBc4tbdQ02MRn6Db5ftEreqFQncihVX+9TfOGfB+wqKR106QZ3vBE4LIxbv7NiLgqIr4REb9R27JSCfx7RFwZnSOWbWxbPCzvyYz/gbOtvC4AFmfm7c355cDiMbbZFl8ff0RnL8NYNvf31C/e2AyDOHucXbzb2uvi6cAdmXnTOLf35etio89Q3y9arBcKVW0kIrYH/i/w1sy8f6Obf0Rnt+9hwD8A/1bcvEq/lZlHAM8D3hARz5juBk2n6Bwo44XAv4xx87b0uvhvsrPfbptf3iQi3gusBz4/zibbwt/Tx4H9gcOB2+ns8t7WvYxN96b23etiU5+hvl+0Ty8UqhM5tOqvt4mIGcCOwIqS1hWLiJl0/sA+n5n/uvHtmXl/Zj7YnP86MDMipuJYwdMuM29rft4JfJnOLrtuEzosbx95HvCjzLxj4xu2pddF444Nwzyan3eOsc028/qIiFcBLwBe3nwQP8IE/p56XmbekZkjmTkK/BNjP8Zt6XUxA3gx8MXxtum318U4n6G+X7RYLxSqEzm06vnAhhl4JwLfHu/NuJc1Y4k+BVyfmX87zja7bhifGxFH0fk/7ruiPSK2i4gdNpynM2HkJxttdj7wyuh4KnBf1+6dfjRuz8i28rro0v2ecArwlTG2uQB4TkQsaHYBP6e5rq9ExHHAu4AXZuaqcbaZyN9Tz9tojPqLGPsxbkuH8z4WuCEzbx3rxn57XWziM9T3izab7tlcEznRmb39UzozMd/bXHc6nTdegNl0dncuAy4H9pvuNk/R8/BbdHZJXA0sbU7PB14HvK7Z5o3AtXRmql4KPG262z1Fz8V+zWO8qnm8G14X3c9FAB9rXjfXAEdOd7un8PnYjk7huWPXddvE64JOcX47sI7OuLHX0BmjfhFwE/AtYGGz7ZHAJ7vu+0fN+8Yy4NXT/Vim6LlYRmds3Yb3jA0rpDwG+Hpzfsy/p14+jfNcfLZ5L7iaTnGy28bPRXP5EZ85vXwa67lorv/0hveIrm379nWxic/QbfL9oldOHplKkiRJrdQLu/4lSZK0DbJQlSRJUitZqEqSJKmVLFQlSZLUShaqkiRJaiULVWkbFxEjEbG063TaFtz3mRHx1a3IHvf+EfGLDQcliIjvP9qMMfLui4gfR8SNEfGdiHhB1+2vi4hXTkbWFrbryIg4szpXktpuxnQ3QNK0W52Zh093IzYlM582ib/uu5n5AoCIOBz4t4hYnZkXZeYnJjFnwjLzCuCK6ciWpDazR1XSmJoezb9qelmviIgjIuKCiPhZRLyua9N5EfG1pofyExEx0Nz/ORHxg4j4UUT8S3N8bSLiuIi4ISJ+ROfwjRvyFkXEv0fEtRHxSToHbNhw24PNz2dGxCURcV7zOz7fdcSt5zfXXRkRZ06kpzczl9I5eMgbm9/xgYh4R3P+koj4aPPYr4+IJ0fEv0bETRHxl11t+8OIuLx5ns6KiMENbY6ID0bEVRFxaUQsbq7//Yj4SXP9d7oe11eb8wsj4t8i4urmfk/oatvZTbtujog3N9dv1zz/VzW/96Qt+X+WpDazUJU0Z6Nd/92Fzq+a3tbv0jmKzYnAU4E/79rmKOBNwOOA/YEXN7vs/ww4NjOPoNNb+PaImE3nGOu/BzwJ2LXr97wf+M/M/A06xxTfa5z2PhF4a5O3H3B083vPAp6XmU8Cdt6Cx/8j4JBxblubmUcCn6BzWMU3AI8HXtUU1o8FTgKObp6nEeDlzX23Ay7NzMOA7wB/3Fz/PuC5zfUvHCPzz4EfZ+YTgP8JfKbrtkOA59J5zt8fneOWHwf8V2YelpmPB765BY9dklrNXf+SNrXrf8Mxzq8Bts/MB4AHImI4IuY3t12emTcDRMS5dA5TuIZOIfm9psNzCPgBnULr55l5U7P954BTm9/zDJoe1sz8WkTcM06bLs/m2OQRsRTYB3gQuDkzf95sc27X792c2MRt3Y//2sy8vcm9GdizeaxPAn7YPM45wJ3NfdYCG3p1rwSe3Zz/HvDpiPgS8K9jZP4W8BKAzPx2UxDPa277WmYOA8MRcSewuGnb30TEh4CvZuZ3J/i4Jan1LFQlbcpw83O06/yGyxvePzY+DnPSKf4uzMyXdd/QjAmdrDZBpwdza9/Hnghcv5ms8R5/AOdk5nvGuO+6fPgY1b9uZ2a+LiKeAvwucGVEPGkL2vqIx56ZP42II+gcs/wvI+KizDx9C36nJLWWu/4lba2jImLfZmzqScB/ApfS2SV/APx6HOVBwA3APhGxf3Pf7kL2O8AfNNs/D1iwBW24EdgvIvZpLk9onGYz/vP/Az62BVndLgJOjIhdmt+3MCL23kzm/pl5WWa+D7iLTs9st+/SDB+IiGcCd2fm/Zv4fY8BVmXm54C/Bo54lI9FklrHHlVJc5pd6Bt8MzMnvEQV8EPgH4EDgIuBL2fmaES8Cjg3ImY12/1Z0/t3KvC1iFhFpyjbobn9z5vtrwW+D/xqog3IzNUR8XrgmxHxUNOm8Tw9In4MzKWzm/7NmXnRRLM2yr0uIv4M+PemUF9HZxzrLzdxt7+OiAPp9MZeBFwF/HbX7R8Azo6Iq4FVwCmbacahze8cbfL/5NE8Fklqo3h4z5Qk9a6I2D4zH2xWAfgYcFNmfnS62yVJevTc9S+pX/xx0zN8LbAjnVUAJEk9zB5VSZIktZI9qpIkSWolC1VJkiS1koWqJEmSWslCVZIkSa1koSpJkqRWslCVJElSK/3/dS5HcfXc1rQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 864x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "pe = PositionalEncoding(20, 0)\n",
    "y = pe.forward(torch.zeros(1, 100, 20))\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.pcolormesh(y[0], cmap='viridis')\n",
    "plt.xlabel('Embedding Dimensions')\n",
    "plt.ylabel('Token Position')\n",
    "plt.colorbar()\n",
    "plt.show()\n",
    "     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_model(\n",
    "    src_vocab, tgt_vocab, N=6, d_model=512, d_ff=2048, h=8, dropout=0.1\n",
    "):\n",
    "    \"Helper: Construct a model from hyperparameters.\"\n",
    "    c = copy.deepcopy\n",
    "    attn = MultiHeadedAttention(h, d_model)\n",
    "    ff = PositionwiseFeedForward(d_model, d_ff, dropout)\n",
    "    position = PositionalEncoding(d_model, dropout)\n",
    "    model = EncoderDecoder(\n",
    "        Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N),\n",
    "        Decoder(DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout), N),\n",
    "        nn.Sequential(Embeddings(d_model, src_vocab), c(position)),\n",
    "        nn.Sequential(Embeddings(d_model, tgt_vocab), c(position)),\n",
    "        Generator(d_model, tgt_vocab),\n",
    "    )\n",
    "\n",
    "    # This was important from their code.\n",
    "    # Initialize parameters with Glorot / fan_avg.\n",
    "    for p in model.parameters():\n",
    "        if p.dim() > 1:\n",
    "            nn.init.xavier_uniform_(p)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "memory: torch.Size([1, 10, 512])\n"
     ]
    }
   ],
   "source": [
    "# test example\n",
    "test_model = make_model(11, 11, 2)\n",
    "test_model.eval()\n",
    "src = torch.LongTensor([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]])\n",
    "src_mask = torch.ones(1, 1, 10)\n",
    "memory = test_model.encode(src, src_mask)\n",
    "ys = torch.zeros(1, 1).type_as(src)\n",
    "print(f'memory: {memory.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example Untrained Model Prediction: tensor([[0, 4, 8, 8, 8, 8, 8, 9, 4, 5]])\n"
     ]
    }
   ],
   "source": [
    "for i in range(9):\n",
    "    out = test_model.decode(\n",
    "        memory, src_mask, ys, subsequent_mask(ys.size(1)).type_as(src.data)\n",
    "    )\n",
    "    prob = test_model.generator(out[:, -1])\n",
    "    _, next_word = torch.max(prob, dim=1)\n",
    "    next_word = next_word.data[0]\n",
    "    ys = torch.cat(\n",
    "        [ys, torch.empty(1, 1).type_as(src.data).fill_(next_word)], dim=1\n",
    "    )\n",
    "\n",
    "print(\"Example Untrained Model Prediction:\", ys)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !python -m spacy download de_core_news_sm\n",
    "# !python -m spacy download en_core_web_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_tokenizers():\n",
    "\n",
    "    try:\n",
    "        spacy_de = spacy.load(\"de_core_news_sm\")\n",
    "    except IOError:\n",
    "        os.system(\"python -m spacy download de_core_news_sm\")\n",
    "        spacy_de = spacy.load(\"de_core_news_sm\")\n",
    "\n",
    "    try:\n",
    "        spacy_en = spacy.load(\"en_core_web_sm\")\n",
    "    except IOError:\n",
    "        os.system(\"python -m spacy download en_core_web_sm\")\n",
    "        spacy_en = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "    return spacy_de, spacy_en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text, tokenizer):\n",
    "    return [tok.text for tok in tokenizer.tokenizer(text)]\n",
    "\n",
    "\n",
    "def yield_tokens(data_iter, tokenizer, index):\n",
    "    for from_to_tuple in data_iter:\n",
    "        yield tokenizer(from_to_tuple[index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_vocabulary(spacy_de, spacy_en):\n",
    "    def tokenize_de(text):\n",
    "        return tokenize(text, spacy_de)\n",
    "\n",
    "    def tokenize_en(text):\n",
    "        return tokenize(text, spacy_en)\n",
    "\n",
    "    print(\"Building German Vocabulary ...\")\n",
    "    train, val, test = datasets.Multi30k(language_pair=(\"de\", \"en\"))\n",
    "    vocab_src = build_vocab_from_iterator(\n",
    "        yield_tokens(train + val + test, tokenize_de, index=0),\n",
    "        min_freq=2,\n",
    "        specials=[\"<s>\", \"</s>\", \"<blank>\", \"<unk>\"],\n",
    "    )\n",
    "\n",
    "    print(\"Building English Vocabulary ...\")\n",
    "    train, val, test = datasets.Multi30k(language_pair=(\"de\", \"en\"))\n",
    "    vocab_tgt = build_vocab_from_iterator(\n",
    "        yield_tokens(train + val + test, tokenize_en, index=1),\n",
    "        min_freq=2,\n",
    "        specials=[\"<s>\", \"</s>\", \"<blank>\", \"<unk>\"],\n",
    "    )\n",
    "\n",
    "    vocab_src.set_default_index(vocab_src[\"<unk>\"])\n",
    "    vocab_tgt.set_default_index(vocab_tgt[\"<unk>\"])\n",
    "\n",
    "    return vocab_src, vocab_tgt\n",
    "\n",
    "\n",
    "def load_vocab(spacy_de, spacy_en):\n",
    "    if not exists(\"vocab.pt\"):\n",
    "        vocab_src, vocab_tgt = build_vocabulary(spacy_de, spacy_en)\n",
    "        torch.save((vocab_src, vocab_tgt), \"vocab.pt\")\n",
    "    else:\n",
    "        vocab_src, vocab_tgt = torch.load(\"vocab.pt\")\n",
    "    print(\"Finished.\\nVocabulary sizes:\")\n",
    "    print(len(vocab_src))\n",
    "    print(len(vocab_tgt))\n",
    "    return vocab_src, vocab_tgt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished.\n",
      "Vocabulary sizes:\n",
      "8316\n",
      "6385\n",
      "Some words in vocab: ['<s>', '</s>', '<blank>', '<unk>', 'a', '\\n', '.', 'A', 'in', 'the', 'on', 'is', 'and', 'man', 'of', 'with', ',', 'woman', 'are', 'to']\n"
     ]
    }
   ],
   "source": [
    "# build vocabulary\n",
    "spacy_de, spacy_en = load_tokenizers()\n",
    "vocab_src, vocab_tgt = load_vocab(spacy_de, spacy_en)\n",
    "print(f'Some words in vocab: {vocab_tgt.get_itos()[:20]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[9, 10, 13, 15, 14, 17]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# example of tokenize some english words\n",
    "vocab_tgt(tokenize(\"the on man with of woman\", spacy_en))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "original sequence: tensor([1, 2, 3, 4, 5, 6, 7])\n",
      "Add 3 external token to a tensor([1, 2, 3, 4, 5, 6, 7, 0, 0, 0])\n",
      "truncate the sequence a  tensor([1, 2, 3, 4])\n"
     ]
    }
   ],
   "source": [
    "# example of pad\n",
    "a = torch.tensor([1,2,3,4,5,6,7])\n",
    "print(f'original sequence: {a}')\n",
    "\n",
    "# pad使用 pad(input, (padding_left,padding_right), value)\n",
    "# 当长度超过当前sequence的长度会用value进行padding，当长度小于当前sequence长度时，将会截断当前sequence。\n",
    "\n",
    "print(f'Add 3 external token to a {pad(a,(0,3),value=0)}')\n",
    "print(f'truncate the sequence a  {pad(a,(0,-3),value=0)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Batch\n",
    "分批处理对速度非常重要。我们希望有非常均匀的批处理，并将填充量降到最低。要做到这一点，我们必须在默认的Torchtext批处理方面做一些改动。这段代码对他们的默认批处理进行了修补，以确保我们在足够多的句子上进行搜索以找到紧凑的批次。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collate_batch(\n",
    "    batch,\n",
    "    src_pipeline,  # src tokenizer tokenize(\"the on man with of woman\", spacy_en)\n",
    "    tgt_pipeline,\n",
    "    src_vocab,\n",
    "    tgt_vocab,\n",
    "    device,\n",
    "    max_padding=128,\n",
    "    pad_id=2,\n",
    "):\n",
    "    bs_id = torch.tensor([0], device=device)  # <s> token id\n",
    "    eos_id = torch.tensor([1], device=device)  # </s> token id\n",
    "    src_list, tgt_list = [], []\n",
    "    for (_src, _tgt) in batch:\n",
    "        processed_src = torch.cat(\n",
    "            [\n",
    "                bs_id,\n",
    "                torch.tensor(\n",
    "                    src_vocab(src_pipeline(_src)),\n",
    "                    dtype=torch.int64,\n",
    "                    device=device,\n",
    "                ),\n",
    "                eos_id,\n",
    "            ],\n",
    "            0,\n",
    "        )\n",
    "        processed_tgt = torch.cat(\n",
    "            [\n",
    "                bs_id,\n",
    "                torch.tensor(\n",
    "                    tgt_vocab(tgt_pipeline(_tgt)),\n",
    "                    dtype=torch.int64,\n",
    "                    device=device,\n",
    "                ),\n",
    "                eos_id,\n",
    "            ],\n",
    "            0,\n",
    "        )\n",
    "        src_list.append(\n",
    "            # 做padding让同一批次的长度一致\n",
    "            pad(\n",
    "                processed_src,\n",
    "                (\n",
    "                    0,\n",
    "                    max_padding - len(processed_src),\n",
    "                ),\n",
    "                value=pad_id,\n",
    "            )\n",
    "        )\n",
    "        tgt_list.append(\n",
    "            pad(\n",
    "                processed_tgt,\n",
    "                (0, max_padding - len(processed_tgt)),\n",
    "                value=pad_id,\n",
    "            )\n",
    "        )\n",
    "\n",
    "    src = torch.stack(src_list)\n",
    "    tgt = torch.stack(tgt_list)\n",
    "    return (src, tgt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dataloaders(\n",
    "    device,\n",
    "    vocab_src,\n",
    "    vocab_tgt,\n",
    "    spacy_de,\n",
    "    spacy_en,\n",
    "    batch_size=12000,\n",
    "    max_padding=128,\n",
    "):\n",
    "    def tokenize_de(text):\n",
    "        return tokenize(text, spacy_de)\n",
    "\n",
    "    def tokenize_en(text):\n",
    "        return tokenize(text, spacy_en)\n",
    "\n",
    "    def collate_fn(batch):\n",
    "        return collate_batch(\n",
    "            batch,\n",
    "            tokenize_de,\n",
    "            tokenize_en,\n",
    "            vocab_src,\n",
    "            vocab_tgt,\n",
    "            device,\n",
    "            max_padding=max_padding,\n",
    "            pad_id=vocab_src.get_stoi()[\"<blank>\"],\n",
    "        )\n",
    "\n",
    "    train_iter, valid_iter, test_iter = datasets.Multi30k(language_pair=(\"de\", \"en\"))\n",
    "\n",
    "    train_iter_map = to_map_style_dataset(train_iter)\n",
    "\n",
    "    valid_iter_map = to_map_style_dataset(valid_iter)\n",
    "\n",
    "\n",
    "    train_dataloader = DataLoader(\n",
    "        train_iter_map,\n",
    "        batch_size=batch_size,\n",
    "        shuffle = True,\n",
    "        collate_fn=collate_fn,\n",
    "    )\n",
    "    valid_dataloader = DataLoader(\n",
    "        valid_iter_map,\n",
    "        batch_size=batch_size,\n",
    "        shuffle = True,\n",
    "        collate_fn=collate_fn,\n",
    "    )\n",
    "    return train_dataloader, valid_dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of one batch: torch.Size([2, 128])\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cpu\")\n",
    "# torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "train_exp ,text_exp = create_dataloaders(device, vocab_src, vocab_tgt, spacy_de, spacy_en, batch_size=2, max_padding=128)\n",
    "print(f'Size of one batch: {next(iter(text_exp))[0].shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 需要对相应的src, tgt生成mask,主要针对于进行了padding的序列。\n",
    "\n",
    "class Batch:\n",
    "    \"\"\"Object for holding a batch of data with mask during training.\"\"\"\n",
    "\n",
    "    def __init__(self, src, tgt=None, pad=2):  # 2 = <blank>\n",
    "        self.src = src\n",
    "        self.src_mask = (src != pad).unsqueeze(-2)\n",
    "        if tgt is not None:\n",
    "            self.tgt = tgt[:, :-1]  # decoder的输入（即期望输出除了最后一个token以外的部分)\n",
    "            self.tgt_y = tgt[:, 1:]  # decoder的期望输出（trg基础上再删去句子起始符）\n",
    "            self.tgt_mask = self.make_std_mask(self.tgt, pad)\n",
    "            self.ntokens = (self.tgt_y != pad).data.sum()\n",
    "\n",
    "    @staticmethod\n",
    "    def make_std_mask(tgt, pad):\n",
    "        \"Create a mask to hide padding and future words.\"\n",
    "        tgt_mask = (tgt != pad).unsqueeze(-2)\n",
    "        tgt_mask = tgt_mask & subsequent_mask(tgt.size(-1)).type_as(\n",
    "            tgt_mask.data\n",
    "        )\n",
    "        return tgt_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 1, 128])\n",
      "torch.Size([2, 127, 127])\n"
     ]
    }
   ],
   "source": [
    "# show some example\n",
    "pad_idx = vocab_tgt[\"<blank>\"]\n",
    "batch = Batch(next(iter(text_exp))[0], next(iter(text_exp))[1], pad_idx)\n",
    "print(batch.src_mask.shape)\n",
    "print(batch.tgt_mask.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch.src: torch.Size([2, 128]), batch.tgt: torch.Size([2, 127]), batch.src_mask: torch.Size([2, 1, 128]), batch.tgt_mask: torch.Size([2, 127, 127])\n",
      "output shape: torch.Size([2, 127, 512])\n"
     ]
    }
   ],
   "source": [
    "# 训练时，每个位置的输入都是ground truth,输出是更新在当前位置\n",
    "# 因此可以看到batch 中tgt和tgt_y是错开一位的。\n",
    "# 所以在原文中才叫right-shift output\n",
    "# 训练时采用这种策略可以在一定程度上防止错误累加。\n",
    "test_model = make_model(src_vocab = len(vocab_src), tgt_vocab = len(vocab_tgt))\n",
    "print(f'batch.src: {batch.src.shape}, batch.tgt: {batch.tgt.shape}, batch.src_mask: {batch.src_mask.shape}, batch.tgt_mask: {batch.tgt_mask.shape}')\n",
    "output = test_model(batch.src, batch.tgt, batch.src_mask, batch.tgt_mask)\n",
    "print(f'output shape: {output.shape}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.11 ('py38')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "694a5ca4ff500403e563beeba6426a1c919db37382c2fe6c52fb893497b50295"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}