3_signal/notebook-runs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from main import Task, train, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "task = Task()\n",
    "samples, labels = task.next_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 11, 3)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# (batchsize, max_len + 1, vocab_size)\n",
    "samples.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 1., 0.],\n",
       "       [0., 1., 0.],\n",
       "       [0., 1., 0.],\n",
       "       [0., 0., 1.],\n",
       "       [0., 0., 1.],\n",
       "       [0., 0., 1.],\n",
       "       [0., 1., 0.],\n",
       "       [0., 0., 1.],\n",
       "       [0., 0., 1.],\n",
       "       [0., 1., 0.],\n",
       "       [0., 0., 1.]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "samples[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 1, 11)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# (batchsize, 1, max_len + 1)\n",
    "labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Without Positional encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 250 - Loss 1.6700439453125\n",
      "Iteration 500 - Loss 1.3713023662567139\n",
      "Iteration 750 - Loss 1.5837630033493042\n",
      "Iteration 1000 - Loss 1.2754212617874146\n",
      "Iteration 1250 - Loss 1.1670920848846436\n",
      "Iteration 1500 - Loss 1.1033684015274048\n",
      "Epoch  1724: reducing learning rate of group 0 to 1.0000e-04.\n",
      "Iteration 1750 - Loss 0.9997305870056152\n",
      "Iteration 2000 - Loss 0.9309741258621216\n",
      "Iteration 2000 - Loss 0.9309741258621216\n",
      "Training complete!\n"
     ]
    }
   ],
   "source": [
    "_ = train(print_every=250, pos_enc=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Input: \n",
      "[[['C']\n",
      "  ['A']\n",
      "  ['C']\n",
      "  ['C']\n",
      "  ['C']\n",
      "  ['C']\n",
      "  ['C']\n",
      "  ['B']\n",
      "  ['B']\n",
      "  ['B']\n",
      "  ['B']]]\n",
      "\n",
      "Prediction: \n",
      "[[5]]\n",
      "\n",
      "Encoder-Decoder Attention: \n",
      "Output step 0 attended mainly to Input steps: [ 7  8  9 10]\n",
      "[0.072, 0.058, 0.072, 0.072, 0.072, 0.072, 0.072, 0.127, 0.127, 0.127, 0.127]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAB/CAYAAAAq7S8sAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAERxJREFUeJzt3X+QVeV9x/H354KAigL+mghS0ahNoSpTiGKqqbEajY2ilqjUGtBEamunY9pEo/nVZKwapzOajskwmkSqTUUT468mykisjUkwKlYQEn8sQor4K6IYEX8B3/5xnrt79mbv3sue3XvYu5/XzJk953nOeb7P8+zhfu8557JXEYGZmVkRlbI7YGZmg5+TiZmZFeZkYmZmhTmZmJlZYU4mZmZWmJOJmZkV5mRibUHSA5I+XXY/BoKkkHRA2f0YSJJWSjq67H5Y3zmZWGGS1kh6S9LG3HJt2f0qQtICSe9KeiMtKyRdIWlM2X3rD5KOTknq4prySal8eK5srqSf9WPsBZIuy5dFxJSIeKC/YljrOZlYfzkpIkbnlr8vu0PNyL9o9uCqiNgF2BM4B5gB/FzSzi3pXEENxjYHeBX4ZIu6Y23OycQGVPVdraR/lfSapNWSPpar303SDZKeT/V35OrOk9Qh6VVJd0kan6s7TtKTkl5PV0GqiXuupF+nNhdJ2jdXF5IukPQM8EyjMUTE2xHxCHAysDtZYmkmzhRJ96X+vyTp0lQ+UtI1aczPp/WRueM+J+mFVHduzbhGprn8v9TmfEk7prqjJT0n6WJJLwI31Pmd7AzMAi4ADpQ0PVf90/RzQ7rCPAKYDxyRtjdsQz/+SdLLaSznpLp5wFnARam9u1P5GknHNpqf3tq2cjmZWCscDjwF7AFcBXxHUvXF/yZgJ2AKsBdwNYCkY4ArgNOBvYHfAAtT3R7AD4EvpjZXAX9aDSZpJnApcBrZVcWDwM01fTol9Wtys4OIiDeA+4CjGsWRtAuwGLgXGA8cAPwkNfUFsqucqcChwGFpLEg6AfgscBxwIHBsTTeuBA5Kxx4ATAC+nKt/H7AbsC8wr85QTgM2At8HFpFdpVR9OP0cm64wlwDnA0vS9tht6MeYVP4p4JuSxkXEdcD3yK76RkfEST30r+789NZ2nbFaq0SEFy+FFmAN2YvThtxyXqqbC3Tk9t0JCLIXhL2BrcC4Htr8DtkLTnV7NPAeMIns1sxDuToBzwGfTtv3AJ/K1VeATcC+aTuAYxqMaQFwWQ/lVwL3NYoDzAb+t07bq4ATc9vHA2vS+neBK3N1B6X+HpDG+Sbw/lz9EcDqtH408C4wqsHYFgPXpPXZwG+BHdL2pBRveG7/ucDPaua7UT/eqmnjZWBGvblN59CxTcxPr217KW/xlYn1l1MiYmxuuT5X92J1JSI2pdXRwETg1Yh4rYf2xpNdjVSP2wisJ3s3Oh5Ym6uL/DbZi/k3JG1It2VeJXsBnJDbp3N/SZeq64MD8xuMc0Jqr1GciWQvij3pNra0Pj5Xt7amrmpPsmS8NBfz3lRe9duIeLte5yVNBD5CdnUAcCcwCviLesf0oJl+rI+IzbntTWS/82b0Nj9F27YB4mRiZVoL7CZpbA91z5O9WAOd9/l3B9YBL5C9WFfrlN9O7f5NTXLbMSJ+kdun889lR8Tl0fXBgfPrdVbSaLLbTg82EWctsH+dprqNDfiDVEbt2FJd1Stk78qn5OKNiYj8C2mjPwN+Ntm/+7vTc5VnyZJJ9VZXT8fXljXTj9406mNv82PbKScTK01EvEB2q+hbksZJ2kFS9Z79zcA5kqamh6+XA7+MiDXAj4Apkk5T9omlfyC7bVY1H7hE0hQASWMkfaKv/UwPhKcBdwCv0fVgu7c4/wXsLenCdPwukg7Pje2LkvZMz3++DPxHqrsVmCtpsqSdgK/k5msrcD1wtaS9UswJko7fhuHMAb5K9jyiuvwlcKKk3clueW2leyJ8CdhH0oh+6sdL1E+00Pv82HbKycT6y93q/v9Mbm/yuLPJnoU8SXbv+0KAiFgMfAm4jezd+vuBM1PdK8AnyJ5frCd7UP3zaoMRcTvwdWChpN8BK4DOT5Btg4skvZFi3AgsBT4UEW82ihPZw/rjgJPIbvM9Q3Z7CeAy4FFgOfAE8FgqIyLuAa4B7gc60s+8i1P5QynmYuAPmxmMpBlk7/i/GREv5pa7Upuz023IfyH7CPSGdMz9wErgRUmvFO0H2fOwyan9O3qorzs/tv1SdrvZzMys73xlYmZmhTmZmJlZYU4mZmZWmJOJmZkV5mRiZmaF9fZXRdvKBbqllI+t3TRqdRlhATj77f1Ki23t7/J3Lim7C9YCY0Y8q8Z7+crEzMz6gZOJmZkV5mRiZmaFOZmYmVlhTiZmZlaYk4mZmRXmZGJmZoU5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhfV7MpE0V9L4AsdPkvRXTex3iaQOSU9t43dgm5lZPxuIK5O5QJ+TCTAJ6DWZSJpM9n3gU4ATgG9JGlYgppmZFdBUMpH0j5JWpOXCdPWwIlf/WUn/LGkWMB34nqTHJe0oaY2kqyQ9IelhSQekYxak/attbEyrVwJHpeM/U6dLM4GFEfFORKwGOoDDtn34ZmbWHxomE0nTgHOAw4EZwHnAuJ72jYgfAI8CZ0XE1Ih4K1W9HhEHA9cC1zQI+XngwXT81XX2mQCszW0/l8pq+z5P0qOSHl3J4gZhzcysr5q5MjkSuD0i3oyIjcAPgaO2Mc7NuZ9HbOOxfRYR10XE9IiYPoVjWxXWzGzI6eszk7E1x45qsH/0sL652oakCjBiG+KvAybmtvdJZWZmVoJmksmDwCmSdpK0M3AqcA+wl6TdJY0EPp7b/w1gl5o2zsj9XJLW1wDT0vrJwA69HF/rLuBMSSMl7QccCDzcxFjMzGwANPza3oh4TNICul6svx0Rj0j6WipbBzyZO2QBMF/SW3Td0honaTnwDjA7lV0P3ClpGXAv8GYqXw5sSeULenpuEhErJd0K/IrsCueCiNjS5JjNzKyfKWJgvxpd0hpgekS8MqCBGvB3wJv1L38H/NDg74A3M7OWaXibq6iImNTXY9P/bP96TfHqiDi1UKfMzKxfDXgyKSIiFgGLyu6HmZn1zre5zMysMCcTMzMrzMnEzMwKczIxM7PCnEzMzKwwJxMzMysuIrw0WIB5Qy22xzw0Yg+1uB7zwC2+MmnOvCEY22MeGrGHWtwyY7f1mJ1MzMysMCcTMzMrzMmkOdcNwdge89CIPdTilhm7rcc84H+C3szM2p+vTMzMrDAnEzMzK8zJpBeS3idpoaRVkpZK+rGkg1oU+xRJIekDrYiXi1vmmEuJ7TG3NO4WSY9LWibpMUkfGuiYZcYtM3bL45b1n2i29wUQsAQ4P1d2KHBUi+LfAjwIfHUojLms2B5zy8e8Mbd+PPA/Ax2zzLhDaczb9ZdjlewjwHsRMb9aEBHLWhFY0mjgyNSHu4GvtCIuJY65xNgec+vi1toVeG0IxS0z9oDHdTKp74+BpSXFngncGxFPS1ovaVpEtKIvZY65rNgec2vtKOlxYBSwN3BMm8ctM3ZL4/qZyfZpNrAwrS9M22bt4K2ImBoRHwBOAG6UpDaOW2bslsZ1MqlvJTCt1UEl7Ub2DuLbktYAnwNOb9HJV8qYS47tMZckIpYAewB7DoW4ZcZuRVwnk/ruB0ZK6vwDaZIOkXTUAMedBdwUEftGxKSImAisBgY6LpQ35jJje8yti9tN+qTiMGD9UIhbZuxWxHUyqSOyj0CcChybPj65ErgCeHGAQ88Gbq8pu40W3OoqccylxfaYWztm0n38dC//FmBORGxp47hlxm5pXP85FTMzK8xXJmZmVpiTiZmZFeZkYmZmhTmZmJlZYU4mZmZWmJOJmZkV5mRiZmaFOZmYmVlhTiZmZlaYk4mZmRXmZGJmZoU5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhTmZmJlZYcPL7kCr7LrjlTEsxDCyr1IfhhgWFSppu0JWV0kLQCXovp3W1W0bKtG1rWoZ+bLcdtRsQzom21Z0tavcPhWE0veYdR2T6YqZtqN6PL+3T/WL5Cud7eTaVH7/rvqKcmXKHSMQUVMfVL+tXura7tyncztq9knblZrtXH213c7tSp020nolv13pKu/WRrd9fr+N7vtTE7eHNppsU93mI7KTrZk28+PoNS691lOp04/OON2PoWbs5Nus6Vu3uJXaftJtrNV2uo+/h31q4pL73ZMbT+c+ubjU9Ktz325tppi5fbrH7bk+94+uK2auLNuPzmMi93vprKvQbTu6HZMtkf/HX4HI1Uf1Hy9p/ffqVVMmooc2IjcfUQmoZF/KOGbEs9U9e+UrEzMzK8zJxMzMCnMyMTOzwpxMzMysMCcTMzMrzMnEzMwKczIxM7PCnEzMzKwwJxMzMyvMycTMzApzMjEzs8KcTMzMrDAnEzMzK8zJxMzMCnMyMTOzwpxMzMysMEVE2X3YLkiaFxHXld2P7ZnnqDHPUWOeo8YG4xz5yqTLvLI7MAh4jhrzHDXmOWps0M2Rk4mZmRXmZGJmZoU5mXQZVPcnS+I5asxz1JjnqLFBN0d+AG9mZoX5ysTMzApr22Qi6QRJT0nqkPT5HupHSrol1f9S0qRc3SGSlkhaKekJSaNS+bS03SHp3ySpdSPqX32dH0lnSXo8t2yVNDXVPZDarNbt1dpR9a8CczRC0g3pXFkm6ejcMW1zDkFTc/RhSY9J2ixpVk3dHEnPpGVOrnyonUc9zpGkqbnXoeWSzsjV7ZfOuY50Do5o1Xjqioi2W4BhwCpgf2AEsAyYXLPP3wHz0/qZwC1pfTiwHDg0be8ODEvrDwMzAAH3AB8re6ytnp+afQ4GVuW2HwCmlz2+sucIuAC4Ia3vBSwFKu10Dm3DHE0CDgFuBGblyncDnk0/x6X1cUP0PKo3RwcBB6b18cALwNi0fStwZlqfD/xt2WNt1yuTw4COiHg2It4FFgIza/aZCfx7Wv8B8OfpXeJHgeURsQwgItZHxBZJewO7RsRDkf0GbwROacVgBkCR+cmbnY5tR0XmaDJwP0BEvAxsAKa32TkETcxRRKyJiOXA1ppjjwfui4hXI+I14D7ghFZ0usX6PEcR8XREPJPWnwdeBvZM59gxZOccZOdg6edRuyaTCcDa3PZzqazHfSJiM/A62VXIQUBIWpQuPS/K7f9cgzYHiyLzk3cGcHNN2Q3p1sSXBvktnCJztAw4WdJwSfsB04CJtNc5BM3NUV+PHUrnUUOSDiO7sllFdo5tSOdcn9vsb8PL7sB2aDhwJPBBYBPwE0lLyV4oLJF0OLApIlbkis+KiHWSdgFuA84me/c91HwX+CPgUeA3wC+ALaX2aHDxeZSTrmhvAuZExNbtNbe265XJOrJ3glX7pLIe95E0HBgDrCfL8j+NiFciYhPwY+BP0v77NGhzsCgyP1VnUnNVEhHr0s83gP8ku8QfrPo8RxGxOSI+ExFTI2ImMBZ4mvY6h6C5OdrmY4fgeVSXpF2BHwFfiIiHUvF6YGw657a5zYHSrsnkEeDA9ImHEWQvfHfV7HMXUP0EySzg/nQfexFwsKSd0i/rz4BfRcQLwO8kzUiX3Z8E7mzFYAZAkflBUgU4ndzzknRLZ4+0vgPwcWAFg1ef5yidOzsDSDoO2BwR7XYOQXNzVM8i4KOSxkkaR/asctEQPY96lPa/HbgxIqrPR0j/Dv+b7JyD7Bws/zwq+xMAA7UAJ5K9G1xFltUBvgacnNZHAd8HOsg+YbN/7ti/BlaSncRX5cqnp7JVwLWk//Q5GJeC83M08FBNezuTfWppeZq7b5A+BTdYl77OEdmnc54Cfg0sBvZtx3OoyTn6INnV/ptk76hX5o49N81dB3DOED6Pepyj9Dr0HvB4bpma6vZP51xHOgdHlj1O/w94MzMrrF1vc5mZWQs5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhTmZmJlZYU4mZmZWmJOJmZkV9v8fL1PMtrpOaQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = test(pos_enc=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# With Positional encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 250 - Loss 1.8333065509796143\n",
      "Iteration 500 - Loss 1.487678050994873\n",
      "Iteration 750 - Loss 1.1412402391433716\n",
      "Iteration 1000 - Loss 0.48354998230934143\n",
      "Iteration 1250 - Loss 0.2923632562160492\n",
      "Iteration 1500 - Loss 1.1677156686782837\n",
      "Iteration 1750 - Loss 0.23710504174232483\n",
      "Iteration 2000 - Loss 0.18062952160835266\n",
      "Iteration 2000 - Loss 0.18062952160835266\n",
      "Training complete!\n"
     ]
    }
   ],
   "source": [
    "loss_hist = train(print_every=250, pos_enc=True, num_enc_layers=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Input: \n",
      "[[['B']\n",
      "  ['C']\n",
      "  ['A']\n",
      "  ['B']\n",
      "  ['C']\n",
      "  ['C']\n",
      "  ['C']\n",
      "  ['B']\n",
      "  ['B']\n",
      "  ['C']\n",
      "  ['C']]]\n",
      "\n",
      "Prediction: \n",
      "[[3]]\n",
      "\n",
      "Encoder-Decoder Attention: \n",
      "Output step 0 attended mainly to Input steps: [2]\n",
      "[0.03, 0.038, 0.64, 0.034, 0.038, 0.038, 0.038, 0.034, 0.034, 0.038, 0.038]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAB/CAYAAAAq7S8sAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEIVJREFUeJzt3XmQHOV5x/HvbyQkBBJCXGUkZBaMFEfyQSKFw7YoTIFR7HAaE2SXg7CNTBWUiyTGGELsgIkRlCuQKstRwIcKx0EhscEiDlAQCnMEAuJGhENCmwhxGUkcAnFIevJHv7PbO+zuzKpnplc7v09V13a/b/f7vO87x7PdPbujiMDMzKyIStkdMDOz7Z+TiZmZFeZkYmZmhTmZmJlZYU4mZmZWmJOJmZkV5mRiI4Kk2yV9rex+tIKkkHRA2f1oJUkrJB1edj9s2zmZWGGSuiVtkrQxt/yw7H4VIWmJpHclvZGWxyVdImli2X1rBkmHpyR1bk15VyofnSubL+muJsZeIunifFlEzIyI25sVw9rPycSa5ZiIGJ9bziq7Q43Iv2n247KImADsCZwGHALcLWnntnSuoDpjOxVYD/xZm7pjI5yTibVU9bdaST+QtEHSakl/nKvfTdLPJD2f6q/P1Z0uaaWk9ZKWSZqcqztK0pOSXktnQaqJ+xVJ/5PavFnSvrm6kHSmpGeAZ+qNISLejoj7gWOB3ckSSyNxZkq6JfX/JUnnp/Kxkq5IY34+rY/NHXeOpBdS3VdqxjU2zeX/pTYXSxqX6g6X9JykcyW9CPxsgMdkZ+Ak4ExgmqTZueo70s9X0xnmocBi4NC0/eoQ+vGXkl5OYzkt1S0AvgR8K7V3QyrvlnRkvfkZrG0rl5OJtcPBwFPAHsBlwE8kVd/8fw7sBMwE9gIuB5B0BHAJcDKwN/C/wNJUtwfwK+CC1OYq4JPVYJKOA84HTiQ7q7gTuKamT8enfs1odBAR8QZwCzCnXhxJE4BbgZuAycABwH+mpv6K7CznQODjwEFpLEiaC3wTOAqYBhxZ042FwPR07AHAFOA7ufoPALsB+wILBhjKicBG4F+Bm8nOUqoOSz93TWeY9wBnAPek7V2H0I+JqfyrwCJJkyLiSuAXZGd94yPimH76N+D8DNb2AGO1dokIL14KLUA32ZvTq7nl9FQ3H1iZ23cnIMjeEPYGtgKT+mnzJ2RvONXt8cB7QBfZpZl7c3UCngO+lrZvBL6aq68AbwH7pu0AjqgzpiXAxf2ULwRuqRcHmAc8NEDbq4DP5raPBrrT+k+Bhbm66am/B6Rxvgl8KFd/KLA6rR8OvAvsWGdstwJXpPV5wO+AHdJ2V4o3Orf/fOCumvmu149NNW28DBwy0Nym59CRDczPoG17KW/xmYk1y/ERsWtuuSpX92J1JSLeSqvjganA+ojY0E97k8nORqrHbQTWkf02OhlYk6uL/DbZm/nfS3o1XZZZT/YGOCW3T8/+ks5X7wcHFtcZ55TUXr04U8neFPvTZ2xpfXKubk1NXdWeZMn4gVzMm1J51e8i4u2BOi9pKvBpsrMDgF8DOwKfG+iYfjTSj3URsTm3/RbZY96IweanaNvWIk4mVqY1wG6Sdu2n7nmyN2ug5zr/7sBa4AWyN+tqnfLbqd2v1yS3cRHxX7l9ev5ddkR8P3o/OHDGQJ2VNJ7sstOdDcRZA+w/QFN9xgZ8MJVRO7ZUV/UK2W/lM3PxJkZE/o203r8B/zLZ6/6GdF/lWbJkUr3U1d/xtWWN9GMw9fo42PzYMOVkYqWJiBfILhX9SNIkSTtIql6zvwY4TdKB6ebr94H/johu4DfATEknKvvE0jfILptVLQbOkzQTQNJESV/Y1n6mG8KzgOuBDfTe2B4szr8De0s6Ox0/QdLBubFdIGnPdP/nO8A/pbprgfmSZkjaCfhubr62AlcBl0vaK8WcIunoIQznVOBCsvsR1eXzwGcl7U52yWsrfRPhS8A+ksY0qR8vMXCihcHnx4YpJxNrlhvU9+9MrmvwuC+T3Qt5kuza99kAEXEr8NfAL8l+W/8QcEqqewX4Atn9i3VkN6rvrjYYEdcBlwJLJb0OPA70fIJsCL4l6Y0U42rgAeATEfFmvTiR3aw/CjiG7DLfM2SXlwAuBpYDjwKPAQ+mMiLiRuAK4DZgZfqZd24qvzfFvBX4vUYGI+kQst/4F0XEi7llWWpzXroM+bdkH4F+NR1zG7ACeFHSK0X7QXY/bEZq//p+6gecHxu+lF1uNjMz23Y+MzEzs8KcTMzMrDAnEzMzK8zJxMzMCnMyMTOzwgb7r6Ijyi7jFpbysbVxMaqMsAA8/fo/lBK3a5evlxJ3jH83aqt32Vp2F9quE59jL719jurv5TMTMzNrAicTMzMrzMnEzMwKczIxM7PCnEzMzKwwJxMzMyvMycTMzApzMjEzs8KcTMzMrDAnEzMzK8zJxMzMCmt6MpE0X9LkAsd3SfpiA/udJ2mlpKeG+B3YZmbWZK04M5kPbHMyAbqAQZOJpBlk3wc+E5gL/EhSef9R0cyswzWUTCT9haTH03J2Ont4PFf/TUl/I+kkYDbwC0kPSxonqVvSZZIek3SfpAPSMUvS/tU2NqbVhcCcdPyfD9Cl44ClEfFORKwGVgIHDX34ZmbWDHWTiaRZwGnAwcAhwOnApP72jYh/A5YDX4qIAyNiU6p6LSI+CvwQuKJOyG8Dd6bjLx9gnynAmtz2c6mstu8LJC2XtPzdzffVCWtmZtuqkTOTTwHXRcSbEbER+BUwZ4hxrsn9PHSIx26ziLgyImZHxOwxo33iYmbWKtt6z2TXmmN3rLN/9LO+udqGpAowZgjx1wJTc9v7pDIzMytBI8nkTuB4STtJ2hk4AbgR2EvS7pLGAn+S2/8NYEJNG3+a+3lPWu8GZqX1Y4EdBjm+1jLgFEljJe0HTAN8HcvMrCR1v7Y3Ih6UtITeN+sfR8T9ki5KZWuBJ3OHLAEWS9pE7yWtSZIeBd4B5qWyq4BfS3oEuAl4M5U/CmxJ5Uv6u28SESskXQs8QXaGc2ZEbGlwzGZm1mSKaO1Xo0vqBmZHxCstDVSHvwO+ffwd8J3B3wHfGfwd8GZm1jZ1L3MVFRFd23ps+sv2S2uKV0fECYU6ZWZmTdXyZFJERNwM3Fx2P8zMbHC+zGVmZoU5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhTmZmJlZcRHhpc4CLOi02B5zZ8TutLgec+sWn5k0ZkEHxvaYOyN2p8UtM/aIHrOTiZmZFeZkYmZmhTmZNObKDoztMXdG7E6LW2bsET3mlv8LejMzG/l8ZmJmZoU5mZiZWWFOJoOQtEXSw5IekfSgpE+0MfYHJC2VtErSA5L+Q9L0NsU+XlJI+nA74qWYnTrXpcQuMW4pj7OfX22IXdYf0WwPC7Axt3408Ns2xRVwD3BGruzjwJw2xf8X4E7gQs/1yItd8pjLepz9/Gpx7GH95VjDzC7AhjbF+jTwXkQsrhZExCPtCCxpPPCp1IcbgO+2I26NjpjrEmOXOea8dj7OZcXtmOeXk8ngxkl6GNgR2Bs4ok1xPwI80KZYtY4DboqIpyWtkzQrItrRl06c67Jilznmsh5nP79azPdMBrcpIg6MiA8Dc4GrJansTrXYPGBpWl+attuhE+e6E5X1OPv51WJOJg2KiHuAPYA92xBuBTCrDXH6kLQb2W9sP5bUDZwDnNzuF10nzHXJscscc482P85lxe2Y55eTSYPSJ5tGAevaEO42YKyknn/OJuljkua0OO5JwM8jYt+I6IqIqcBqoNVx++iQuS4zdplj7tHmx7msuB3z/HIyGdy49HHCh8k+4XRqRGxpddDIPnZxAnBk+kjfCuAS4MUWh54HXFdT9kvac6mr0+a6tNhljpmSHuey4nbS88v/TsXMzArzmYmZmRXmZGJmZoU5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhTmZmJlZYU4mZmZWmJOJmZkV5mRiZmaFOZmYmVlhTiZmZlaYk4mZmRXmZGJmZoU5mZiZWWFOJmZmVtjosjvQLruMWxijQowi+zrzUYhRUaGStitkdZW0AFSCvttpXX22oRK926qWkS/LbUfNNqRjsm1Fb7vK7VNBKH2PWe8xmd6YaTuqx/O+fapf5l7paSfXpvL799ZXlCtT7hiBiJr6oPqN8VLvds8+PdtRs0/artRs5+qr7fZsVwZoI61X8tuV3vI+bfTZ5/1t9N2fmrj9tNFgm+ozH5E92RppMz+OQeMyaD2VAfrRE6fvMdSMnXybNX3rE7dS20/6jLXaTt/x97NPTVxyjz258fTsk4tLTb969u3TZoqZ26dv3P7rcy+63pi5smw/eo6J3OPSU1ehz3b0OSZbIv/ir0Dk6qP64iWtv69eNWUi+mkjcvMRlYBK9kWUE8c8W91zUD4zMTOzwpxMzMysMCcTMzMrzMnEzMwKczIxM7PCnEzMzKwwJxMzMyvMycTMzApzMjEzs8KcTMzMrDAnEzMzK8zJxMzMCnMyMTOzwpxMzMysMCcTMzMrzMnEzMwKU0SU3QdrI0kLIuLKsvuxvfM8No/nsjnKnkefmXSeBWV3YITwPDaP57I5Sp1HJxMzMyvMycTMzApzMuk8vjbdHJ7H5vFcNkep8+gb8GZmVpjPTMzMrDAnkxFC0lxJT0laKenb/dQfJulBSZslnVRTt0XSw2lZ1r5eD08NzOUZkh5L83WXpBm5uvPScU9JOrq9PR9e6s1jbr/PSwpJs9N2l6RNuefk4vb1evhpZB4lnSzpCUkrJP1zrvxUSc+k5dSWdjQivGznCzAKWAXsD4wBHgFm1OzTBXwMuBo4qaZuY9ljGC5Lg3O5S279WOCmtD4j7T8W2C+1M6rsMQ3XeUz7TQDuAO4FZqeyLuDxsscwHJYGn4/TgIeASWl7r/RzN+DZ9HNSWp/Uqr76zGRkOAhYGRHPRsS7wFLguPwOEdEdEY8CW8vo4Hakkbl8Pbe5M1C98XgcsDQi3omI1cDK1F4nqjuPyfeAS4G329m57Ugj83g6sCgiNgBExMup/GjglohYn+puAea2qqNOJiPDFGBNbvu5VNaoHSUtl3SvpOOb27XtTkNzKelMSauAy4BvDOXYDlF3LiT9ITA1In7Tz/H7SXpI0m8lzWlhP4e7Rp5T04Hpku5Or+G5Qzi2aUa3qmHbruwbEWsl7Q/cJumxiFhVdqeGs4hYBCyS9EXgAqC116NHGEkV4O+A+f1UvwB8MCLWSZoFXC9pZs0ZofUaTXap63BgH+AOSR9tdyd8ZjIyrAWm5rb3SWUNiYi16eezwO3AHzSzc9uZoc7lUqB6NlfocRhh6s3FBOAjwO2SuoFDgGWSZqfLhOsAIuIBsnsG09vS6+GnkefUc8CyiHgvXV59miy5tPX56GQyMtwPTJO0n6QxwClAQ5/KkjRJ0ti0vgfwSeCJlvV0+Ks7l5Km5TY/BzyT1pcBp0gaK2k/shf0fW3o83A06DxGxGsRsUdEdEVEF9kN+GMjYrmkPSWNAkhny9PIbh53okZe29eTnZVUX8PTyebrZuAz6TU+CfhMKmsJX+YaASJis6SzyJ4oo4CfRsQKSRcByyNimaQ/Aq4j+1THMZIujIiZwO8D/yhpK9kvFwsjomOTSSNzCZwl6UjgPWAD6RJX2u9asmS8GTgzIraUMpCSNTiPAzkMuEjSe2QfGDkjIta3vtfDT4PzWE0aTwBbgHOqZ3aSvkeWkAAuauU8+i/gzcysMF/mMjOzwpxMzMysMCcTMzMrzMnEzMwKczIxM7PCnEzMzKwwJxMzMyvMycTMzAr7f0ARxFKb6iYhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "L2-Norm of Input Positional Encoding:\n",
      "[4.017, 0.431, 0.43, 0.432, 0.434, 0.434, 0.429, 0.433, 0.433, 0.426, 0.432]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAB/CAYAAAAn819rAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEi5JREFUeJzt3XnQXFWZx/Hv700IJISAGHCABAKjqGDJKlgCgwoOyBKwJqXBCSKF4qjgguWCyLAUoCMOo2i5YKRYHAjI4iCCjsg+IzsEiYgDIQkBJBIJkIBAwjN/nNPve7vT75r73hu6f5+qrtzl3POcc273fe7Sb1oRgZmZWRl66m6AmZl1DicVMzMrjZOKmZmVxknFzMxK46RiZmalcVIxM7PSOKlYpSRtKWm5pDEDlFkuaZtRbse7JS0ezRhrStICSfvm6a9Kml13m8wG46RiA8oHthfzgf4pSedJmjjS+iJiUURMjIhVuf4bJX2spczEiJi/pm1fE5JC0orc78brS3W1JyLOiIiPDV6yPHlfn9Zm+aaSLpb0hKRnJf2PpN0HqSsk/V5ST2HZaZLOG4WmW42cVGwoDo6IicDOwK7A12puT1V2yAmu8fpm3Q1aS0wE7gR2ATYGzgd+OYSTjc2BmWsaXNLYNa3DRo+Tig1ZRDwOXAu8DUDS5pKukvRXSQ9L+nijrKTdJN0l6bl8hXNWXj4tn7WOlXQ6sBfwvXwl8L1cJiS9MU9vKOkCSX+RtFDS1xpnu5I+KulWSd+S9IykRyW9v9CGIyU9KOl5SfMlfaKMcZB0sqRLc7uelzRP0q6F9VMlXZHbvLTQr57c/oWSluTtNyxsd3het1TSCW1i/rRlDI+QtEjS08XyksZLOj+PyYOSvlS81Sfpy5Iez21/SNI+w+l/RMyPiLMi4smIWBUR5wDjgDcPsuk3gVP6SwqSpuexXJavYN9aWLcgt/t+YEV+/yyQ9EVJ9+eryp9IeoOka3PfrpP0uuH0zdack4oNmaSpwAHAvXnRHGAx6Qx0BnCGpPfmdd8BvhMRk4C/By5trS8iTgBuAY7JVwLHtAn7XWBDYBtgb+AjwJGF9bsDDwGTSQetn0hSXrcEOAiYlLf5D0k7j6Dr7Uwn9X8j4CqgkTjGAFcDC4FpwBa5HMBH8+s9uT8TC9ttB/wAOJw0nq8HpgzShj1JB/J9gH8tHIRPyrG3Ad4HzGpsIOnNwDHAOyJiA2A/YMHwut5M0o6kpPLwIEWvAJ4jjUFrHdsCFwOfAzYBrgF+IWlcodhhwIHARhGxMi/7J1IftwUOJp30fDXX0QN8ZkSdshFzUrGh+LmkZcCtwE2k5DEV2AP4ckT8LSLuA2aTDvoArwBvlDQ5IpZHxG3DDZoP0DOB4yPi+YhYAPw76cDbsDAifpyf0ZwPbAa8ASAifhkRj0RyE/DfpCujobonnzU3XvsV1t0aEdfkuBcCO+Tlu5GSwhcjYkUem1vzun8Gzspn+suB44GZ+cx9BnB1RNwcES8BJwKvDtK+UyLixYiYC8wttOGDwBkR8UxELAbOLmyzClgX2E7SOhGxICIeGcaYNJE0Kff/lIh4dpDiQerXiS3JAuBDwC8j4jcR8QrwLWA88K5CmbMj4rGIeLGw7LsR8VS+ir4FuD0i7o2IvwFXAjuNtG82Mk4qNhSHRsRGEbFVRHwqf6g3B/4aEc8Xyi0knZkDHEU6e/yjpDslHTSCuJOBdXK97WIA/LkxEREv5MmJAJLeL+m2fHtuGekqa/Iw4u+c+914/bpdXOAFYL2cHKaSEt1KVrd5m76MJSXBzYHHCn1ZASwdpH2tbWg802iqq6Xeh0lXAycDSyTNkbT5IHHakjQe+AVwW0R8vbB8nvq+3NCUxCPiGtLVbeutyKaxiYhXc7uL+/oxVvdUYfrFNvMj/lKJjYyTio3UE8DGkjYoLNsSeBwgIv4vIg4DNgX+DbhM0vpt6hnov8l+mnTFs1W7GAORtC5wOemM9w0RsRHplooG3HDNPQZs2c9zgydYvS8rSQfCJ0kJCQBJE0i3wEbiSZpvnU0troyIiyJiz9yWIO2fYcnj+3PaJIiI2L7w5YZb2mx+AukW1YTCsqaxybcwp9K8r/1fqr8GOKnYiETEY8D/Al+XtJ6kt5OuThoPk2dJ2iSfcS7Lm7W7nfMU6d5/uxirSM9iTpe0gaStgOMaMQYxjnSb5y/AyvwA/x+H3MGRu4N0UP+GpPXz2OyR110MfF7S1krflDoDuCRf1VwGHCRpz3xr6FRG/vm8FDhe0uskbUF6hgKkZyqS3puTwt9IZ/MD3WYbk/vQeI2TtE5u74vAEXkfD1lE3Ag8ABzR0uYDJe2T6/8C8BLpPWavIU4qtiYOIz0QfoJ0//qkiLgur9sfmCdpOemh/cyWe+EN3wFm5G8qnd1m/bHACmA+6ZnORcC5gzUs35b7DOlg9QzwYdID9eGYq+a/U/n2EOKuIj0wfiOwiHQm/6G8+lzS84ebgUdJB/Vj83bzgE+T+vdkbvNI/zjz1Lzto8B1pATwUl63LvAN0lXgn0lXkscPUNdXSMmj8bqe9JzjIFKSXtbfra5BfI30dWQAIuIh0hcKvpvbdjDpq+wvD6NOWwvIP9Jl1tkkfZKU1Peuuy3W+XylYtZhJG0maQ+lv4t5M+lW0pV1t8u6g/8y1azzjAN+BGxNep41B/h+rS2yruHbX2ZmVhrf/jIzs9I4qZiZWWm64pnKsy9vU8s9vhkTTqwjrJlZKX6z8shh/7Gwr1TMzKw0TipmZlYaJxUzMyuNk4qZmZXGScXMzErjpGJmZqVxUjEzs9I4qZiZWWmcVMzMrDROKmZmVppBk0r+5b7WZcdJ+oOk+yX9Nv/Ma7ttT5b0gqRNB6rPzMw6w0ivVO4Fdo2It5N+qvSbA5R9mvQjQcOmxFdTZmavESM6YEfEDRHxQp69DZgyQPFzgQ9J2rh1Rb7ieSC/PpeXTZP0kKQLgAeAqfn3r8+UNE/SdZJ2k3SjpPmSpo+kD2ZmVr4yrgKOAq4dYP1yUmL5bHGhpF2AI4HdgXcCH5e0U179JuD7EbF9RCwE1geuj4jtgeeB04D3AR8ATi2hD2ZmVoI1SiqSZgG7AmcOUvRs4AhJGxSW7QlcGRErImI5cAWwV163MCJuK5R9GfhVnv49cFNEvJKnp/XTtqMl3SXprvNmPzecbpmZ2QiN+PdUJO0LnADsHREv5WWnAwcCRMSOjbIRsUzSRcCnh1j9ipb5V6Lvd49fBV7K9b4qqW0fIuIc4Byo7/dUzMy6zYiuVPJtqh8B0yNiSWN5RJwQETsWE0rBWcAn6EtktwCHSpogaX3SraxbRtIeMzNbOwzlSmWCpMWF+bOAA4CJwM8kASyKiAEfmEfE05KuBD6f5++RdB5wRy4yOyLulTRtWD0wM7O1hvruKnUu/5ywmdnw+eeEzcysVk4qZmZWGicVMzMrjZOKmZmVxknFzMxK46RiZmalcVIxM7PSOKmYmVlpnFTMzKw0TipmZlaeiPBrgBdwdLfF7ra47nN3xHWfq3n5SmVwR3dh7G6LW2ds97k7YndNn51UzMysNE4qZmZWGieVwZ3ThbG7LW6dsd3n7ojdNX3uit9TMTOzavhKxczMSuOkYmZmpXFS6YekVZLukzRX0j2S3lVh7L+TNEfSI5LulnSNpG0rin2opJD0liri5ZhdN9Y17+O6+lzLfu7G91etsev6g5y1/QUsL0zvB9xUUVwBvwP+pbBsB2CviuJfAtwCnOKx7qy4a0HsuvZzV72/6o49dk2TUpeYBDxTUaz3AK9ExA8bCyJibhWBJU0E9sxt+AVwUhVxW3TDWNe2j2uOXVTlfq4rblfuZyeV/o2XdB+wHrAZ8N6K4r4NuLuiWK0OAX4VEX+StFTSLhFRRVu6bazr3Md1xq5rP3fb+6vW2H6m0r8XI2LHiHgLsD9wgSTV3ahRdhgwJ0/PyfNV6Max7kZ17We/vyrkpDIEEfE7YDKwSQXh5gG7VBCniaSNSWdwsyUtAL4IfLDqD183jHWNceuO3avi/VxX3K7cz04qQ5C/CTUGWFpBuOuBdSX1/idwkt4uaa9RjjsDuDAitoqIaRExFXgUGO24TbpkrOuKW3fsXhXv57riduV+dlLp3/j8NcT7SN+IOiIiVo120Ehf0/gAsG/+KuA84OvAn0c59GHAlS3LLqeaW2BdNdY17uNaY1PTfq4rbrfuZ/83LWZmVhpfqZiZWWmcVMzMrDROKmZmVhonFTMzK42TipmZlcZJxczMSuOkYmZmpXFSMTOz0jipmJlZaZxUzMysNE4qZmZWGicVMzMrjZOKmZmVxknFzMxK46RiZmalcVIxM7PSjK27AVWYNP4bMSbEGNILYEz00IPoaczn6cZ8T9A737sMoaZ56Im++bSOlvK5nmiUadRLYRuh/FtpjXUqxsjrm7dpLQOKvnnRrnzfstXKqG9Z03o16kjTzcuidxlAjwIp1VWc7y3fOx8tZQL1tMy3rl+tfLsy0RunpzjfQ9P6Yvm+MqvX0ainMT6NuMX17bYZrE41jUdAzwB1tIxP77o25Yv9ZYB20dNPO9r0j2KZntXjouYxborb0zIvmvrau64pZpsyLXEp7Htaxq+pfGHMaIlDU500zTemUz00LWvU0Vt/S1/7PlCNGPRuEz201EHjINI7H00x0yuK9fZAFD64qU6lf9uuV2GZerdprSMK/Y+egJ70w5gbjpvfaM2Q+UrFzMxK46RiZmalcVIxM7PSOKmYmVlpnFTMzKw0TipmZlYaJxUzMyuNk4qZmZXGScXMzErjpGJmZqVxUjEzs9I4qZiZWWmcVMzMrDROKmZmVhonFTMzK42TipmZlUYRUXcbRp2koyPinLrbsTbwWCQehz4eiz4ei2RNxqFbrlSOrrsBaxGPReJx6OOx6OOxSEY8Dt2SVMzMrAJOKmZmVppuSSpdf4+0wGOReBz6eCz6eCySEY9DVzyoNzOzanTLlYqZmVWgo5KKpP0lPSTpYUlfabN+XUmX5PW3S5pWfStH3xDGYUtJN0i6V9L9kg6oo52jTdK5kpZIemCAMu+WdJ+keZJuqrJ9VZI0Ne/zP+S+fnaAsu+QtFLSjCrbWAVJ60m6Q9LcPA6ntClzXB6n+yX9VtJWdbS1KpLG5GPB1W3WDf+YGREd8QLGAI8A2wDjgLnAdi1lPgX8ME/PBC6pu901jcM5wCfz9HbAgrrbPUpj8Q/AzsAD/azfCPgDsGWe37TuNo/iWGwG7JynNwD+1Pq+KLx/rgeuAWbU3e5RGAcBE/P0OsDtwDtbyrwHmJCnP9mJx4mW/h4HXARc3WbdsI+ZnXSlshvwcETMj4iXgTnAIS1lDgHOz9OXAftIUoVtrMJQxiGASXl6Q+CJCttXmYi4GfjrAEU+DFwREYty+SWVNKwGEfFkRNyTp58HHgS2aFP0WOByoCPHIpLleXad/IqWMjdExAt59jZgSoVNrJSkKcCBwOx+igz7mNlJSWUL4LHC/GJW/9D0lomIlcCzwOsraV11hjIOJwOzJC0mnZEeW03T1jrbAq+TdKOkuyV9pO4GVSHfwtiJdJZeXL4F8AHgB9W3qjr5ds99pMT5m4i4fYDiRwHXVtOyWnwb+BLwaj/rh33M7KSkYkN3GHBeREwBDgAulNSN74WxwC6kM7X9gBMlbVtvk0aXpImkK5HPRcRzLau/DXw5Ivo7wHSEiFgVETuSrkB2k/S2duUkzQJ2Bc6ssn1VkXQQsCQi7i6z3rFlVlazx4GphfkpeVm7MosljSXd+llaTfMqM5RxOArYHyAifidpPWAyHXrLYwCLgaURsQJYIelmYAfS84aOI2kdUkL5z4i4ok2RXYE5+e7GZOAASSsj4ucVNrMyEbFM0g2kz0LTlzkk7QucAOwdES/V0b4K7AFMz1/UWQ+YJOmnETGrUGbYx8xOOju9E3iTpK0ljSM9VLqqpcxVwBF5egZwfeQnUB1kKOOwCNgHQNJbSW+ov1TayrXDfwF7ShoraQKwO+lZQ8fJ98F/AjwYEWe1KxMRW0fEtIiYRrp//qlOSyiSNpG0UZ4eD7wP+GNLmZ2AHwHTO/w52/ERMSXv75mk4+GslmLDPmZ2zJVKRKyUdAzwa9I3WM6NiHmSTgXuioirSB+qCyU9THqAO7O+Fo+OIY7DF4AfS/o86SHlRzswuSLpYuDdwOT8/Ogk0oNZIuKHEfGgpF8B95PuKc+OiH6/fvwatwdwOPD7/DwB4KvAlpDGo66GVWwz4HxJY0gn1ZdGxNUtn48zgYnAz/JV26KImF5biyu2psdM/0W9mZmVppNuf5mZWc2cVMzMrDROKmZmVhonFTMzK42TipmZlcZJxczMSuOkYmZmpXFSMTOz0vw/qWS/uz9CedQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    " samples, labels, predictions, attention, input_pos_enc = test(pos_enc=True, num_enc_layers=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# With positional encodings and more encoder layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 250 - Loss 1.2629730701446533\n",
      "Iteration 500 - Loss 0.46846190094947815\n",
      "Iteration 750 - Loss 0.21491453051567078\n",
      "Iteration 1000 - Loss 0.04504004120826721\n",
      "Iteration 1250 - Loss 0.019231684505939484\n",
      "Iteration 1250 - Loss 0.019231684505939484\n",
      "Training complete!\n"
     ]
    }
   ],
   "source": [
    "loss_hist = train(print_every=250, pos_enc=True,\n",
    "                  num_enc_layers=6, steps=1250)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Input: \n",
      "[[['B']\n",
      "  ['B']\n",
      "  ['B']\n",
      "  ['A']\n",
      "  ['C']\n",
      "  ['A']\n",
      "  ['C']\n",
      "  ['A']\n",
      "  ['B']\n",
      "  ['B']\n",
      "  ['B']]]\n",
      "\n",
      "Prediction: \n",
      "[[5]]\n",
      "\n",
      "Encoder-Decoder Attention: \n",
      "Output step 0 attended mainly to Input steps: [6]\n",
      "[0.085, 0.083, 0.083, 0.096, 0.106, 0.096, 0.106, 0.096, 0.083, 0.083, 0.083]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAB/CAYAAAAq7S8sAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEgVJREFUeJzt3XmUXGWZx/HvrxMSsiAJCEoSTIPAeMioHIkYVDyRRRCFAOIM6AgBIeMZ5szBFVFHR0UFxtEMR50MLkTQIS7sKDAgwxAVlUXAIAECyci+hDWAsj3zx32r+3ZR3VXdb9+qpuv3Oeeevvd9732f96nuqqfuvdXdigjMzMxy9HR6AmZm9tLnYmJmZtlcTMzMLJuLiZmZZXMxMTOzbC4mZmaWzcXExgVJV0g6qtPzqIKkkLRdp+dRJUk3SVrY6XnYyLmYWDZJ6yQ9LWlDaflGp+eVQ9JySc9IeiItqyR9RdKmnZ7baJC0MBWp4+rae1P7xFLbYkm/HMXYyyWdUG6LiHkRccVoxbD2czGx0bJfREwvLf/Y6Qm1ovyi2cDJEbEJsAVwBLAA+JWkaW2ZXKYmuR0OPAwc1qbp2DjnYmKVqr2rlfRVSY9IWivpnaX+zSSdJume1H9uqe9oSWskPSzpfEmzSn17SVot6bF0FqS6uEdKujmNeYmkuaW+kHSMpNuA25rlEBF/joirgf2BzSkKSytx5km6NM3/fkmfSu2TJS1NOd+T1ieXjvu4pHtT35F1eU1Oj+Wf0pjLJE1JfQsl3SXpOEn3AacN8j2ZBhwMHANsL2l+qfvK9PXRdIa5K7AM2DVtPzqMeXxU0gMplyNS3xLg/cAn0ngXpPZ1kvZs9vgMNbZ1louJtcObgFuAlwMnA9+VVHvxPwOYCswDtgS+DiBpd+ArwN8AWwH/B6xIfS8HzgY+k8a8HXhLLZikRcCngIMozipWAmfWzemANK8dW00iIp4ALgV2axZH0ibAZcDFwCxgO+AXaahPU5zl7AS8Htgl5YKkfYCPAXsB2wN71k3jRGCHdOx2wGzgs6X+VwKbAXOBJYOkchCwAfgJcAnFWUrN29LXGekM8yrgQ8BVaXvGMOaxaWr/IPBNSTMj4lTghxRnfdMjYr8G8xv08Rlq7EFytXaJCC9eshZgHcWL06Ol5ejUtxhYU9p3KhAULwhbAS8AMxuM+V2KF5za9nTgWaCX4tLMb0p9Au4CjkrbFwEfLPX3AE8Bc9N2ALs3yWk5cEKD9hOBS5vFAQ4Ffj/I2LcD+5a29wbWpfXvASeW+nZI890u5fkk8OpS/67A2rS+EHgG2LhJbpcBS9P6ocCDwEZpuzfFm1jafzHwy7rHu9k8nq4b4wFgwWCPbfoZ2rOFx2fIsb10bvGZiY2WAyJiRmn5dqnvvtpKRDyVVqcDWwMPR8QjDcabRXE2UjtuA7Ce4t3oLODOUl+UtylezP9d0qPpsszDFC+As0v79O0v6VPq/+DAsiZ5zk7jNYuzNcWLYiMDckvrs0p9d9b11WxBUYyvLcW8OLXXPBgRfx5s8pK2Bt5OcXYAcB6wMfCuwY5poJV5rI+I50rbT1F8z1sx1OOTO7ZVxMXEOulOYDNJMxr03UPxYg30XeffHLgbuJfixbrWp/J2Gvfv64rblIj4dWmfvj+XHRFfjv4PDnxosMlKmk5x2WllC3HuBLYdZKgBuQGvSm3U55b6ah6ieFc+rxRv04gov5A2+zPgH6B43l+Q7qvcQVFMape6Gh1f39bKPIbSbI5DPT42RrmYWMdExL0Ul4q+JWmmpI0k1a7ZnwkcIWmndPP1y8BvI2Id8DNgnqSDVHxi6Z8oLpvVLAOOlzQPQNKmkt470nmmG8I7A+cCj9B/Y3uoOBcCW0k6Nh2/iaQ3lXL7jKQt0v2fzwI/SH0/BhZL2lHSVOBzpcfrBeDbwNclbZlizpa09zDSORz4PMX9iNryHmBfSZtTXPJ6gYGF8H5gjqRJozSP+xm80MLQj4+NUS4mNlou0MDfMzmnxeM+QHEvZDXFte9jASLiMuCfgbMo3q2/Gjgk9T0EvJfi/sV6ihvVv6oNGBHnACcBKyQ9DqwC+j5BNgyfkPREinE6cC3w5oh4slmcKG7W7wXsR3GZ7zaKy0sAJwDXADcCfwCuS21ExEXAUuByYE36WnZcav9NinkZ8FetJCNpAcU7/m9GxH2l5fw05qHpMuSXKD4C/Wg65nLgJuA+SQ/lzoPiftiOafxzG/QP+vjY2KXicrOZmdnI+czEzMyyuZiYmVk2FxMzM8vmYmJmZtlcTMzMLNtQf1V0XNllo//syMfWVk9s9Mvd49vVi+7qSNxXnv7zjsQFuP/wkXzyON8rvn9RR+IC3HfYvh2J+8bz5nQkbrd6/OlPqvlePjMxM7NR4GJiZmbZXEzMzCybi4mZmWVzMTEzs2wuJmZmls3FxMzMsrmYmJlZNhcTMzPL5mJiZmbZXEzMzCzbqBcTSYslzco4vlfS+1rY73hJayTdMsz/gW1mZqOsijOTxcCIiwnQCwxZTCTtSPH/wOcB+wDfkjQhI6aZmWVoqZhI+oikVWk5Np09rCr1f0zSv0g6GJgP/FDS9ZKmSFon6WRJf5D0O0nbpWOWp/1rY2xIqycCu6XjPzzIlBYBKyLiLxGxFlgD7DL89M3MbDQ0LSaSdgaOAN4ELACOBmY22jcifgpcA7w/InaKiKdT12MR8VrgG8DSJiE/CaxMx399kH1mA3eWtu9KbfVzXyLpGknXPPDCyiZhzcxspFo5M3krcE5EPBkRG4Czgd2GGefM0tddh3nsiEXEqRExPyLmb9kz3CmbmVmrRnrPZEbdsRs32T8arD9XG0NSDzBpGPHvBrYubc9JbWZm1gGtFJOVwAGSpkqaBhwIXARsKWlzSZOBd5f2fwLYpG6Mvy19vSqtrwN2Tuv7AxsNcXy984FDJE2WtA2wPfC7FnIxM7MKNP23vRFxnaTl9L9Yfycirpb0hdR2N7C6dMhyYJmkp+m/pDVT0o3AX4BDU9u3gfMk3QBcDDyZ2m8Enk/tyxvdN4mImyT9GPgjxRnOMRHxfIs5m5nZKGvpf8BHxNeAr9W1nQKc0mDfs4CzatuSAP41Io6r2+9+ihv6Ncel9meB3VuY05eAL7UyfzMzq5Z/A97MzLK1dGaSIyJ6R3ps+s32k+qa10bEgVmTMjOzUVV5MckREZcAl3R6HmZmNjRf5jIzs2wuJmZmls3FxMzMsrmYmJlZNhcTMzPL5mJiZmb5IsJLkwVY0m2xnXN3xO62uM65usVnJq1Z0oWxnXN3xO62uJ2MPa5zdjExM7NsLiZmZpbNxaQ1p3ZhbOfcHbG7LW4nY4/rnJVuzpiZmY2Yz0zMzCybi4mZmWVzMRmCpOclXS/pBknXSXrzeI47BmIfICkkvaZdMVPcV0paIel2SddK+rmkHdoU2zm3J2bXPafaHrdTv0TzUliADaX1vYH/Hc9xx0DsHwErgc+3MaaAq4APldpeD+zmnMdVzl33nGp3XJ+ZtO5lwCNdFLetsSVNB94KfBA4pB0xk7cDz0bEslpDRNwQESurDuyc25pzWVc8p9odd0z/p8UxYIqk64GNga2A3cd53E7GXgRcHBG3SlovaeeIuLYNcf8aaEecRpxz+3LuxudUW+P6zGRoT0fEThHxGmAf4HRJGsdxOxn7UGBFWl+Rtsc759y+nLvxOdXWuP49kyFI2hAR00vb9wOvjYgHxmPcTsWWtBlwF/AgEMCE9HVuVPwDKmkP4HMR8bYq4zSI65zbm3NXPac6EddnJi1KnzyZAKzvhrhtjn0wcEZEzI2I3ojYGlgL7FZxXIDLgcmS+v4QnqTXSao6tnNub859uuQ51fa4vmcytNo1Ryg+AXN4RDw/juN2KvahwEl1bWel9iurDBwRIelAYKmk44A/A+uAY6uMi3OuaUvOdN9zqu1xfZnLzMyy+TKXmZllczExM7NsLiZmZpbNxcTMzLK5mJiZWTYXEzMzy+ZiYmZm2VxMzMwsm4uJmZllczExM7NsLiZmZpbNxcTMzLK5mJiZWTYXEzMzy+ZiYmZm2VxMzMwsW9f8p8WXTTkxJoSYgACYgJgQPfSk7R6Kvp60APQEA7fTugZsQ0/0b6vWRrmttB1125COKbYV/eOqtE8PQun/mPUfU+iPmbajdjwv2kdQGnPgtlTev7+/R6U2lY4RiKjrD1TbVv923z5921G3T9ruqdsu9dfG7dvuGWSMtN5T3u7pbx8wxoB9XjzGwP2pi9tgjBbH1IDHI4oftlbGLOcxZFyG7KdnkHn0xRl4DHW5Ux6zbm4D4vbUz5MBudbGGZh/g33q4lL63lPKp2+fUlzq5tW374AxU8zSPgPjNu4vPen6Y5baiv3oOyZK35e+vh4GbMeAY4olyk/+HohSf9SevKT1F/Wrrk1EgzGi9HhET0BP8U8ZN510R23PIfnMxMzMsrmYmJlZNhcTMzPL5mJiZmbZXEzMzCybi4mZmWVzMTEzs2wuJmZmls3FxMzMsrmYmJlZNhcTMzPL5mJiZmbZXEzMzCybi4mZmWVzMTEzs2wuJmZmlk0R0ek5jEmSlkTEqZ2eRzt0U67QXfk61/FrrOXrM5PBLen0BNqom3KF7srXuY5fYypfFxMzM8vmYmJmZtlcTAY3Zq5FtkE35Qrdla9zHb/GVL6+AW9mZtl8ZmJmZtm6ophI2kfSLZLWSPpkg/7Jkn6U+n8rqTe1byTp+5L+IOlmScfXHTdB0u8lXdieTFpTRb6SPizpJkmrJJ0paeP2ZTS4jFwnSTot5XqDpIWpfaqkn0lanfI9sa0JDWG0cy31nSrp1pTze9qWUBMt5Ps2SddJek7SwXV9h0u6LS2Hl9p3To/DGkmnSFI7cmmmilxL/edLWlXl/AGIiHG9ABOA24FtgUnADcCOdfv8A7AsrR8C/Citvw9YkdanAuuA3tJxHwH+C7iw03lWmS8wG1gLTEl9PwYWv8RzPQY4La1vCVxL8eZqKvD21D4JWAm8czzmmrY/D5yQ1nuAl3c612Hk2wu8DjgdOLjUvhlwR/o6M63PTH2/AxYAAi56CX1vh51r6j+I4jVqVdV5dMOZyS7Amoi4IyKeAVYAi+r2WQR8P63/FNgjvWMJYJqkicAU4BngcQBJc4B3Ad+pPoVhqSRfYCIwJfVNBe6pNo2W5OS6I3A5QEQ8ADwKzI+IpyLif1L7M8B1wJzKM2lu1HNN+x0JfCX1vRARD1WaReua5hsR6yLiRuCFumP3Bi6NiIcj4hHgUmAfSVsBL4uI30TxSns6cEDlmTQ36rkCSJpO8Yb3hKoTgO64zDUbuLO0fVdqa7hPRDwHPAZsTvGEfBK4F/gT8NWIeDgdsxT4BC/+5nbaqOcbEXcDX01t9wKPRcR/V5lEi3JyvQHYX9JESdsAOwNblw+UNAPYD/hFJbMfnlHPNeUH8MV0CeUnkl5RZRLD0Eq+wz12dlofyZhVqiJXgC8C/wY8lTvBVnRDMcmxC/A8MAvYBviopG0lvRt4ICKu7ejsRt9g+c6keKe0TeqbJunvOjfNUfE9iifeNRRvDH5NkTsA6QzsTOCUiLijIzMcPYPlOpHirOvXEfEG4CqKNw32EidpJ+DVEXFOu2J2QzG5m4HvOOektob7pBeRTYH1FPcQLo6IZ9PlgV9RXB54C8U7vXUUp6S7S/pBlUkMQxX57gmsjYgHI+JZ4GzgzZVm0ZoR5xoRz0XEhyNip4hYBMwAbi0ddypwW0QsrWz2w1NFrusp3rWenY7/CfCG6lIYllbyHe6xdzPwkuVwxqxSFbnuCsxPr1G/BHaQdEX2TIfQDcXkamB7SdtImkRxY/L8un3OB2qfgjgYuDxdU/0TsDuApGkUN+5WR8TxETEnInrTeJdHxFh5pz7q+ab2BemTTgL2AG6uPJPmRpxrymUagKS9gOci4o9p+wSKF+Jj25FEi0Y91/Q9vwBYmI7ZA/hjxXm0qpV8B3MJ8A5JM9NZ9TuASyLiXuBxSQvSz/FhwHlVTH6Yqsj1PyJiVnqNeitwa0QsrGDu/aq+wz8WFmBfinditwOfTm1fAPZP6xtTvCtbQ/Fpj21T+/TUfhPFk+zjDcZeyBj6NFdV+VJ86mc1sAo4A5jc6Twzc+0FbqEoipcBc1P7HIoPItwMXJ+WozqdZxW5pr65wJXAjRT3hl7V6TyHke8bKS7fPUlxlnVT6dgj0+OwBjii1D4//QzfDnyD9IvbnV6qyLXU30sbPs3l34A3M7Ns3XCZy8zMKuZiYmZm2VxMzMwsm4uJmZllczExM7NsLiZmZpbNxcTMzLK5mJiZWbb/B1kp4SwRzN8GAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "L2-Norm of Input Positional Encoding:\n",
      "[1.805, 0.237, 0.241, 0.259, 0.242, 0.244, 0.234, 0.24, 0.245, 0.25, 0.239]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAB/CAYAAADFL0QSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEURJREFUeJzt3XuQnFWdxvHvMwmBQLgacUUCkVVUZAGBBVeDqOh6Q9CSWoOCYKG4KrpeyguiclHxzipariJSoFsSWAVFAd1lEQirUVEuEllcCAkEkAACknBNePaP9/Skp9OZ6Zk30z0z/XyquvJeznt+55zp7t/7nrc7LdtERESM1UCvGxAREZNbEklERNSSRBIREbUkkURERC1JJBERUUsSSURE1JJEEl0laQdJKyVNG6bMSkk7jXM7Xixp+XjGqEvSUkkvK8sfk3R6r9sU0U4SSQyrvJk9XN7c75J0pqRZY63P9q22Z9leU+q/TNLbWsrMsr2kbtvrkGRJq0q/G48P96o9tk+2/baRS2445W/96Tbbt5V0tqQ7JD0g6X8k7TtCXZb0B0kDTds+LenMcWh6dFkSSXTitbZnAXsCewMf73F7umX3ktQajy/0ukETxCzgt8BewDbAWcCFHZxgbAfMrxtc0vS6dcSGlUQSHbN9O3AxsCuApO0kXSDpL5JukvT2RllJ+0i6StJfy5XMKWX73HJ2Ol3SZ4D9gK+XM/6vlzKW9IyyvKWk70q6W9IySR9vnNVKOlLSlZK+JOk+SbdIelVTG94q6QZJD0paIukdG2IcJJ0g6dzSrgclLZa0d9P+OZLOK22+t6lfA6X9yyStKMdv2XTc4WXfvZKOaxPz31vG8AhJt0q6p7m8pJmSzipjcoOkDzdP40n6iKTbS9tvlHTAaPpve4ntU2zfaXuN7dOAGcCzRjj0C8CJ60sEkg4qY3l/uVJ9TtO+paXd1wGryvNnqaQPSbquXD1+R9JTJF1c+naJpK1H07cYmySS6JikOcCrgavLpgXAcqozzUOAkyW9tOz7KvBV21sAfwuc21qf7eOAhcAx5Yz/mDZhvwZsCewE7A+8BXhr0/59gRuB2VRvVN+RpLJvBXAgsEU55l8l7TmGrrdzEFX/twIuABrJYhrwU2AZMBd4WikHcGR5vKT0Z1bTcbsA/wYcTjWeTwK2H6EN86jevA8APtn0xnt8ib0T8HLgsMYBkp4FHAP8ve3NgVcAS0fX9aEk7UGVSG4aoeh5wF+pxqC1jp2Bs4H3AU8GLgJ+ImlGU7FDgdcAW9leXba9gaqPOwOvpTrR+VipYwB475g6FaOSRBKd+JGk+4ErgcupEsYc4IXAR2w/Yvsa4HSqN3qAx4FnSJpte6XtRaMNWt6U5wPH2n7Q9lLgy1Rvtg3LbH+73HM5C3gq8BQA2xfavtmVy4H/pLoC6tTvy9lx4/GKpn1X2r6oxP0esHvZvg9VIviQ7VVlbK4s+94MnFLO6FcCxwLzyxn6IcBPbV9h+1HgE8ATI7TvRNsP274WuLapDf8EnGz7PtvLgVObjlkDbAzsImkj20tt3zyKMRlC0hal/yfafmCE4qbq1ydaEgTAG4ELbf+X7ceBLwEzgRc0lTnV9m22H27a9jXbd5Wr5YXAr21fbfsR4HzgeWPtW3QuiSQ68TrbW9ne0fa7ygt5O+Avth9sKreM6gwc4Ciqs8T/lfRbSQeOIe5sYKNSb7sYAH9uLNh+qCzOApD0KkmLytTb/VRXU7NHEX/P0u/G4+ft4gIPAZuUhDCHKrmtZl3btenLdKrEtx1wW1NfVgH3jtC+1jY07lEMqaul3puozvpPAFZIWiBpuxHitCVpJvATYJHtzzZtX6y1H1AYkrhtX0R1Fds6zThkbGw/Udrd/Le+jXXd1bT8cJv1MX8wJDqXRBJjdQewjaTNm7btANwOYPv/bB8KbAt8HviBpM3a1DPcfz99D9WVzY7tYgxH0sbAD6nObJ9ieyuq6RINe2B9twE7rOc+wB2s25fVVG9+d1IlIQAkbUo1vTUWdzJ0WmxO807b37c9r7TFVH+fUSnj+yPaJAXbz236gMLCNocfRzX9tGnTtiFjU6Yn5zD0b53/qnyCSiKJMbF9G/BL4LOSNpG0G9VVSOOG8GGSnlzOLO8vh7WbqrmLai6/XYw1VPdWPiNpc0k7Ah9oxBjBDKopnLuB1eUm/D923MGx+w3VG/nnJG1WxuaFZd/ZwPslPV3VJ5xOBs4pVy8/AA6UNK9M+5zE2F+f5wLHStpa0tOo7okA1T0SSS8tieARqrP24abQppU+NB4zJG1U2vswcET5G3fM9mXA9cARLW1+jaQDSv0fBB6leo7FBJdEEnUcSnVT9w6q+ejjbV9S9r0SWCxpJdWN9/ktc9sNXwUOKZ8wOrXN/vcAq4AlVPdovg+cMVLDypTbe6neoO4D3kR1U3w0rtXQ75F8pYO4a6hu+j4DuJXqjP2NZfcZVPcTrgBuoXojf085bjHwbqr+3VnaPNYvTJ5Ujr0FuITqTf/Rsm9j4HNUV3t/prpiPHaYuj5KlTAaj0up7lscSJWY71/fNNYIPk710WEAbN9I9aGAr5W2vZbqY+ePjaLO6BHlh60ipjZJ76RK5Pv3ui0xNeWKJGKKkfRUSS9U9b2VZ1FNE53f63bF1JVviEZMPTOAbwFPp7o/tQD4Rk9bFFNaprYiIqKWTG1FREQtSSQREVFLX9wjeeCxnXoyf/eWmSf2ImxPzw6mjfv3/dpb04ffVRvVlzemiF49t9Wj5zX09rn94zWHd9TxXJFEREQtSSQREVFLEklERNSSRBIREbUkkURERC1JJBERUUsSSURE1JJEEhERtSSRRERELUkkERFRy4iJpPzCXeu2D0j6o6TrJP13+QnUdseeIOkhSdsOV19ERExeY70iuRrY2/ZuVD/j+YVhyt5D9cM6o6ZKrpoiIiawMb1J2/6F7YfK6iJg+2GKnwG8UdI2rTvKlc315fG+sm2upBslfRe4HphTfg/6i5IWS7pE0j6SLpO0RNJBY+lDRERsGBvibP8o4OJh9q+kSib/0rxR0l7AW4F9gecDb5f0vLL7mcA3bD/X9jJgM+BS288FHgQ+DbwceD1w0gboQ0REjFGtRCLpMGBv4IsjFD0VOELS5k3b5gHn215leyVwHrBf2bfM9qKmso8BPyvLfwAut/14WZ67nrYdLekqSVedefpfR9OtiIgYhTH/HomklwHHAfvbfrRs+wzwGgDbezTK2r5f0veBd3dY/aqW9ce99jeBnwAeLfU+IaltH2yfBpwGvfs9koiIfjCmK5IyBfUt4CDbKxrbbR9ne4/mJNLkFOAdrE1eC4HXSdpU0mZU01QLx9KeiIjonU6uSDaVtLxp/RTg1cAs4D8kAdxqe9ib3rbvkXQ+8P6y/ntJZwK/KUVOt321pLmj6kFERPSU1s4YTV35qd3uyU/tdk9+ard78lO7w8t3NCIiopYkkoiIqCWJJCIiakkiiYiIWpJIIiKiliSSiIioJYkkIiJqSSKJiIhakkgiIqKWJJKIiKjHdh7DPICj+y12v8VNn/sjbvo8fo9ckYzs6D6M3W9xexk7fe6P2FO6z0kkERFRSxJJRETUkkQystP6MHa/xe1l7PS5P2JP6T73xe+RRETE+MkVSURE1JJEEhERtSSRrIekNZKukXStpN9LesFUjjsBYr9OkiU9u1sxS9y/kbRA0s2SfifpIkk7dyl2+tydmH33mup63F59SWaiP4CVTcuvAC6fynEnQOxzgIXAiV2MKeBXwD83bdsd2C99nlJ97rvXVLfj5oqkM1sA9/VR3K7GljQLmAccBczvRsziJcDjtr/Z2GD7WtsLxztw+tzVPjfri9dUt+NOH8/KJ7mZkq4BNgGeCrx0isftZeyDgZ/Z/pOkeyXtZft3XYi7K9CNOO2kz93rcz++proaN1ck6/ew7T1sPxt4JfBdSZrCcXsZ+1BgQVleUNanuvS5e33ux9dUV+PmeyTrIWml7VlN63cBf2d7xVSM26vYkrYBlgN3AwamlX939Dg/OSUdABxv+0XjGadN3PS5u33uq9dUL+LmiqQD5RMm04B7+yFul2MfAnzP9o6259qeA9wC7DfOcQEuBTaWNPif2knaTdJ4x06fu9vnQX3ymup63NwjWb/GHCNUn3Q5wvaaKRy3V7EPBT7fsu2HZfsV4xnYtiW9HviKpI8AjwBLgfeNZ1zS54au9Jn+e011PW6mtiIiopZMbUVERC1JJBERUUsSSURE1JJEEhERtSSRRERELUkkERFRSxJJRETUkkQSERG1JJFEREQtSSQREVFLEklERNSSRBIREbUkkURERC1JJBERUUsSSURE1JJEEhERtfTFLyRuMfNznmYxjeoBMM0DDCAGGutlubE+YAbXB7chNGQdBrx2vdpHS/lSjxtlGvXSdIxQ+X2xxj41xyj7hx7TWgbkteuiXfm129Ypo7XbhuxXo45qeeg2D24DGJCRqrqa1wfLD667pYzRQMt66/51yrcr48E4A83rAwzZ31x+bZl162jU0xifRtzm/e2OGalODRkPw8AwdbSMz+C+NuWb+8sw7WJgPe1o0z+aywysGxcNHeMhcQda1sWQvg7uGxKzTZmWuDT97WkZvyHlm8aMljgMqZMh643lqh6GbGvUMVh/S1/XvqAaMRg8xgO01EHjTWRw3UNiVg831zsAbnrhVnWq+rftfjVt0+AxrXW4qf8eMAxUP6a45YwljdYMK1ckERFRSxJJRETUkkQSERG1JJFEREQtSSQREVFLEklERNSSRBIREbUkkURERC1JJBERUUsSSURE1JJEEhERtSSRRERELUkkERFRSxJJRETUkkQSERG1JJFEREQtst3rNow7SUfbPq3X7ZjoMk6dyTh1LmPVmck+Tv1yRXJ0rxswSWScOpNx6lzGqjOTepz6JZFERMQ4SSKJiIha+iWRTNq5xy7LOHUm49S5jFVnJvU49cXN9oiIGD/9ckUSERHjZEolEkmvlHSjpJskfbTN/o0lnVP2/1rS3O63svdGGqemcm+QZEl7d7N9E0UHz6cdJP1C0tWSrpP06l60s9cknSFphaTr17P/zWV8/iDpl5J273YbJ4KRxqmUebGkayQtlnR5N9tXi+0p8QCmATcDOwEzgGuBXVrKvAv4ZlmeD5zT63ZPxHEq5TYHrgAWAXv3ut0TcZyo5rXfWZZ3AZb2ut09GqsXAXsC169n/wuArcvyq4Bf97rNE3SctgL+COxQ1rftdZs7fUylK5J9gJtsL7H9GLAAOLilzMHAWWX5B8ABktTFNk4EnYwTwKeAzwOPdLNxE0gn42Rgi7K8JXBHF9s3Ydi+AvjLMPt/afu+sroI2L4rDZtgRhon4E3AebZvLeVXdKVhG8BUSiRPA25rWl9etrUtY3s18ADwpK60buIYcZwk7QnMsX1hNxs2wXTyfDoBOEzScuAi4D3dadqkdhRwca8bMUHtDGwt6TJJv5P0ll43qFPTe92AmFgkDQCnAEf2uCmTwaHAmba/LOkfgO9J2tX2E71u2EQk6SVUiWRer9syQU0H9gIOAGYCv5K0yPafetuskU2lK5LbgTlN69uXbW3LSJpONR1xb1daN3GMNE6bA7sCl0laCjwfuKAPb7h38nw6CjgXwPavgE2A2V1p3SQjaTfgdOBg2/32muvUcuDntlfZvofqHuWk+GDCVEokvwWeKenpkmZQ3Uy/oKXMBcARZfkQ4FKXu1p9ZNhxsv2A7dm259qeSzWnfZDtq3rT3J7p5Pl0K9XZI5KeQ5VI7u5qKycBSTsA5wGHT4az6x76MTBP0nRJmwL7Ajf0uE0dmTJTW7ZXSzoG+DnVJ27OsL1Y0knAVbYvAL5DNf1wE9VNr/m9a3FvdDhOfa/Dcfog8G1J76e68X5kH56YIOls4MXA7HK/6HhgIwDb3wQ+SXUv8hvlsy2rbffbFe6I42T7Bkk/A64DngBOt73ejwpPJPlme0RE1DKVprYiIqIHkkgiIqKWJJKIiKgliSQiImpJIomIiFqSSCIiopYkkoiIqCWJJCIiavl/YCXP8wx5AJwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = test(pos_enc=True, num_enc_layers=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}