text classification - pytorch and torchtext.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook shows how to build a model to classify text with PyTorch and torchtext libraries.\n",
    "\n",
    "It is based on the TREC-6 dataset, which consists on 5,952 questions written in English, classified in the following categories, depending on their answer:\n",
    "\n",
    "* HUM: Human\n",
    "* DESC: Description\n",
    "* ABBR: Abbreviation\n",
    "* LOC: Location\n",
    "* NUM: Number\n",
    "* ENTY: Entity\n",
    "\n",
    "Author: manuelsh (https://github.com/manuelsh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchtext\n",
    "from torchtext import data, datasets\n",
    "import random\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Torchtext version used: 0.3.1\n",
      "PyTorch version used: 0.4.1\n"
     ]
    }
   ],
   "source": [
    "print('Torchtext version used:',torchtext.__version__)\n",
    "print('PyTorch version used:', torch.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 - Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train length: 5452\n",
      "Test length: 500\n"
     ]
    }
   ],
   "source": [
    "# Get datasets\n",
    "text_field = data.Field(lower=True, batch_first=True, tokenize='spacy')\n",
    "label_field = data.Field(sequential=False, unk_token = None)\n",
    "train, test = datasets.TREC.splits(text_field, label_field)\n",
    "\n",
    "print('Train length:',str(len(train)))\n",
    "print('Test length:',str(len(test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "what city 's airport is named logan international ? LOC\n",
      "what is the definition of the term `` weapons system '' ? DESC\n",
      "how many people in america snore ? NUM\n",
      "how do you determine heating requirements for your home ? DESC\n",
      "who killed more people , hitler or stalin ? HUM\n",
      "who banned peter rose from baseball for betting on games ? HUM\n",
      "how many horses are there on a polo team ? NUM\n",
      "what capital is built around monument circle which contains soldiers and sailors monument ? LOC\n",
      "what member of the little rascals has an on - again , off - again sweetheart in darla hood ? HUM\n",
      "what is usenet for the internet ? DESC\n"
     ]
    }
   ],
   "source": [
    "# Show some examples\n",
    "\n",
    "for i in range(10):\n",
    "    random_index = random.randint(0,len(train))\n",
    "    print(' '.join(train.examples[random_index].text), train.examples[random_index].label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary length: 5002\n",
      "Number of classes: 6\n"
     ]
    }
   ],
   "source": [
    "# Build vocabulary\n",
    "vocab_length = 5000\n",
    "text_field.build_vocab(train, max_size=vocab_length )\n",
    "label_field.build_vocab(train)\n",
    "\n",
    "classes_count = len(label_field.vocab)\n",
    "word_count = len(text_field.vocab)\n",
    "print('Vocabulary length:',word_count )\n",
    "print('Number of classes:',classes_count )\n",
    "\n",
    "# Show vocabulary with: text_field.vocab.stoi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2 - Build model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using a simple model with an embedding, an LSTM and a linear layer. Note that the classical softmax layer for a classification model is already inside the CrossEntropyLoss function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNN(\n",
       "  (embeddings): Embedding(5002, 128)\n",
       "  (rnn): LSTM(128, 128, batch_first=True)\n",
       "  (linear): Linear(in_features=128, out_features=6, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class RNN(nn.Module):\n",
    "    def __init__(self, num_tokens, embedding_dim, rnn_dim, num_layers, num_classes):\n",
    "        super(RNN, self).__init__()\n",
    "        self.embeddings = nn.Embedding(num_tokens, embedding_dim)\n",
    "        self.rnn = nn.LSTM(input_size = embedding_dim,\n",
    "                           hidden_size = rnn_dim, \n",
    "                           num_layers = num_layers,\n",
    "                           batch_first = True)\n",
    "        self.linear = nn.Linear(rnn_dim, num_classes)\n",
    "\n",
    "    def forward(self, x):\n",
    "        emb = self.embeddings(x)\n",
    "        rnn_output, rnn_hidden = self.rnn(emb)\n",
    "        output = self.linear(rnn_output[:,-1,:])\n",
    "        return output\n",
    "\n",
    "\n",
    "n_hidden = 128\n",
    "model = RNN(word_count, embedding_dim=128, rnn_dim=128, num_layers=1, num_classes=classes_count)\n",
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3 - Objective function (and optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_function = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "lr_decay = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.99) # Scheduled learnint rate, which decays the\n",
    "                                                                         # learning rate exponentially. This could\n",
    "                                                                         # potentially help arrive to a lower minimum.\n",
    "# Another option could be:\n",
    "#lr_decay = torch.optim.lr_scheduler.ReduceLROnPlateau( optimizer, factor=0.95, patience=10 )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4 - Train model (and test during training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "num_epochs = 5\n",
    "\n",
    "train_iter = data.BucketIterator(train, \n",
    "                                 batch_size=batch_size, \n",
    "                                 sort_within_batch=True, \n",
    "                                 shuffle = True, \n",
    "                                 repeat = False)\n",
    "\n",
    "test_iter = data.BucketIterator(test, \n",
    "                          batch_size=30, \n",
    "                          sort_within_batch=True, \n",
    "                          shuffle = True, \n",
    "                          repeat = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reset variables \n",
    "accuracies = []\n",
    "losses_train = []\n",
    "losses_test = []\n",
    "train_accuracy = 0\n",
    "step_count = 0\n",
    "max_accuracy = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training epoch  0\n",
      "Step:  0 Accuracy in test set: 0.17599999904632568\n",
      "Step:  50 Accuracy in test set: 0.3700000047683716\n",
      "Step:  100 Accuracy in test set: 0.4860000014305115\n",
      "Step:  150 Accuracy in test set: 0.5479999780654907\n",
      "Training epoch  1\n",
      "Step:  200 Accuracy in test set: 0.6399999856948853\n",
      "Step:  250 Accuracy in test set: 0.6320000290870667\n",
      "Step:  300 Accuracy in test set: 0.6800000071525574\n",
      "Training epoch  2\n",
      "Step:  350 Accuracy in test set: 0.6679999828338623\n",
      "Step:  400 Accuracy in test set: 0.6740000247955322\n",
      "Step:  450 Accuracy in test set: 0.6660000085830688\n",
      "Step:  500 Accuracy in test set: 0.671999990940094\n",
      "Training epoch  3\n",
      "Step:  550 Accuracy in test set: 0.6759999990463257\n",
      "Step:  600 Accuracy in test set: 0.6800000071525574\n",
      "Step:  650 Accuracy in test set: 0.6779999732971191\n",
      "Training epoch  4\n",
      "Step:  700 Accuracy in test set: 0.6759999990463257\n",
      "Step:  750 Accuracy in test set: 0.671999990940094\n",
      "Step:  800 Accuracy in test set: 0.6840000152587891\n",
      "Step:  850 Accuracy in test set: 0.6779999732971191\n"
     ]
    }
   ],
   "source": [
    "# Training loop\n",
    "for i in range(num_epochs):\n",
    "    print('Training epoch ',i)\n",
    "    train_iter.init_epoch()\n",
    "    for batch in train_iter:        \n",
    "\n",
    "        x_train = batch.text\n",
    "        y_train = batch.label\n",
    "\n",
    "        # Forward pass\n",
    "        y_model = model(x_train)\n",
    "\n",
    "        # Loss function\n",
    "        loss = loss_function(y_model, y_train)\n",
    "        losses_train.append(float(loss))\n",
    "\n",
    "        # Backward pass \n",
    "        model.zero_grad()\n",
    "        loss.backward()\n",
    "        \n",
    "        # Update parameters\n",
    "        optimizer.step()\n",
    "\n",
    "        # Evaluation in test set\n",
    "\n",
    "        if step_count%50 == 0:\n",
    "\n",
    "            # Calculate model in test set by pieces\n",
    "            model.eval() # Set model to eval (if there is dropout, will set it to zero)\n",
    "            y_model_test_list = []\n",
    "            y_test_list = []\n",
    "            \n",
    "            for test_batch in test_iter:            \n",
    "                y_model_test_list.append(model(test_batch.text))            \n",
    "                y_test_list.append(test_batch.label)\n",
    "            model.train() # Set model to train (if there is dropout, will not be zero anymore)\n",
    "            test_iter.init_epoch()\n",
    "            \n",
    "            # Calculate accuracy\n",
    "            accuracy = float( (torch.cat(y_model_test_list).max(dim=1)[1] == torch.cat(y_test_list)).float().mean() )\n",
    "            print('Step: ', step_count, 'Accuracy in test set:', accuracy)\n",
    "            accuracies.append(accuracy)\n",
    "\n",
    "        lr_decay.step()\n",
    "        step_count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11ca6d7f0>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAD8CAYAAAB9y7/cAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAH4VJREFUeJzt3Xl0nHd97/H3V8vI2mxr8xJLshxvqbM7islS0nCTgIE2bgsEp3CbtpRcTvGFLreXUDi5NL3nlNIWLrQ5LWEpLYfYCWFzOU5DaSiFhERS9jixY9mWLdmxrc3Wvn/vHzOSx/JIGsujeWb5vM7R0Tzz/DT+6tH4M9/5Pc88j7k7IiKSeXKCLkBERBaGAl5EJEMp4EVEMpQCXkQkQyngRUQylAJeRCRDKeBFRDKUAl5EJEMp4EVEMlRePIPMbCvwRSAX+Kq7f3ba+i8Ab4ssFgHL3H3pbI9ZWVnpdXV1F1ywiEg2e+655zrcvSqesXMGvJnlAg8CdwBtQKOZ7Xb31ybHuPsfRY3/n8C1cz1uXV0dTU1N8dQoIiIRZnYk3rHxTNFsAZrd/ZC7jwC7gG2zjL8b2BlvASIisjDiCfhVQGvUclvkvvOY2WpgDfDkxZcmIiIXI56Atxj3zXQKyu3AY+4+HvOBzO41syYza2pvb4+3RhERmYd4Ar4NqIlargaOzzB2O7NMz7j7Q+5e7+71VVVx7SMQEZF5iifgG4H1ZrbGzEKEQ3z39EFmthEoA36R2BJFRGQ+5gx4dx8DdgBPAK8Dj7r7XjN7wMzujBp6N7DLdQUREZGUENdx8O6+B9gz7b77py1/JnFliYjIxYor4EVEstX4hPPqsTM0tnQRysuhpqyImvIiqssKWZSfG3R5s1LAi4hEcXeaT/XxVHMHTx3s5JlDnfQOjcUcu3xxwVTg15QXUVNWSG3k9vLFi8jNiXUQYvIo4GXBtHT0s2Pn84Ryc7h5XSU3rq1gc21Zync9mWZodJzO/hE6+4bp7BuhPfK9s2+Yzv4ROvqG6YgsD46OUxTKpSiUR2F+LkWhXApD4e/Fobyp24WhvMi43Mi4s8tFUeMmfz6Um4NZsGE3m7buAZ5u7uSpgx08fbCT9t5hAGrKC3n3lSu5cW0FN66tAIfW7gGOdg3Q2jUY+T5Aw+Euvv/iMaL3QIZyc1hVVkh1VOjXlhdRU1bEmqpiSgoWPn4tqH2i9fX1rlMVZK627gHe/+VnGBgZY3VFMS+3nWbCoSAvh+vryrlpXQU3ra3kylVLAu9y0s3EhHNmcJTO/nAwd0QFdkf/CB29w+cEeu9w7O6zKJRLRUmIiuICKktCVJYUsCg/l8GRcQZGxxkcGWNgZJz+kbO3B0fGw99HY37UZUa5OXbuC0DUi0dx5MVh+otHcSiXxYX5VJYUTNVZXhxKyPOls2+Ypw92Rr46ONI5AEBlSYib1lZy09oKbl5XSU15UdyPOTI2wfHTg+e8ALR2h18AWrsG6B4YnRr753dezj031c2rdjN7zt3r4xmrDl4S7mTPEB/46rP0Do3y8Idv4IpVS+gZGqXhUFe4Q2ru5HP/th/YT+miPG64tIKbI/+h1i0rSVinNzHhtPcNh/+DdQ9wtHOQ9r4hqsuK2LiilI3LS1m5ZFFKdJbRXfbZjvrscrjTDi939Y8wNnF+Y2YG5UWhqUC8snrpVHBXFIeoKDkb5BUlIYpC8//vPzHhDI2N0z8cCf3Rc18ABiIvCANRLw5T6yMvHv3D4/QMjXGyZyjuF4/J33Hqhak0/LtVlkz+fuHfrbK4gMrSs79j3/AYzx4KB/pTzR3sO9ELQGlBHm+5tJx7bqzj5nWVbFg+/+dfKC+Huspi6iqLY67vHRqdCv1fWrF4Xv/GhVIHLwnV2TfM+x96hjdPD/LN338Lm2vLYo5r7x3mF4c6ebq5g6cOdtDaNQhAVWlBuHtaW8lN6yqoLpu9g+oZGo10SINngzzSMbV1DzI8NnHO+MWL8uiJmk8tXZTHxuWl4cCPhP7GFaUsLQpd5JY4y93p7B+htStcW1v34NTt46cHZ+2yC/NzqSw922VXRIKrojgSZJFQKy8OJay7DVr0i8eZwdHIi9zI1DuWqRe9vpGpKaaZ5sgL83MpLw5xomeI8QknlJdD/eoybl4X7tKvXLWEvNz0Omv6hXTwCnhJmDMDo9z9lWc41NHHN353CzdcWhH3z7Z2DfD0wQ6eag53WR194TnQ2vIibl5XwfV15QyMjNPaPUDb5Nxn9wCno972Qjiwa8oic53l4bnP6si85+RRD6cHRth/opc3TvayL+p7dEgsX1zAhuWlXLaiNPJ9MeuWlVAYir3/oH94LPJ2fDAqyM++TR8YObcrrSwJUVNexKqlhVSVFixIl51NhkbH6eoPv+vp6B8+b5pq5dJF3Ly2ks2r038fkAJekq5veIwPfvVZXjvew1fuqedXNsz/VBTuzhsn+6YC/9lDnVMd7uSOq8kjFqJ3XNWWF7GkKH/e/+aJnqFw4J/oZf+JXvaf7OXAqT5GIu8CzKCuopgNy0uoKSviZO9wOMi7BujsHznn8YpCueEXl6gXm5qyImorwi80Cm6ZLwW8JNXgyDj3/FMDzx3p5h8+sJm3X74ioY8/Nj5Bc3sfixflJ/3Qs7HxCY50DYQDP/L1xsle2k4PsmLxorPvEsqij5IopLw4lBJz+5J5tJNVkmZ4bJx7v9lEY0sXX9x+bcLDHSAvN4fLkrRTKta/vbaqhLVVJbzrypWB1CAyX+m1d0FSyuj4BB/91gv87EAHf/Weq7jz6kuCLklEoijgZV7GJ5w/euRFfvz6Sf5i2+XcVV8z9w+JSFIp4OWCTUw4n/jOy/zw5Tf5s3ddxn+/sS7okkQkBgW8XBB35//s3stjz7Xxh7ev595b1gZdkojMQAEvcXN3/vLxfXzzmSP8j1su5eO3rQ+6JBGZhQJe4vb/fnyAh/7rEL9942rue+dlOgxQJMUp4CUuX/7pQb74Hwd433XVfObXLle4i6QBHQefYV49doaTPUNsXFHKqqWFCQnif/lFC3/5+D5+7epL+Ox7riInA853IpINFPAZ5Mxg+Fwwk+dUKSnIY8PykqmTaG1YET6nSnlx/CfSerSxlft/sJc7Ni3n83ddnREnsxLJFgr4DPL1nx+md2iML26/hr7hMd44ET6J1uOvnmBnQ+vUuMqSgqiTaIWDf8PykvPOj/KDF4/xie++zC0bqvj737qW/DQ7655ItlPAZ4gzg6N8/anDbL18BduuWXXOOnenvXeY/SfPnk9l/8leHm44wtDo2RNp1ZYXTYV+USiPv/nRfrbUlfPlD15HQV56n4FPJBsp4DPEZPf+sRiHLpoZyxYvYtniRbx1/dmzPI5POK1dA2eDP/L9yX2nGJ9wrq1dytd+5/oZT5ErIqlNAZ8Borv3TZfEf1Ku3BybugLNO6JOEjY8Nk5r1wCrK4o1LSOSxhTwGWC27n0+CvJyWbesNCGPJSLBUXuW5ubbvYtI5lPAp7lEd+8ikjkU8GlM3buIzEYBn8bUvYvIbBTwaUrdu4jMRQGfptS9i8hc4gp4M9tqZvvNrNnM7pthzF1m9pqZ7TWzhxNbpkRT9y4i8ZjzOHgzywUeBO4A2oBGM9vt7q9FjVkPfBK42d27zWzZQhUs6t5FJD7xdPBbgGZ3P+TuI8AuYNu0MR8GHnT3bgB3P5XYMmWSuncRiVc8Ab8KaI1abovcF20DsMHMnjKzZ8xsa6IKlHOpexeReMVzqoJYJwD3GI+zHrgVqAZ+ZmZXuPvpcx7I7F7gXoDa2toLLjbbqXsXkQsRTwffBtRELVcDx2OM+YG7j7r7YWA/4cA/h7s/5O717l5fVVU1fbXMQd27iFyIeAK+EVhvZmvMLARsB3ZPG/N94G0AZlZJeMrmUCILzXbq3kXkQs0Z8O4+BuwAngBeBx51971m9oCZ3RkZ9gTQaWavAT8B/tTdOxeq6Gyk7l1ELlRcpwt29z3Anmn33R9124E/jnxJgql7F5H50CdZ04C6dxGZDwV8ilP3LiLzpYBPcV9T9y4i86SAT2FnBkb5p5+rexeR+VHAp7CvPXWY3mF17yIyPwr4FKXuXUQulgI+Ral7F5GLpYBPQereRSQRFPApSN27iCSCAj7FqHsXkURRwKcYde8ikigK+BSi7l1EEkkBn0LUvYtIIingU4S6dxFJNAV8ivjqzw+pexeRhFLAp4CXWk/zjz89yK9dfYm6dxFJGAV8wHqGRtmx83mWlS7i/267IuhyRCSDxHVFJ1kY7s5933mZN08P8ehHbmRJUX7QJYlIBlEHH6BvPXuUPa+c4E/fsZHNtWVBlyMiGUYBH5DXjvfwwA9f49aNVXz4rZcGXY6IZCAFfAD6h8fY8fDzlBXl87fvu5qcHAu6JBHJQJqDTzJ359Pff5WWzn4e/vANVJQUBF2SiGQodfBJ9thzbXzvhWN8/LYN3HBpRdDliEgGU8An0YGTvdz/g73ceGkFO/7buqDLEZEMp4BPksGRcXY8/AJFoVy+uP0acjXvLiILTHPwSfLAD/ey/2Qv//J7W1i2eFHQ5YhIFlAHnwQ/ePEYOxta+YNb13LLhqqgyxGRLKGAX2CHO/r5s+++Qv3qMv74jg1BlyMiWUQBv4CGx8bZ8fDz5Ofl8KW7ryUvV5tbRJJHc/AL6C/37GPv8R6++tv1XLK0MOhyRCTLxNVSmtlWM9tvZs1mdl+M9b9jZu1m9mLk6/cTX2p6+bdXT/CNp1v40C+v4fZNy4MuR0Sy0JwdvJnlAg8CdwBtQKOZ7Xb316YNfcTddyxAjWmntWuA//3YS1xVvYRPbL0s6HJEJEvF08FvAZrd/ZC7jwC7gG0LW1b6Gh2f4GO7XsAd/v7uzYTyNO8uIsGIJ31WAa1Ry22R+6Z7j5m9bGaPmVlNQqpLQ3/zo/28cPQ0n33PVdRWFAVdjohksXgCPtZHLn3a8r8Cde5+FfBj4J9jPpDZvWbWZGZN7e3tF1ZpGvjJ/lN8+aeH+OANtbz7qpVBlyMiWS6egG8DojvyauB49AB373T34cjiV4DrYj2Quz/k7vXuXl9VlVkf+DlxZog/efQlLltRyqffvSnockRE4gr4RmC9ma0xsxCwHdgdPcDMotvVO4HXE1di6huLzLsPjY7z4Ac2syg/N+iSRETmPorG3cfMbAfwBJALfN3d95rZA0CTu+8GPmZmdwJjQBfwOwtYc8r50pPNNBzu4gvvv5q1VSVBlyMiAsT5QSd33wPsmXbf/VG3Pwl8MrGlpYenmzv4uycP8L7rqvmNa6uDLkdEZIqO4btIn/23fdSWF/Hn2y4PuhQRkXMo4C/Cq8fO8HLbGX73pjqKQjrrg4ikFgX8RdjVeJSCvBxNzYhISlLAz9PAyBjff+E4775yJUuK8oMuR0TkPAr4efrhy2/SNzzG3W+pDboUEZGYFPDztLPhKOuWlVC/uizoUkREYlLAz8O+Ez28cPQ026+vwUwXzxaR1KSAn4ddDa2EcnP4zc3auSoiqUsBf4GGRsf57vNtbL1iBeXFoaDLERGZkQL+Au155U16hsbYviVrz4gsImlCAX+BdjYcpa6iiBsvrQi6FBGRWSngL0DzqV4aW7rZvqVWO1dFJOUp4C/AzoZW8nON916nnasikvoU8HGa3Ll6x6blVJYUBF2OiMicFPBxemLvCboHRrl7iz65KiLpQQEfp10NrdSUF3Lz2sqgSxERiYsCPg6HO/r5xaFOtl9fS06Odq6KSHpQwMdhV+NRcnOM92nnqoikEQX8HEbGJnisqY3bLlvGssWLgi5HRCRuCvg5/Pj1k3T2j+i0wCKSdhTwc9jZcJRVSwu5ZX1V0KWIiFwQBfwsWrsG+NmBDu6qryFXO1dFJM0o4Gexq/EoOQZ3Xa+dqyKSfhTwMxgdn+DbTW28beMyVi4pDLocEZELpoCfwZP7TnGqd5jt+uSqiKQpBfwMdjUcZfniAt62UTtXRSQ9KeBjOHZ6kP98o53319eQl6tNJCLpSekVwyONrQDcdb2u2iQi6UsBP83Y+ATfbmrllvVVVJcVBV2OiMi8KeCn+ekb7bx5Zoi7dc1VEUlzcQW8mW01s/1m1mxm980y7r1m5mZWn7gSk2tnQyuVJQXc9kvLgy5FROSizBnwZpYLPAi8E9gE3G1mm2KMKwU+Bjyb6CKT5cSZIZ7cd5L31VeTr52rIpLm4kmxLUCzux9y9xFgF7Atxri/AD4HDCWwvqT6dlMrEw7btXNVRDJAPAG/CmiNWm6L3DfFzK4Fatz9hwmsLakmJpxdja3cvK6C1RXFQZcjInLR4gn4WGfZ8qmVZjnAF4A/mfOBzO41syYza2pvb4+/yiT4WXMHx04P6pqrIpIx4gn4NiB6zqIaOB61XApcAfynmbUANwC7Y+1odfeH3L3e3eurqlLrE6I7nz1KeXGIOzZp56qIZIZ4Ar4RWG9ma8wsBGwHdk+udPcz7l7p7nXuXgc8A9zp7k0LUvECONU7xI9fP8l7r6umIC836HJERBJizoB39zFgB/AE8DrwqLvvNbMHzOzOhS4wGR57ro2xCef92rkqIhkkL55B7r4H2DPtvvtnGHvrxZeVPBMTziONrbxlTTlrq0qCLkdEJGGy/mDvXxzq5EjnAL+la66KSIbJ+oDf2XCUpUX5vOPyFUGXIiKSUFkd8J19wzyx9wS/eW01i/K1c1VEMktWB/x3nz/G6LjrxGIikpGyNuDdnZ0NR6lfXcb65aVBlyMiknBZG/DPHu7iUEe/rrkqIhkrawP+X186TnEol3dfuTLoUkREFkTWBnzD4S6uX1NOYUg7V0UkM2VlwHf3j3DgVB/X15UHXYqIyILJyoBvOtINoIAXkYyWlQHf2NJFKDeHq6qXBF2KiMiCydqAv6p6iT7cJCIZLesCfnBknFfaznD9Gk3PiEhmy7qAf6G1m7EJZ4vm30Ukw2VdwDce7sYMNq8uC7oUEZEFlXUB33Ski43LS1lSmB90KSIiCyqrAn5sfILnj3SzRfPvIpIFsirgX3uzh/6RcR3/LiJZIasCvrFFH3ASkeyRXQF/uIua8kJWLFkUdCkiIgsuawLe3Wls6VL3LiJZI2sC/lBHP539Izr+XUSyRtYEfFNLFwD1CngRyRJZE/ANh7upKA6xtqo46FJERJIiawK+saWL+royzCzoUkREkiIrAv5kzxBHuwa0g1VEskpWBHxjZP5dAS8i2SQ7Av5wF0WhXC6/ZHHQpYiIJE1WBHxDSzfX1i4lLzcrfl0RESALAr5naJR9J3o0PSMiWSeugDezrWa238yazey+GOs/YmavmNmLZvZzM9uU+FLn57kj3bijDziJSNaZM+DNLBd4EHgnsAm4O0aAP+zuV7r7NcDngM8nvNJ5ajzcRV6OcU3t0qBLERFJqng6+C1As7sfcvcRYBewLXqAu/dELRYDnrgSL05jSxeXr1pCUSgv6FJERJIqnoBfBbRGLbdF7juHmX3UzA4S7uA/lpjyLs7Q6DgvtZ5hS50uzyci2SeegI/10c/zOnR3f9Dd1wKfAD4d84HM7jWzJjNram9vv7BK5+GVY2cYGZ/QDlYRyUrxBHwbUBO1XA0cn2X8LuDXY61w94fcvd7d66uqquKvcp4aDusEYyKSveIJ+EZgvZmtMbMQsB3YHT3AzNZHLb4bOJC4EuevqaWLdctKKC8OBV2KiEjSzbnn0d3HzGwH8ASQC3zd3fea2QNAk7vvBnaY2e3AKNAN3LOQRcdjfMJpOtLNr151SdCliIgEIq5DS9x9D7Bn2n33R93+eILrumj7T/TSOzTGljXawSoi2SljP8k6eYKx+tWafxeR7JTRAb9yySKqywqDLkVEJBAZGfDRF9jWBT5EJFtlZMC3dg1ysmeY6/UBJxHJYhkZ8FMX+Fij+XcRyV4ZG/BLCvPZsKw06FJERAKTkQHf0NJF/eoycnI0/y4i2SvjAr6jb5hD7f06PYGIZL2MC/imlm4AfcBJRLJexgV8Y0sXBXk5XLFqSdCliIgEKiMD/uqapRTk5QZdiohIoDIq4PuHx9h7vEfXXxURIcMC/oWjpxmfcB3/LiJChgV8Q0sXOQabdYFtEZHMCvjGw1380srFlC7KD7oUEZHAZUzAj45P8EJrt66/KiISkTEB/+qxMwyNTrBF8+8iIkAGBfzUBT50BkkRESCDAr7hcDd1FUUsK10UdCkiIikhIwJ+YsJ57kiX5t9FRKJkRMAfbO+je2BUAS8iEiUjAr5BF/gQETlPRgR8U0s3lSUF1FUUBV2KiEjKyIiAbzjcxZY1ZbrAtohIlLQP+OOnBzl2epD61ZqeERGJlvYBP3n8uz7gJCJyrowI+JKCPC5boQtsi4hES/+AP9zNtbVLyctN+19FRCSh0joVTw+MsP9kry7wISISQ1oH/HNHwhfY1vHvIiLniyvgzWyrme03s2Yzuy/G+j82s9fM7GUz+w8zW534Us/X0NJFfq5xTY0u8CEiMt2cAW9mucCDwDuBTcDdZrZp2rAXgHp3vwp4DPhcoguNpfFwF1euWsKifF1gW0Rkung6+C1As7sfcvcRYBewLXqAu//E3Qcii88A1Ykt83xDo+O8cuyMpmdERGYQT8CvAlqjltsi983kQ8DjF1NUPF5sPc3ouGsHq4jIDPLiGBPr8/8ec6DZB4F64FdmWH8vcC9AbW1tnCXG1ng4/AGn61brAh8iIrHE08G3ATVRy9XA8emDzOx24FPAne4+HOuB3P0hd6939/qqqqr51DuloaWLjctLWVoUuqjHERHJVPEEfCOw3szWmFkI2A7sjh5gZtcCXyYc7qcSX+a5xsYneP5IN9evUfcuIjKTOQPe3ceAHcATwOvAo+6+18weMLM7I8P+GigBvm1mL5rZ7hkeLiH2neilf2RcF/gQEZlFPHPwuPseYM+0++6Pun17guuaVUNk/l0BLyIys7T8JGtjSxerlhZyydLCoEsREUlZaRfw7k5jS7dODywiMoe0C/iWzgE6+oapr9MOVhGR2aRdwE8e/64POImIzC7tAn5pUT53bFrO2qqSoEsREUlpcR1Fk0refvkK3n75iqDLEBFJeWnXwYuISHwU8CIiGUoBLyKSoRTwIiIZSgEvIpKhFPAiIhlKAS8ikqEU8CIiGcrcY159b+H/YbN24Mg8f7wS6EhgOcmgmpMj3WpOt3pBNSfLTDWvdve4LokXWMBfDDNrcvf6oOu4EKo5OdKt5nSrF1RzsiSiZk3RiIhkKAW8iEiGSteAfyjoAuZBNSdHutWcbvWCak6Wi645LefgRURkbunawYuIyBxSOuDNbKuZ7TezZjO7L8b6AjN7JLL+WTOrS36V59RTY2Y/MbPXzWyvmX08xphbzeyMmb0Y+bo/iFqn1dRiZq9E6mmKsd7M7EuR7fyymW0Oos5ILRujtt2LZtZjZn84bUzg29jMvm5mp8zs1aj7ys3s383sQOR7zOtOmtk9kTEHzOyegGv+azPbF/m7f8/Mls7ws7M+h5Jc82fM7FjU3/9dM/zsrPmS5Jofiaq3xcxenOFnL2w7u3tKfgG5wEHgUiAEvARsmjbmD4B/jNzeDjwScM0rgc2R26XAGzFqvhX4YdDbd1pNLUDlLOvfBTwOGHAD8GzQNUc9R04QPi44pbYxcAuwGXg16r7PAfdFbt8H/FWMnysHDkW+l0VulwVY89uBvMjtv4pVczzPoSTX/Bngf8Xx3Jk1X5JZ87T1fwvcn4jtnMod/Bag2d0PufsIsAvYNm3MNuCfI7cfA24zM0tijedw9zfd/fnI7V7gdWBVUPUk0DbgXzzsGWCpma0MuijgNuCgu8/3A3MLxt3/C+iadnf08/WfgV+P8aPvAP7d3bvcvRv4d2DrghUaJVbN7v4jdx+LLD4DVCejlnjNsJ3jEU++LIjZao7k113AzkT8W6kc8KuA1qjlNs4Py6kxkSfhGaAiKdXNITJddC3wbIzVN5rZS2b2uJldntTCYnPgR2b2nJndG2N9PH+LIGxn5v8IqbaNAZa7+5sQbgaAZTHGpOq2Bvg9wu/kYpnrOZRsOyLTSl+fYSosVbfzW4GT7n5ghvUXtJ1TOeBjdeLTD/mJZ0zSmVkJ8B3gD929Z9rq5wlPKVwN/B3w/WTXF8PN7r4ZeCfwUTO7Zdr6lNvOZhYC7gS+HWN1Km7jeKXctgYws08BY8C3Zhgy13Momf4BWAtcA7xJeMpjupTczsDdzN69X9B2TuWAbwNqopargeMzjTGzPGAJ83u7ljBmlk843L/l7t+dvt7de9y9L3J7D5BvZpVJLnN6Tccj308B3yP89jVaPH+LZHsn8Ly7n5y+IhW3ccTJyamtyPdTMcak3LaO7Oj9VeADHpkIni6O51DSuPtJdx939wngKzPUkorbOQ/4TeCRmcZc6HZO5YBvBNab2ZpIt7Yd2D1tzG5g8iiD9wJPzvQETIbI/NnXgNfd/fMzjFkxuZ/AzLYQ/ht0Jq/K8+opNrPSyduEd6q9Om3YbuC3I0fT3ACcmZxqCNCMnU6qbeMo0c/Xe4AfxBjzBPB2MyuLTC28PXJfIMxsK/AJ4E53H5hhTDzPoaSZtn/oN2aoJZ58SbbbgX3u3hZr5by2czL2Gl/E3uZ3ET4S5SDwqch9DxB+sgEsIvwWvRloAC4NuN5fJvw272XgxcjXu4CPAB+JjNkB7CW81/4Z4KaAa740UstLkbomt3N0zQY8GPk7vALUB1xzEeHAXhJ1X0ptY8IvPm8Co4S7xQ8R3j/0H8CByPfyyNh64KtRP/t7ked0M/C7AdfcTHiuevL5PHnU2iXAntmeQwHW/M3I8/RlwqG9cnrNkeXz8iWomiP3f2PyORw19qK2sz7JKiKSoVJ5ikZERC6CAl5EJEMp4EVEMpQCXkQkQyngRUQylAJeRCRDKeBFRDKUAl5EJEP9f5DYZ1/QpyEDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(accuracies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercises:\n",
    "* Add dropout\n",
    "* Add more complexity to the model (RNN layers, other layers)\n",
    "* Bidirectional RNN\n",
    "* Use pretrained embeddings such as word2vec or GLOVE. Note that you can use: nn.Embedding.from_pretrained(...)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}