NERTagger.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "logger = logging.getLogger()\n",
    "logger.setLevel(logging.DEBUG)\n",
    "\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import torch\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device: cuda\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print('Using device:', device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tesla K80\n",
      "Memory Usage:\n",
      "Allocated: 0.0 GB\n",
      "Cached:    0.0 GB\n"
     ]
    }
   ],
   "source": [
    "if device.type == 'cuda':\n",
    "    print(torch.cuda.get_device_name(0))\n",
    "    print('Memory Usage:')\n",
    "    print('Allocated:', round(torch.cuda.memory_allocated(0)/1024**3,1), 'GB')\n",
    "    print('Cached:   ', round(torch.cuda.memory_cached(0)/1024**3,1), 'GB')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import numpy as np\n",
    "from typing import Dict, List, Tuple\n",
    "\n",
    "from allennlp.common.file_utils import cached_path\n",
    "from allennlp.data import DataLoader\n",
    "from allennlp.data.fields import TextField, SequenceLabelField\n",
    "from allennlp.data.instance import Instance\n",
    "from allennlp.data.samplers import BucketBatchSampler\n",
    "from allennlp.data.token_indexers import TokenIndexer, SingleIdTokenIndexer\n",
    "from allennlp.data.tokenizers.token import Token\n",
    "from allennlp.data.vocabulary import Vocabulary\n",
    "from allennlp.models import Model\n",
    "from allennlp.modules.seq2seq_encoders import Seq2SeqEncoder, PytorchSeq2SeqWrapper\n",
    "from allennlp.modules.text_field_embedders import TextFieldEmbedder, BasicTextFieldEmbedder\n",
    "from allennlp.modules.token_embedders import Embedding\n",
    "from allennlp.nn.util import get_text_field_mask, sequence_cross_entropy_with_logits\n",
    "from allennlp.training.metrics import CategoricalAccuracy, SpanBasedF1Measure\n",
    "from allennlp.training.trainer import GradientDescentTrainer\n",
    "from allennlp.data.dataset_readers.dataset_reader import DatasetReader, AllennlpDataset\n",
    "from overrides import overrides"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_SIZE = 128\n",
    "HIDDEN_SIZE = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NERDatasetReader(DatasetReader):\n",
    "    def __init__(self, token_indexers: Dict[str, TokenIndexer]=None, lazy=False):\n",
    "        super().__init__(lazy=lazy)\n",
    "        self.token_indexers = token_indexers or {'tokens': SingleIdTokenIndexer()}\n",
    "\n",
    "    @overrides\n",
    "    def text_to_instance(self, tokens: List[Token], labels: List[str]=None):\n",
    "        fields = {}\n",
    "\n",
    "        text_field = TextField(tokens, self.token_indexers)\n",
    "        fields['tokens'] = text_field\n",
    "        if labels:\n",
    "            fields['labels'] = SequenceLabelField(labels, text_field)\n",
    "\n",
    "        return Instance(fields)\n",
    "\n",
    "    def _convert_sentence(self, rows: List[Tuple[str]]) -> Tuple[List[Token], List[str]]:\n",
    "        \"\"\"Given a list of rows, returns tokens and labels.\"\"\"\n",
    "        _, tokens, _, labels = zip(*rows)\n",
    "        tokens = [Token(t) for t in tokens]\n",
    "\n",
    "        # NOTE: the original dataset seems to confuse gpe with geo, and the distinction\n",
    "        # seems arbitrary. Here we replace both with 'gpe'\n",
    "        labels = [label.replace('geo', 'gpe') for label in labels]\n",
    "        return tokens, labels\n",
    "\n",
    "    @overrides\n",
    "    def _read(self, file_path: str):\n",
    "\n",
    "        file_path = cached_path(file_path)\n",
    "        sentence = []\n",
    "        with open(file_path, mode='r', encoding='utf-8', errors='ignore') as csv_file:\n",
    "            next(csv_file)\n",
    "            reader = csv.reader(csv_file)\n",
    "\n",
    "            for row in reader:\n",
    "                if row[0] and sentence:\n",
    "                    tokens, labels = self._convert_sentence(sentence)\n",
    "                    yield self.text_to_instance(tokens, labels)\n",
    "\n",
    "                    sentence = [row]\n",
    "                else:\n",
    "                    sentence.append(row)\n",
    "\n",
    "            if sentence:\n",
    "                tokens, labels = self._convert_sentence(sentence)\n",
    "                yield self.text_to_instance(tokens, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pixiedust": {
     "displayParams": {}
    }
   },
   "outputs": [],
   "source": [
    "class LstmTagger(Model):\n",
    "    def __init__(self,\n",
    "                 embedder: TextFieldEmbedder,\n",
    "                 encoder: Seq2SeqEncoder,\n",
    "                 vocab: Vocabulary) -> None:\n",
    "        super().__init__(vocab)\n",
    "        self.embedder = embedder\n",
    "        self.encoder = encoder\n",
    "        self.hidden2labels = torch.nn.Linear(in_features=encoder.get_output_dim(),\n",
    "                                             out_features=vocab.get_vocab_size('labels'))\n",
    "        self.accuracy = CategoricalAccuracy()\n",
    "        self.f1 = SpanBasedF1Measure(vocab, tag_namespace='labels')\n",
    "\n",
    "    def forward(self,\n",
    "                tokens: Dict[str, torch.Tensor],\n",
    "                labels: torch.Tensor = None) -> Dict[str, torch.Tensor]:\n",
    "        mask = get_text_field_mask(tokens)\n",
    "        embeddings = self.embedder(tokens)\n",
    "        encoder_out = self.encoder(embeddings, mask)\n",
    "        logits = self.hidden2labels(encoder_out)\n",
    "        output = {'logits': logits}\n",
    "        if labels is not None:\n",
    "            self.accuracy(logits, labels, mask)\n",
    "            self.f1(logits, labels, mask)\n",
    "            output['loss'] = sequence_cross_entropy_with_logits(logits, labels, mask)\n",
    "\n",
    "        return output\n",
    "\n",
    "    def get_metrics(self, reset: bool = False) -> Dict[str, float]:\n",
    "        f1_metrics = self.f1.get_metric(reset)\n",
    "        return {'accuracy': self.accuracy.get_metric(reset),\n",
    "                'prec': f1_metrics['precision-overall'],\n",
    "                'rec': f1_metrics['recall-overall'],\n",
    "                'f1': f1_metrics['f1-measure-overall']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67488863a0b5416eb0fbb6852c111ce5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "reader = NERDatasetReader()\n",
    "dataset = reader.read('https://s3.amazonaws.com/realworldnlpbook/data/entity-annotated-corpus/ner_dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(tokens: List[str], model: LstmTagger) -> List[str]:\n",
    "    token_indexers = {'tokens': SingleIdTokenIndexer()}\n",
    "    tokens = [Token(t) for t in tokens]\n",
    "    inst = Instance({'tokens': TextField(tokens, token_indexers)})\n",
    "    logits = model.forward_on_instance(inst)['logits']\n",
    "    label_ids = np.argmax(logits, axis=1)\n",
    "    labels = [model.vocab.get_token_from_index(label_id, 'labels')\n",
    "              for label_id in label_ids]\n",
    "    return labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = AllennlpDataset([inst for i, inst in enumerate(dataset) if i % 10 != 0])\n",
    "dev_dataset = AllennlpDataset([inst for i, inst in enumerate(dataset) if i % 10 == 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8fafd5e71ebc4feda9d32a2b1c2b7997",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=47959.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "vocab = Vocabulary.from_instances(train_dataset + dev_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "token_embedding = Embedding(num_embeddings=vocab.get_vocab_size('tokens'),\n",
    "                            embedding_dim=EMBEDDING_SIZE)\n",
    "word_embeddings = BasicTextFieldEmbedder({\"tokens\": token_embedding})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm = PytorchSeq2SeqWrapper(\n",
    "    torch.nn.LSTM(EMBEDDING_SIZE, HIDDEN_SIZE, bidirectional=True, batch_first=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LstmTagger(word_embeddings, lstm, vocab)\n",
    "\n",
    "train_dataset.index_with(vocab)\n",
    "dev_dataset.index_with(vocab)\n",
    "\n",
    "train_data_loader = DataLoader(train_dataset,\n",
    "                         batch_sampler=BucketBatchSampler(\n",
    "                             train_dataset,\n",
    "                             batch_size=16,\n",
    "                             sorting_keys=[\"tokens\"]))\n",
    "dev_data_loader = DataLoader(dev_dataset,\n",
    "                         batch_sampler=BucketBatchSampler(\n",
    "                             dev_dataset,\n",
    "                             batch_size=16,\n",
    "                             sorting_keys=[\"tokens\"]))\n",
    "\n",
    "optimizer = optim.Adam(model.parameters())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9cdcabf0e840479eaf1a1ee0c86c03f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2698.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03f66ac44bad478f96b5d4db1b353077",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=300.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "464a3844ae224574b078661bce1d8bb6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2698.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf80c4bbb9da497485bc11ce69c25323",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=300.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f71a49b832d474cafc03fed75bef82a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2698.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c90ccc902fe4686a9a37b8eab75b64f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=300.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8abdc3add4949daac80de17b86463f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2698.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8338dbde6d5a4db89d29299f95e93f18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=300.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c7b8292862349d795f09e83b17a6588",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2698.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33a8833561eb46fab4fce7b4de9107ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=300.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b4c50cb7b3e420cb1e225e10ec512ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2698.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8158937939024803ba975b1272e69d49",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=300.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cde6a5fdb10240ee918097218ed97d27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2698.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a3436a300d1407cb0cf50ed485e69a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=300.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6833c1264c37463ab03500b6ed527754",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2698.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bfac30baefd44284b37a19ec42ccf41d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=300.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bed4400166b44ef787b6cf9325cd333b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2698.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "132fbf5324aa4f7e83039458cd3075b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=300.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85235ba065f94164bf0d5e1f2d24eea2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2698.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d61738b79c64ebb90e902da790f38ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=300.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'best_epoch': 1,\n",
       " 'peak_worker_0_memory_MB': 924.472,\n",
       " 'peak_gpu_0_memory_MB': 11,\n",
       " 'training_duration': '0:29:12.683499',\n",
       " 'training_start_epoch': 0,\n",
       " 'training_epochs': 9,\n",
       " 'epoch': 9,\n",
       " 'training_accuracy': 0.9946135223474193,\n",
       " 'training_prec': 0.9599348051563195,\n",
       " 'training_rec': 0.9646228522080939,\n",
       " 'training_f1': 0.9622731188544787,\n",
       " 'training_loss': 0.014742819040041922,\n",
       " 'training_reg_loss': 0.0,\n",
       " 'training_worker_0_memory_MB': 924.472,\n",
       " 'training_gpu_0_memory_MB': 11,\n",
       " 'validation_accuracy': 0.9658073140456411,\n",
       " 'validation_prec': 0.7929292929292929,\n",
       " 'validation_rec': 0.8168281306063868,\n",
       " 'validation_f1': 0.8047013078825941,\n",
       " 'validation_loss': 0.14975996050285176,\n",
       " 'validation_reg_loss': 0.0,\n",
       " 'best_validation_accuracy': 0.9706674305356632,\n",
       " 'best_validation_prec': 0.8318417755503849,\n",
       " 'best_validation_rec': 0.8337818442770003,\n",
       " 'best_validation_f1': 0.8328106800465408,\n",
       " 'best_validation_loss': 0.0901502693273748,\n",
       " 'best_validation_reg_loss': 0.0}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer = GradientDescentTrainer(\n",
    "    model=model,\n",
    "    optimizer=optimizer,\n",
    "    data_loader=train_data_loader,\n",
    "    validation_data_loader=dev_data_loader,\n",
    "    patience=10,\n",
    "    num_epochs=10)\n",
    "\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Amazon/B-org has/O built/O Kindle/O to/O rule/O the/O market/O in/O the/O US/B-gpe\n"
     ]
    }
   ],
   "source": [
    "tokens = [\"Amazon\", \"has\", \"built\", \"Kindle\", \"to\", \"rule\", \"the\", \"market\", \"in\", \"the\", \"US\"]\n",
    "labels = predict(tokens, model)\n",
    "print(' '.join('{}/{}'.format(token, label) for token, label in zip(tokens, labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}