logistic_regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basics\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# Text pre-processing\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "from gensim.models import Word2Vec\n",
    "from gensim.utils import simple_preprocess\n",
    "from gensim.sklearn_api import W2VTransformer\n",
    "# Models\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment</th>\n",
       "      <th>parent_comment</th>\n",
       "      <th>original_post</th>\n",
       "      <th>divisiveness</th>\n",
       "      <th>divisiveness_keyword</th>\n",
       "      <th>human_incivility</th>\n",
       "      <th>stereotype</th>\n",
       "      <th>namecalling</th>\n",
       "      <th>aspersion</th>\n",
       "      <th>...</th>\n",
       "      <th>third_party_attack</th>\n",
       "      <th>civility</th>\n",
       "      <th>comment_de</th>\n",
       "      <th>comment_en</th>\n",
       "      <th>bert_aug</th>\n",
       "      <th>attention_aug</th>\n",
       "      <th>roberta</th>\n",
       "      <th>attention_mask</th>\n",
       "      <th>roberta_aug</th>\n",
       "      <th>attention_mask_aug</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fysijl2</td>\n",
       "      <td>If I wanted to watch a stable genius, I'd rath...</td>\n",
       "      <td>It's going to devolve into a campaign stump sp...</td>\n",
       "      <td>Today President Trump is expected to hold his ...</td>\n",
       "      <td>1</td>\n",
       "      <td>Trump, coronavirus</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Wenn ich ein stabiles Genie sehen wollte, würd...</td>\n",
       "      <td>If I wanted to see a stable genius, I'd rather...</td>\n",
       "      <td>[101, 2065, 1045, 2359, 2000, 2156, 1037, 6540...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 1106, 38, 770, 7, 1183, 10, 4375, 16333, 6...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 1106, 38, 770, 7, 192, 10, 4375, 16333, 6,...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fygrqc9</td>\n",
       "      <td>If they're willing to shoot a deceptive video ...</td>\n",
       "      <td>Look, you're simply wrong or haven't done the ...</td>\n",
       "      <td>Media are just plain lying about Kayleigh McEn...</td>\n",
       "      <td>1</td>\n",
       "      <td>Kayleigh McEnany</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Wenn sie bereit sind, ein irreführendes Video ...</td>\n",
       "      <td>If they are willing to make a misleading video...</td>\n",
       "      <td>[101, 2065, 2027, 2024, 5627, 2000, 2191, 1037...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 1106, 51, 214, 2882, 7, 4511, 10, 31405, 5...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 1106, 51, 32, 2882, 7, 146, 10, 12030, 569...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>g0ql4qq</td>\n",
       "      <td>Would be thrilling if he ended up actually bei...</td>\n",
       "      <td>Second challenge is disputing just about every...</td>\n",
       "      <td>Today President Trump is expected to hold his ...</td>\n",
       "      <td>1</td>\n",
       "      <td>Kanye West</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Wäre spannend, wenn er am Ende tatsächlich für...</td>\n",
       "      <td>It would be exciting if he was actually punish...</td>\n",
       "      <td>[101, 2009, 2052, 2022, 10990, 2065, 2002, 200...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 29042, 28, 16208, 114, 37, 1249, 62, 888, ...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 243, 74, 28, 3571, 114, 37, 21, 888, 14459...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fz1nlup</td>\n",
       "      <td>Nope they like to let one vote side against th...</td>\n",
       "      <td>LETS GOOOO. Romney found his balls!!</td>\n",
       "      <td>Romney reportedly will vote against controvers...</td>\n",
       "      <td>1</td>\n",
       "      <td>Romney, controversial Fed nominee</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Nein, sie lassen gerne eine Seite gegen die Di...</td>\n",
       "      <td>No, they like to let one side vote against thi...</td>\n",
       "      <td>[101, 2053, 1010, 2027, 2066, 2000, 2292, 2028...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 487, 9877, 51, 101, 7, 905, 65, 900, 526, ...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 3084, 6, 51, 101, 7, 905, 65, 526, 900, 13...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>fzd1ea2</td>\n",
       "      <td>Most conservatives agree with you. However, co...</td>\n",
       "      <td>you know you can commit a violent crime and no...</td>\n",
       "      <td>If african americans commit 3x more violent cr...</td>\n",
       "      <td>1</td>\n",
       "      <td>african american crime, police</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Die meisten Konservativen stimmen Ihnen zu. Ab...</td>\n",
       "      <td>Most conservatives agree with you, but to comm...</td>\n",
       "      <td>[101, 2087, 11992, 5993, 2007, 2017, 1010, 202...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 2895, 9930, 2854, 19, 47, 4, 635, 6, 11389...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 2895, 9930, 2854, 19, 47, 6, 53, 7, 6225, ...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                            comment  \\\n",
       "0  fysijl2  If I wanted to watch a stable genius, I'd rath...   \n",
       "1  fygrqc9  If they're willing to shoot a deceptive video ...   \n",
       "2  g0ql4qq  Would be thrilling if he ended up actually bei...   \n",
       "3  fz1nlup  Nope they like to let one vote side against th...   \n",
       "4  fzd1ea2  Most conservatives agree with you. However, co...   \n",
       "\n",
       "                                      parent_comment  \\\n",
       "0  It's going to devolve into a campaign stump sp...   \n",
       "1  Look, you're simply wrong or haven't done the ...   \n",
       "2  Second challenge is disputing just about every...   \n",
       "3               LETS GOOOO. Romney found his balls!!   \n",
       "4  you know you can commit a violent crime and no...   \n",
       "\n",
       "                                       original_post divisiveness  \\\n",
       "0  Today President Trump is expected to hold his ...            1   \n",
       "1  Media are just plain lying about Kayleigh McEn...            1   \n",
       "2  Today President Trump is expected to hold his ...            1   \n",
       "3  Romney reportedly will vote against controvers...            1   \n",
       "4  If african americans commit 3x more violent cr...            1   \n",
       "\n",
       "                divisiveness_keyword  human_incivility  stereotype  \\\n",
       "0                 Trump, coronavirus                 2           0   \n",
       "1                   Kayleigh McEnany                 0           0   \n",
       "2                         Kanye West                 1           0   \n",
       "3  Romney, controversial Fed nominee                 0           0   \n",
       "4     african american crime, police                 2           1   \n",
       "\n",
       "   namecalling  aspersion  ...  third_party_attack  civility  \\\n",
       "0            3          0  ...                   1         4   \n",
       "1            0          0  ...                   0         1   \n",
       "2            0          0  ...                   0         1   \n",
       "3            0          0  ...                   0         0   \n",
       "4            0          1  ...                   0         3   \n",
       "\n",
       "                                          comment_de  \\\n",
       "0  Wenn ich ein stabiles Genie sehen wollte, würd...   \n",
       "1  Wenn sie bereit sind, ein irreführendes Video ...   \n",
       "2  Wäre spannend, wenn er am Ende tatsächlich für...   \n",
       "3  Nein, sie lassen gerne eine Seite gegen die Di...   \n",
       "4  Die meisten Konservativen stimmen Ihnen zu. Ab...   \n",
       "\n",
       "                                          comment_en  \\\n",
       "0  If I wanted to see a stable genius, I'd rather...   \n",
       "1  If they are willing to make a misleading video...   \n",
       "2  It would be exciting if he was actually punish...   \n",
       "3  No, they like to let one side vote against thi...   \n",
       "4  Most conservatives agree with you, but to comm...   \n",
       "\n",
       "                                            bert_aug  \\\n",
       "0  [101, 2065, 1045, 2359, 2000, 2156, 1037, 6540...   \n",
       "1  [101, 2065, 2027, 2024, 5627, 2000, 2191, 1037...   \n",
       "2  [101, 2009, 2052, 2022, 10990, 2065, 2002, 200...   \n",
       "3  [101, 2053, 1010, 2027, 2066, 2000, 2292, 2028...   \n",
       "4  [101, 2087, 11992, 5993, 2007, 2017, 1010, 202...   \n",
       "\n",
       "                                       attention_aug  \\\n",
       "0  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "1  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "2  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "3  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "4  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "\n",
       "                                             roberta  \\\n",
       "0  [0, 1106, 38, 770, 7, 1183, 10, 4375, 16333, 6...   \n",
       "1  [0, 1106, 51, 214, 2882, 7, 4511, 10, 31405, 5...   \n",
       "2  [0, 29042, 28, 16208, 114, 37, 1249, 62, 888, ...   \n",
       "3  [0, 487, 9877, 51, 101, 7, 905, 65, 900, 526, ...   \n",
       "4  [0, 2895, 9930, 2854, 19, 47, 4, 635, 6, 11389...   \n",
       "\n",
       "                                      attention_mask  \\\n",
       "0  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "1  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "2  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "3  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "4  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "\n",
       "                                         roberta_aug  \\\n",
       "0  [0, 1106, 38, 770, 7, 192, 10, 4375, 16333, 6,...   \n",
       "1  [0, 1106, 51, 32, 2882, 7, 146, 10, 12030, 569...   \n",
       "2  [0, 243, 74, 28, 3571, 114, 37, 21, 888, 14459...   \n",
       "3  [0, 3084, 6, 51, 101, 7, 905, 65, 526, 900, 13...   \n",
       "4  [0, 2895, 9930, 2854, 19, 47, 6, 53, 7, 6225, ...   \n",
       "\n",
       "                                  attention_mask_aug  \n",
       "0  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "1  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "2  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "3  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "4  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_pickle('./data/labeled/final_annotated_data_roberta.pickle')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf = TfidfVectorizer(sublinear_tf=True, min_df=2,\n",
    "                        ngram_range=(1, 2), \n",
    "                        stop_words='english')\n",
    "\n",
    "corpus = []\n",
    "corpus_string = []\n",
    "lemmatizer = WordNetLemmatizer() \n",
    "for t in  list(df['comment']):\n",
    "    corpus_string.append(t)\n",
    "    tokens_raw = simple_preprocess(t)\n",
    "    tokens_lemm = []\n",
    "    for token in tokens_raw:\n",
    "        tokens_lemm.append(lemmatizer.lemmatize(token))\n",
    "    corpus.append(tokens_lemm)\n",
    "w2v_model = Word2Vec(corpus, min_count=1)\n",
    "\n",
    "features = tfidf.fit_transform(df.comment).toarray()\n",
    "\n",
    "count = CountVectorizer(max_df=0.8, min_df=2, stop_words='english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentence_vector(tokens):\n",
    "    sums = np.zeros(100)\n",
    "    for word in tokens:\n",
    "        new_word = lemmatizer.lemmatize(word)\n",
    "        sums = sums + np.array(w2v_model[new_word])\n",
    "    return sums\n",
    "\n",
    "def extract_w2v_features(training_data, testing_data):\n",
    "    X_train = []\n",
    "    X_test = []\n",
    "    for sentence in training_data['comment']:\n",
    "        vec = sentence_vector(simple_preprocess(sentence))\n",
    "        X_train.append(vec)\n",
    "    for sentence in testing_data['comment']:\n",
    "        vec = sentence_vector(simple_preprocess(sentence))\n",
    "        X_test.append(vec)\n",
    "    return X_train, X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score, precision_score, roc_auc_score, recall_score\n",
    "def flat_accuracy(pred_flat, labels_flat):\n",
    "    try:\n",
    "        roc = roc_auc_score(pred_flat, labels_flat)\n",
    "    except ValueError:\n",
    "        roc = 0\n",
    "    return f1_score(pred_flat, labels_flat, average='macro'), precision_score(pred_flat, labels_flat, average='macro'), recall_score(pred_flat, labels_flat, average='macro'), roc, np.sum(pred_flat == labels_flat) / len(labels_flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing **aspersion** comments...\n",
      "Processing **civility** comments...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing **vulgarity** comments...\n",
      "Processing **personal_attack** comments...\n",
      "Processing **third_party_attack** comments...\n",
      "Processing **stereotype** comments...\n",
      "Processing **aspersion** comments...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing **civility** comments...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing **vulgarity** comments...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing **personal_attack** comments...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing **third_party_attack** comments...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing **stereotype** comments...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing **aspersion** comments...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing **civility** comments...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing **vulgarity** comments...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing **personal_attack** comments...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing **third_party_attack** comments...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing **stereotype** comments...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "/Users/yujia/opt/anaconda3/lib/python3.9/site-packages/sklearn/linear_model/_sag.py:328: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n"
     ]
    }
   ],
   "source": [
    "tfidfPipeline = Pipeline(steps=[('vect',tfidf), ('clasfi', OneVsRestClassifier(LogisticRegression(solver='sag'), n_jobs=-1))])\n",
    "countPipeline = Pipeline(steps=[('vect',count), ('clasfi', OneVsRestClassifier(LogisticRegression(solver='sag'), n_jobs=-1))])\n",
    "word2vecPipeline = Pipeline(steps=[('clasfi', OneVsRestClassifier(LogisticRegression(solver='sag'), n_jobs=-1))])\n",
    "\n",
    "pipelines = { 'tfidf': tfidfPipeline, 'count': countPipeline, 'word2vec': word2vecPipeline }\n",
    "training_stats = []\n",
    "\n",
    "categories = ['aspersion', 'civility', 'vulgarity', 'personal_attack', 'third_party_attack', 'stereotype']\n",
    "# categories = ['aspersion']\n",
    "for pname, p in pipelines.items(): \n",
    "    for c in categories:\n",
    "        print(f'Processing **{c}** comments...')\n",
    "        \n",
    "        total_folds = 5\n",
    "        current_fold = 0\n",
    "        fold=StratifiedKFold(n_splits=total_folds, shuffle=True, random_state=1000)\n",
    "\n",
    "        for train_index, test_index in fold.split(df, df[c]):\n",
    "            current_fold += 1\n",
    "            train, test = df.iloc[train_index], df.iloc[test_index]\n",
    "            \n",
    "            if pname == 'word2vec':\n",
    "                X_train, X_test = extract_w2v_features(train, test)\n",
    "            else:\n",
    "                X_train, X_test = train['comment'], test['comment']\n",
    "            p.fit(X_train, train[c])\n",
    "            prediction = p.predict(X_test)\n",
    "            f1, precision, recall, auc, accuracy = flat_accuracy(test[c].astype(bool).astype(int), prediction)\n",
    "            \n",
    "            training_stats.append(\n",
    "                {\n",
    "                    'category': c,\n",
    "                    'Valid. Accur.': accuracy,\n",
    "                    'f1_score' : f1,\n",
    "                    'recall_score': recall,\n",
    "                    'precision_score': precision,\n",
    "                    'AUC_score': auc,\n",
    "                    'fold' : current_fold,\n",
    "                    'model': pname\n",
    "                }\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>Valid. Accur.</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>recall_score</th>\n",
       "      <th>precision_score</th>\n",
       "      <th>AUC_score</th>\n",
       "      <th>fold</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aspersion</td>\n",
       "      <td>0.900990</td>\n",
       "      <td>0.473958</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.450495</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1</td>\n",
       "      <td>tfidf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aspersion</td>\n",
       "      <td>0.900990</td>\n",
       "      <td>0.473958</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.450495</td>\n",
       "      <td>0.5</td>\n",
       "      <td>2</td>\n",
       "      <td>tfidf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>aspersion</td>\n",
       "      <td>0.900990</td>\n",
       "      <td>0.473958</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.450495</td>\n",
       "      <td>0.5</td>\n",
       "      <td>3</td>\n",
       "      <td>tfidf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>aspersion</td>\n",
       "      <td>0.899340</td>\n",
       "      <td>0.473501</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.449670</td>\n",
       "      <td>0.5</td>\n",
       "      <td>4</td>\n",
       "      <td>tfidf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>aspersion</td>\n",
       "      <td>0.899340</td>\n",
       "      <td>0.473501</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.449670</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5</td>\n",
       "      <td>tfidf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>stereotype</td>\n",
       "      <td>0.948845</td>\n",
       "      <td>0.486876</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.474422</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1</td>\n",
       "      <td>word2vec</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>stereotype</td>\n",
       "      <td>0.948845</td>\n",
       "      <td>0.486876</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.474422</td>\n",
       "      <td>0.5</td>\n",
       "      <td>2</td>\n",
       "      <td>word2vec</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>stereotype</td>\n",
       "      <td>0.948845</td>\n",
       "      <td>0.486876</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.474422</td>\n",
       "      <td>0.5</td>\n",
       "      <td>3</td>\n",
       "      <td>word2vec</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>stereotype</td>\n",
       "      <td>0.947195</td>\n",
       "      <td>0.486441</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.473597</td>\n",
       "      <td>0.5</td>\n",
       "      <td>4</td>\n",
       "      <td>word2vec</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>stereotype</td>\n",
       "      <td>0.947195</td>\n",
       "      <td>0.486441</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.473597</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5</td>\n",
       "      <td>word2vec</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>90 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      category  Valid. Accur.  f1_score  recall_score  precision_score  \\\n",
       "0    aspersion       0.900990  0.473958           0.5         0.450495   \n",
       "1    aspersion       0.900990  0.473958           0.5         0.450495   \n",
       "2    aspersion       0.900990  0.473958           0.5         0.450495   \n",
       "3    aspersion       0.899340  0.473501           0.5         0.449670   \n",
       "4    aspersion       0.899340  0.473501           0.5         0.449670   \n",
       "..         ...            ...       ...           ...              ...   \n",
       "85  stereotype       0.948845  0.486876           0.5         0.474422   \n",
       "86  stereotype       0.948845  0.486876           0.5         0.474422   \n",
       "87  stereotype       0.948845  0.486876           0.5         0.474422   \n",
       "88  stereotype       0.947195  0.486441           0.5         0.473597   \n",
       "89  stereotype       0.947195  0.486441           0.5         0.473597   \n",
       "\n",
       "    AUC_score  fold     model  \n",
       "0         0.5     1     tfidf  \n",
       "1         0.5     2     tfidf  \n",
       "2         0.5     3     tfidf  \n",
       "3         0.5     4     tfidf  \n",
       "4         0.5     5     tfidf  \n",
       "..        ...   ...       ...  \n",
       "85        0.5     1  word2vec  \n",
       "86        0.5     2  word2vec  \n",
       "87        0.5     3  word2vec  \n",
       "88        0.5     4  word2vec  \n",
       "89        0.5     5  word2vec  \n",
       "\n",
       "[90 rows x 8 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stats = pd.DataFrame(data=training_stats)\n",
    "df_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Valid. Accur.</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>recall_score</th>\n",
       "      <th>precision_score</th>\n",
       "      <th>AUC_score</th>\n",
       "      <th>fold</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>0.824202</td>\n",
       "      <td>0.477598</td>\n",
       "      <td>0.479839</td>\n",
       "      <td>0.522881</td>\n",
       "      <td>0.547967</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tfidf</th>\n",
       "      <td>0.833993</td>\n",
       "      <td>0.450182</td>\n",
       "      <td>0.488554</td>\n",
       "      <td>0.495236</td>\n",
       "      <td>0.505458</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word2vec</th>\n",
       "      <td>0.832563</td>\n",
       "      <td>0.452094</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.416282</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Valid. Accur.  f1_score  recall_score  precision_score  AUC_score  \\\n",
       "model                                                                         \n",
       "count          0.824202  0.477598      0.479839         0.522881   0.547967   \n",
       "tfidf          0.833993  0.450182      0.488554         0.495236   0.505458   \n",
       "word2vec       0.832563  0.452094      0.500000         0.416282   0.500000   \n",
       "\n",
       "          fold  \n",
       "model           \n",
       "count      3.0  \n",
       "tfidf      3.0  \n",
       "word2vec   3.0  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stats.groupby('model').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "incivility",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}