apply_new_labels.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob\n",
    "import os\n",
    "import torch.nn.functional as F\n",
    "import torch"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "need to do 20201129_20210201 third party attack\n",
    "20200813_20201025 all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['./data/comments/20200201_20200315_all_comments_with_labels.pickle', './data/comments/20200315_20200528_all_comments_with_labels.pickle', './data/comments/20200528_20200713_all_comments_with_labels.pickle', './data/comments/20200713_20200813_all_comments_with_labels.pickle', './data/comments/20200813_20201025_all_comments_with_labels.pickle', './data/comments/20201025_20201128_all_comments_with_labels.pickle', './data/comments/20201129_20210201_all_comments_with_labels.pickle', './data/comments/all_comments.pickle', './data/comments/all_comments_no_liwc.pickle']\n"
     ]
    }
   ],
   "source": [
    "dir = './data/comments'\n",
    "datasets_dir = \"./data/datasets\"\n",
    "\n",
    "files = sorted(glob.glob(f'{dir}/*.pickle'))\n",
    "\n",
    "print(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./data/comments/20201129_20210201_all_comments_with_labels.pickle\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_utc</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>link_id</th>\n",
       "      <th>id</th>\n",
       "      <th>author</th>\n",
       "      <th>body</th>\n",
       "      <th>subreddit</th>\n",
       "      <th>link</th>\n",
       "      <th>score</th>\n",
       "      <th>author_flair_text</th>\n",
       "      <th>...</th>\n",
       "      <th>social</th>\n",
       "      <th>posemo</th>\n",
       "      <th>negemo</th>\n",
       "      <th>sad</th>\n",
       "      <th>we</th>\n",
       "      <th>anger</th>\n",
       "      <th>swear</th>\n",
       "      <th>i</th>\n",
       "      <th>you</th>\n",
       "      <th>stereotype</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1612153203</td>\n",
       "      <td>t3_l9gngx</td>\n",
       "      <td>t3_l9gngx</td>\n",
       "      <td>glk8hec</td>\n",
       "      <td>runrain</td>\n",
       "      <td>Lol not even. Where I heard about it most was ...</td>\n",
       "      <td>AskThe_Donald</td>\n",
       "      <td>/r/AskThe_Donald/comments/l9gngx/doj_arrest_re...</td>\n",
       "      <td>1</td>\n",
       "      <td>:T: NOVICE</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.234234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1612152460</td>\n",
       "      <td>t3_l9t1ha</td>\n",
       "      <td>t3_l9t1ha</td>\n",
       "      <td>glk757a</td>\n",
       "      <td>SilverHerfer</td>\n",
       "      <td>To paraphrase one of the great conservatives o...</td>\n",
       "      <td>AskThe_Donald</td>\n",
       "      <td>/r/AskThe_Donald/comments/l9t1ha/we_definitely...</td>\n",
       "      <td>3</td>\n",
       "      <td>:T: NOVICE</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.536770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1612151108</td>\n",
       "      <td>t1_glit0la</td>\n",
       "      <td>t3_l9lonl</td>\n",
       "      <td>glk4jhg</td>\n",
       "      <td>Psychowitz</td>\n",
       "      <td>Like coding.</td>\n",
       "      <td>AskThe_Donald</td>\n",
       "      <td>/r/AskThe_Donald/comments/l9lonl/keystone_pipe...</td>\n",
       "      <td>0</td>\n",
       "      <td>:T: NOVICE</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.288935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1612150714</td>\n",
       "      <td>t3_l9t1ha</td>\n",
       "      <td>t3_l9t1ha</td>\n",
       "      <td>glk3rkp</td>\n",
       "      <td>Salmankhan42069</td>\n",
       "      <td>Wait, this is a meme?</td>\n",
       "      <td>AskThe_Donald</td>\n",
       "      <td>/r/AskThe_Donald/comments/l9t1ha/we_definitely...</td>\n",
       "      <td>-1</td>\n",
       "      <td>:T: NOVICE</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.249884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1612150618</td>\n",
       "      <td>t3_l9t1ha</td>\n",
       "      <td>t3_l9t1ha</td>\n",
       "      <td>glk3kvz</td>\n",
       "      <td>Pile_of_Walthers</td>\n",
       "      <td>Which one is more useful?</td>\n",
       "      <td>AskThe_Donald</td>\n",
       "      <td>/r/AskThe_Donald/comments/l9t1ha/we_definitely...</td>\n",
       "      <td>2</td>\n",
       "      <td>:T: NOVICE</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.308906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7953553</th>\n",
       "      <td>1606658031</td>\n",
       "      <td>t1_ge13eq1</td>\n",
       "      <td>t3_k37m33</td>\n",
       "      <td>ge15l1b</td>\n",
       "      <td>The_Countryof_Poland</td>\n",
       "      <td>hmmm you’re right. the deep state goes even de...</td>\n",
       "      <td>VoteDEM</td>\n",
       "      <td>/r/VoteDEM/comments/k37m33/daily_discussion_th...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.495541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7953555</th>\n",
       "      <td>1606657109</td>\n",
       "      <td>t3_k37m33</td>\n",
       "      <td>t3_k37m33</td>\n",
       "      <td>ge13eq1</td>\n",
       "      <td>arthurpenhaligon</td>\n",
       "      <td>Trump has lost essentially every lawsuit in th...</td>\n",
       "      <td>VoteDEM</td>\n",
       "      <td>/r/VoteDEM/comments/k37m33/daily_discussion_th...</td>\n",
       "      <td>1</td>\n",
       "      <td>:Missouri: Missouri</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.332276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7953561</th>\n",
       "      <td>1606653320</td>\n",
       "      <td>t3_k37m33</td>\n",
       "      <td>t3_k37m33</td>\n",
       "      <td>ge0v2ee</td>\n",
       "      <td>screen317</td>\n",
       "      <td>Last night we passed 20,000 VoteDEM members! C...</td>\n",
       "      <td>VoteDEM</td>\n",
       "      <td>/r/VoteDEM/comments/k37m33/daily_discussion_th...</td>\n",
       "      <td>1</td>\n",
       "      <td>:New_Jersey: NJ-12</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.401069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7953562</th>\n",
       "      <td>1606653034</td>\n",
       "      <td>t3_k37m33</td>\n",
       "      <td>t3_k37m33</td>\n",
       "      <td>ge0uf7z</td>\n",
       "      <td>Jeffery_G</td>\n",
       "      <td>The wife and her ladies group produced 200+ po...</td>\n",
       "      <td>VoteDEM</td>\n",
       "      <td>/r/VoteDEM/comments/k37m33/daily_discussion_th...</td>\n",
       "      <td>1</td>\n",
       "      <td>:Georgia: Georgia</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.289892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7953565</th>\n",
       "      <td>1606651495</td>\n",
       "      <td>t3_k37m33</td>\n",
       "      <td>t3_k37m33</td>\n",
       "      <td>ge0r9un</td>\n",
       "      <td>acroporaguardian</td>\n",
       "      <td>I live in GA, and if you really want to help u...</td>\n",
       "      <td>VoteDEM</td>\n",
       "      <td>/r/VoteDEM/comments/k37m33/daily_discussion_th...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.265009</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6786187 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         created_utc   parent_id    link_id       id                author  \\\n",
       "1         1612153203   t3_l9gngx  t3_l9gngx  glk8hec               runrain   \n",
       "2         1612152460   t3_l9t1ha  t3_l9t1ha  glk757a          SilverHerfer   \n",
       "4         1612151108  t1_glit0la  t3_l9lonl  glk4jhg            Psychowitz   \n",
       "6         1612150714   t3_l9t1ha  t3_l9t1ha  glk3rkp       Salmankhan42069   \n",
       "9         1612150618   t3_l9t1ha  t3_l9t1ha  glk3kvz      Pile_of_Walthers   \n",
       "...              ...         ...        ...      ...                   ...   \n",
       "7953553   1606658031  t1_ge13eq1  t3_k37m33  ge15l1b  The_Countryof_Poland   \n",
       "7953555   1606657109   t3_k37m33  t3_k37m33  ge13eq1      arthurpenhaligon   \n",
       "7953561   1606653320   t3_k37m33  t3_k37m33  ge0v2ee             screen317   \n",
       "7953562   1606653034   t3_k37m33  t3_k37m33  ge0uf7z             Jeffery_G   \n",
       "7953565   1606651495   t3_k37m33  t3_k37m33  ge0r9un      acroporaguardian   \n",
       "\n",
       "                                                      body      subreddit  \\\n",
       "1        Lol not even. Where I heard about it most was ...  AskThe_Donald   \n",
       "2        To paraphrase one of the great conservatives o...  AskThe_Donald   \n",
       "4                                             Like coding.  AskThe_Donald   \n",
       "6                                    Wait, this is a meme?  AskThe_Donald   \n",
       "9                                Which one is more useful?  AskThe_Donald   \n",
       "...                                                    ...            ...   \n",
       "7953553  hmmm you’re right. the deep state goes even de...        VoteDEM   \n",
       "7953555  Trump has lost essentially every lawsuit in th...        VoteDEM   \n",
       "7953561  Last night we passed 20,000 VoteDEM members! C...        VoteDEM   \n",
       "7953562  The wife and her ladies group produced 200+ po...        VoteDEM   \n",
       "7953565  I live in GA, and if you really want to help u...        VoteDEM   \n",
       "\n",
       "                                                      link  score  \\\n",
       "1        /r/AskThe_Donald/comments/l9gngx/doj_arrest_re...      1   \n",
       "2        /r/AskThe_Donald/comments/l9t1ha/we_definitely...      3   \n",
       "4        /r/AskThe_Donald/comments/l9lonl/keystone_pipe...      0   \n",
       "6        /r/AskThe_Donald/comments/l9t1ha/we_definitely...     -1   \n",
       "9        /r/AskThe_Donald/comments/l9t1ha/we_definitely...      2   \n",
       "...                                                    ...    ...   \n",
       "7953553  /r/VoteDEM/comments/k37m33/daily_discussion_th...      1   \n",
       "7953555  /r/VoteDEM/comments/k37m33/daily_discussion_th...      1   \n",
       "7953561  /r/VoteDEM/comments/k37m33/daily_discussion_th...      1   \n",
       "7953562  /r/VoteDEM/comments/k37m33/daily_discussion_th...      1   \n",
       "7953565  /r/VoteDEM/comments/k37m33/daily_discussion_th...      1   \n",
       "\n",
       "           author_flair_text  ...  social  posemo  negemo  sad  we  anger  \\\n",
       "1                 :T: NOVICE  ...       7       6       3    1   2      0   \n",
       "2                 :T: NOVICE  ...       2       2       0    0   1      0   \n",
       "4                 :T: NOVICE  ...       0       0       0    0   0      0   \n",
       "6                 :T: NOVICE  ...       0       0       0    0   0      0   \n",
       "9                 :T: NOVICE  ...       0       1       0    0   0      0   \n",
       "...                      ...  ...     ...     ...     ...  ...  ..    ...   \n",
       "7953553                  NaN  ...       1       0       0    0   0      0   \n",
       "7953555  :Missouri: Missouri  ...       3       1       3    1   0      0   \n",
       "7953561   :New_Jersey: NJ-12  ...       2       0       0    0   1      0   \n",
       "7953562    :Georgia: Georgia  ...       4       2       0    0   0      0   \n",
       "7953565                  NaN  ...       8       3       1    0   1      1   \n",
       "\n",
       "         swear  i  you  stereotype  \n",
       "1            0  0    0    0.234234  \n",
       "2            0  0    0    0.536770  \n",
       "4            0  0    0    0.288935  \n",
       "6            0  0    0    0.249884  \n",
       "9            0  0    0    0.308906  \n",
       "...        ... ..  ...         ...  \n",
       "7953553      0  1    1    0.495541  \n",
       "7953555      0  0    0    0.332276  \n",
       "7953561      0  0    0    0.401069  \n",
       "7953562      0  0    0    0.289892  \n",
       "7953565      0  0    3    0.265009  \n",
       "\n",
       "[6786187 rows x 29 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fname = files[6]\n",
    "date = fname.split('/')[-1].split('_all_comments')[0]\n",
    "print(fname)\n",
    "df = pd.read_pickle(fname)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['created_utc', 'parent_id', 'link_id', 'id', 'author', 'body',\n",
       "       'subreddit', 'link', 'score', 'author_flair_text', 'vulgarity',\n",
       "       'personal_attacks', 'incivility', 'third_party_attacks', 'aspersion',\n",
       "       'comment_len', 'compound_sentiment', 'categories', 'negate', 'social',\n",
       "       'posemo', 'negemo', 'sad', 'we', 'anger', 'swear', 'i', 'you',\n",
       "       'stereotype'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "aspersion\n",
      "\n",
      "no dataset under path data/labels/20201129_20210201_all_comments_with_labels_roberta/aspersion.npy\n",
      "_first\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_first_roberta/aspersion.npy\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yujia/opt/anaconda3/envs/incivility/lib/python3.7/site-packages/ipykernel_launcher.py:13: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  del sys.path[0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3000000\n",
      "_second\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_second_roberta/aspersion.npy\n",
      "2000000\n",
      "_third\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_third_roberta/aspersion.npy\n",
      "1786187\n",
      "civility\n",
      "\n",
      "no dataset under path data/labels/20201129_20210201_all_comments_with_labels_roberta/civility.npy\n",
      "_first\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_first_roberta/civility.npy\n",
      "3000000\n",
      "_second\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_second_roberta/civility.npy\n",
      "2000000\n",
      "_third\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_third_roberta/civility.npy\n",
      "1786187\n",
      "vulgarity\n",
      "\n",
      "no dataset under path data/labels/20201129_20210201_all_comments_with_labels_roberta/vulgarity.npy\n",
      "_first\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_first_roberta/vulgarity.npy\n",
      "3000000\n",
      "_second\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_second_roberta/vulgarity.npy\n",
      "2000000\n",
      "_third\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_third_roberta/vulgarity.npy\n",
      "1786187\n",
      "personal_attack\n",
      "\n",
      "no dataset under path data/labels/20201129_20210201_all_comments_with_labels_roberta/personal_attack.npy\n",
      "_first\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_first_roberta/personal_attack.npy\n",
      "3000000\n",
      "_second\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_second_roberta/personal_attack.npy\n",
      "2000000\n",
      "_third\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_third_roberta/personal_attack.npy\n",
      "1786187\n",
      "third_party_attack\n",
      "\n",
      "no dataset under path data/labels/20201129_20210201_all_comments_with_labels_roberta/third_party_attack.npy\n",
      "_first\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_first_roberta/third_party_attack.npy\n",
      "3000000\n",
      "_second\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_second_roberta/third_party_attack.npy\n",
      "2000000\n",
      "_third\n",
      "there exists dataset under path data/labels/20201129_20210201_all_comments_with_labels_third_roberta/third_party_attack.npy\n",
      "94720\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Length of values (5094720) does not match length of index (6786187)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/1d/mj1m58c50b790y544kd1bflm0000gn/T/ipykernel_82959/388444805.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"no dataset under path \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m     \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/incivility/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__setitem__\u001b[0;34m(self, key, value)\u001b[0m\n\u001b[1;32m   3610\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3611\u001b[0m             \u001b[0;31m# set column\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3612\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_item\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3613\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3614\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_setitem_slice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mslice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/incivility/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_set_item\u001b[0;34m(self, key, value)\u001b[0m\n\u001b[1;32m   3782\u001b[0m         \u001b[0mensure\u001b[0m \u001b[0mhomogeneity\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3783\u001b[0m         \"\"\"\n\u001b[0;32m-> 3784\u001b[0;31m         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sanitize_column\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3785\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3786\u001b[0m         if (\n",
      "\u001b[0;32m~/opt/anaconda3/envs/incivility/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_sanitize_column\u001b[0;34m(self, value)\u001b[0m\n\u001b[1;32m   4507\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4508\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mis_list_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4509\u001b[0;31m             \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequire_length_match\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4510\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0msanitize_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_2d\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4511\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/incivility/lib/python3.7/site-packages/pandas/core/common.py\u001b[0m in \u001b[0;36mrequire_length_match\u001b[0;34m(data, index)\u001b[0m\n\u001b[1;32m    530\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m         raise ValueError(\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0;34m\"Length of values \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m             \u001b[0;34mf\"({len(data)}) \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m             \u001b[0;34m\"does not match length of index \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Length of values (5094720) does not match length of index (6786187)"
     ]
    }
   ],
   "source": [
    "categories = ['aspersion', 'civility', 'vulgarity', 'personal_attack', 'third_party_attack', 'stereotype']\n",
    "files = ['', '_first', '_second', '_third']\n",
    "for c in categories:\n",
    "    print(c)\n",
    "    labels = []\n",
    "    for f in files:\n",
    "        print(f)\n",
    "        path = f'data/labels/{date}_all_comments_with_labels{f}_roberta/{c}.npy'\n",
    "        if os.path.isfile(path):\n",
    "            print(\"there exists dataset under path \" + path)\n",
    "            with open(path, 'rb') as f:\n",
    "                labels_dict = np.load(f, allow_pickle=True)\n",
    "                acts = (F.softmax(torch.tensor(np.stack(labels_dict)))).cpu().detach().numpy()[:,1]\n",
    "                print(len(acts))\n",
    "                labels.extend(acts)\n",
    "        else:\n",
    "            print(\"no dataset under path \" + path)\n",
    "    df[c] = labels\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle(f'data/roberta/{date}.pickle')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "incivility",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}