transform_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yujia/miniconda3/envs/incivility/lib/python3.7/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "2023-04-11 01:47:24.997068: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-04-11 01:47:25.888828: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2023-04-11 01:47:25.888927: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2023-04-11 01:47:25.888938: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob\n",
    "from datasets import Dataset\n",
    "from transformers import AutoTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['./data/comments/20200201_20200315_all_comments_with_labels.pickle', './data/comments/20200201_20200315_all_comments_with_labels_dup.pickle', './data/comments/20200315_20200528_all_comments_with_labels.pickle', './data/comments/20200528_20200713_all_comments_with_labels.pickle', './data/comments/20200713_20200813_all_comments_roberta.pickle', './data/comments/20200713_20200813_all_comments_with_labels.pickle', './data/comments/20200813_20201025_all_comments_with_labels.pickle', './data/comments/20201025_20201128_all_comments_with_labels.pickle', './data/comments/20201129_20210201_all_comments_with_labels.pickle']\n"
     ]
    }
   ],
   "source": [
    "dir = './data/comments'\n",
    "datasets_dir = \"./data/datasets\"\n",
    "\n",
    "files = sorted(glob.glob(f'{dir}/*.pickle'))\n",
    "\n",
    "print(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20200315_20200528_all_comments_with_labels\n"
     ]
    }
   ],
   "source": [
    "# for f in files[2:3]:\n",
    "f = files[2]\n",
    "head, tail = os.path.split(f)\n",
    "fname = tail.split('.')[0]\n",
    "print(fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6310154\n",
      "6310154\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_pickle(f)\n",
    "print(len(df))\n",
    "df = df.dropna(subset=['body', 'id'])\n",
    "df = df[(df['author'] != '[deleted]') & (df['author'] != '[removed]') & (df['body'] != '[deleted]') & (df['body'] != '[removed]')]\n",
    "print(len(df))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.iloc[:3000000,:]\n",
    "dataset = Dataset.from_pandas(df[['id', 'body']])\n",
    "del df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_path = os.path.join(datasets_dir, fname+\"_first_roberta\")\n",
    "dataset = Dataset.load_from_disk(dataset_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3311/3311 [08:35<00:00,  6.43ba/s]\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"roberta-base\")\n",
    "dataset = dataset.map(lambda e: tokenizer(e['body'], truncation=True, padding='max_length'), batched=True)\n",
    "dataset.save_to_disk(dataset_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from resource import getrusage, RUSAGE_SELF\n",
    "# print(getrusage(RUSAGE_SELF).ru_maxrss)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "incivility",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}