prepare_VDJ-New.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1dda4b24-cb77-42ec-831d-8e1c8531dc35",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "from datetime import datetime\n",
    "from pathlib import Path\n",
    "from typing import List, Tuple, Dict\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from prettytable import PrettyTable\n",
    "\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cb95f381-3a9c-41f3-83b4-ae2bf2970261",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-06-13 08:13:54\n"
     ]
    }
   ],
   "source": [
    "print(datetime.today().strftime(\"%Y-%m-%d %H:%M:%S\"))\n",
    "today = datetime.today().strftime(\"%Y%m%d%H%M%S\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bcd4aa16-df2f-4420-af5c-22e10943699d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+-------+-------+\n",
      "|           | Train | Valid |\n",
      "+-----------+-------+-------+\n",
      "| Data size | 10000 |  2000 |\n",
      "+-----------+-------+-------+\n"
     ]
    }
   ],
   "source": [
    "test_data = [['', 'Train', 'Valid'],\n",
    "             ['Data size', 10000, 2000]]\n",
    "\n",
    "x = PrettyTable()\n",
    "x.field_names = ['', 'Train', 'Valid']\n",
    "x.add_row(['Data size', 10000, 2000])\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6ab0ef6-2bb2-4f42-8310-863853320b0e",
   "metadata": {},
   "source": [
    "## Create docker container\n",
    "\n",
    "```bash\n",
    "docker run -ti --runtime=runc --name immcantation_cm --workdir /data -v /data/personal/jiewen.du/CM_new:/data immcantation/suite:4.3.0 /bin/bash\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0ccbaefc-7386-47b1-8e0f-876e32e53a72",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ROOT = Path('/data/personal/jiewen.du/CM_new')\n",
    "SCRIPT = ROOT / 'script'\n",
    "DATA_HOME = ROOT / 'data'\n",
    "\n",
    "DATA_DIR = DATA_HOME / 'ymc_copy'\n",
    "CELL_PASS_DIR = DATA_HOME / 'cell_pass'\n",
    "FMT7_DIR = DATA_HOME / 'fmt7'\n",
    "FMT19_DIR = DATA_HOME / 'fmt19'\n",
    "\n",
    "WORK_DIR = Path('/data')\n",
    "CONTAINER = 'immcantation_cm'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5edb5e64-8920-478b-a5a8-73156d8163ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "CURRENT_DATASET = ['7_new']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e46d0e1b-ee81-4b57-a510-0dbb93a7f36e",
   "metadata": {},
   "outputs": [],
   "source": [
    "statics_header = ['dataset', 'All cells', 'Non-B cells', 'No-Germlines', 'B cells', \n",
    "                  'B cells train', 'B cells train clone_pass', 'unique NT train clone_pass', \n",
    "                  'B cells train clone_pass_germ_pass', 'unique NT train clone_pass_germ_pass', \n",
    "                  'B cells valid', 'B cells valid clone_pass', 'unique NT valid clone_pass', \n",
    "                  'B cells valid clone_pass_germ_pass', 'unique NT valid clone_pass_germ_pass'\n",
    "                 ]\n",
    "\n",
    "statistic = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "547cd909-251a-470c-90b8-5355d68a04e1",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Cell-pass\n",
    "过滤掉非B cells, CDR3为空，非productive BCR的细胞数据\n",
    "\n",
    "备注：\n",
    "1. CM0~CM8: CM*bcr.samples.IGHV.igblast.tsv均为fmt19格式\n",
    "2. CM9, CM10: CM*bcr.samples.IGHV.igblast.tsv均为fmt7格式  \n",
    "\n",
    "Source: CM00   | Cells: 56762  | Non-Bcells: 2099   | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 54663  \n",
    "Source: CM1    | Cells: 32201  | Non-Bcells: 4365   | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 27836  \n",
    "Source: CM2    | Cells: 32651  | Non-Bcells: 6202   | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 26449  \n",
    "Source: CM3    | Cells: 53432  | Non-Bcells: 9572   | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 43860  \n",
    "Source: CM4    | Cells: 90319  | Non-Bcells: 2402   | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 87917  \n",
    "Source: CM5    | Cells: 13781  | Non-Bcells: 266    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 13515  \n",
    "Source: CM5_2  | Cells: 144084 | Non-Bcells: 781    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 143303  \n",
    "Source: CM6    | Cells: 53426  | Non-Bcells: 1286   | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 52140  \n",
    "Source: CM7    | Cells: 30031  | Non-Bcells: 967    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 29064  \n",
    "Source: CM8    | Cells: 66527  | Non-Bcells: 1694   | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 64833  \n",
    "Source: CM9    | Cells: 103685 | Non-Bcells: 317    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 103368  \n",
    "Source: CM10   | Cells: 48743  | Non-Bcells: 340    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 48403    \n",
    "Source: CM11   | Cells: 204154 | Non-Bcells: 723    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 203431  \n",
    "Source: CM12   | Cells: 138900 | Non-Bcells: 726    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 138174  \n",
    "Source: CM13   | Cells: 153536 | Non-Bcells: 788    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 152748  \n",
    "Source: CM15   | Cells: 152141 | Non-Bcells: 578    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 151563  \n",
    "Source: CM20   | Cells: 156568 | Non-Bcells: 278    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 156290  \n",
    "Source: CM21   | Cells: 130004 | Non-Bcells: 286    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 129718   \n",
    "Source: CM22   | Cells: 137704 | Non-Bcells: 755    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 136949  \n",
    "Source: CM28   | Cells: 73189  | Non-Bcells: 142    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 73047   \n",
    "Source: CM29   | Cells: 152204 | Non-Bcells: 197    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 152007  \n",
    "Source: CM32   | Cells: 135856 | Non-Bcells: 575    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 135281  \n",
    "Source: CM36   | Cells: 200225 | Non-Bcells: 841    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 199384  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "ee1f9a1c-722d-4a53-8bad-fe9a79e7e7a3",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['CM00', '21440/54663', '21440/54663', '2268/3635', '2268/3635'],\n",
       " ['CM1', '15618/24200', '15618/24200', '2268/3635', '2268/3635'],\n",
       " ['CM2', '17525/23363', '17525/23363', '2404/3078', '2404/3078'],\n",
       " ['CM3', '24016/43190', '24016/43190', '624/669', '624/669'],\n",
       " ['CM4', '46610/86160', '46610/86160', '1134/1754', '1134/1754'],\n",
       " ['CM5', '6167/9736', '6167/9736', '1099/3778', '1099/3778'],\n",
       " ['CM6', '22033/51771', '22033/51771', '224/366', '224/366'],\n",
       " ['CM7', '17321/29044', '17321/29044', '19/19', '19/19'],\n",
       " ['CM8', '27507/64371', '27507/64371', '276/460', '276/460'],\n",
       " ['CM9', '61346/102588', '61346/102588', '564/662', '564/662'],\n",
       " ['CM10', '28987/47865', '28987/47865', '331/371', '331/371']]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "txt ='''Source: CM00   | Cells: 56762  | Non-Bcells: 2099   | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 54663  \n",
    "Source: CM1    | Cells: 32201  | Non-Bcells: 4365   | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 27836  \n",
    "Source: CM2    | Cells: 32651  | Non-Bcells: 6202   | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 26449  \n",
    "Source: CM3    | Cells: 53432  | Non-Bcells: 9572   | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 43860  \n",
    "Source: CM4    | Cells: 90319  | Non-Bcells: 2402   | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 87917  \n",
    "Source: CM5    | Cells: 13781  | Non-Bcells: 266    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 13515  \n",
    "Source: CM6    | Cells: 53426  | Non-Bcells: 1286   | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 52140  \n",
    "Source: CM7    | Cells: 30031  | Non-Bcells: 967    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 29064  \n",
    "Source: CM8    | Cells: 66527  | Non-Bcells: 1694   | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 64833  \n",
    "Source: CM9    | Cells: 103685 | Non-Bcells: 317    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 103368  \n",
    "Source: CM10   | Cells: 48743  | Non-Bcells: 340    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 48403  '''\n",
    "\n",
    "txt2 = '''Source: CM00 | Train clone: 21440/54663  | Train germ: 21440/54663  | Valid germ: 2268/3635   | Valid germ: 2268/3635\n",
    "Source: CM1 | Train clone: 15618/24200  | Train germ: 15618/24200  | Valid germ: 2268/3635   | Valid germ: 2268/3635  \n",
    "Source: CM2 | Train clone: 17525/23363  | Train germ: 17525/23363  | Valid germ: 2404/3078   | Valid germ: 2404/3078  \n",
    "Source: CM3 | Train clone: 24016/43190  | Train germ: 24016/43190  | Valid germ: 624/669    | Valid germ: 624/669   \n",
    "Source: CM4 | Train clone: 46610/86160  | Train germ: 46610/86160  | Valid germ: 1134/1754   | Valid germ: 1134/1754  \n",
    "Source: CM5 | Train clone: 6167/9736   | Train germ: 6167/9736   | Valid germ: 1099/3778   | Valid germ: 1099/3778  \n",
    "Source: CM6 | Train clone: 22033/51771  | Train germ: 22033/51771  | Valid germ: 224/366    | Valid germ: 224/366   \n",
    "Source: CM7 | Train clone: 17321/29044  | Train germ: 17321/29044  | Valid germ: 19/19     | Valid germ: 19/19    \n",
    "Source: CM8 | Train clone: 27507/64371  | Train germ: 27507/64371  | Valid germ: 276/460    | Valid germ: 276/460   \n",
    "Source: CM9 | Train clone: 61346/102588 | Train germ: 61346/102588 | Valid germ: 564/662    | Valid germ: 564/662   \n",
    "Source: CM10 | Train clone: 28987/47865  | Train germ: 28987/47865  | Valid germ: 331/371    | Valid germ: 331/371   '''\n",
    "\n",
    "data = []\n",
    "for row in txt2.split('\\n'):\n",
    "    sub_data = []\n",
    "    for item in row.split(' | '):\n",
    "        sub_data.append(item.split(':')[-1].strip())\n",
    "    data.append(sub_data)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "1a5a564b-2e1f-46a3-822a-004c84a7d9d7",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('table.csv', 'w') as f:\n",
    "    f_csv = csv.writer(f)\n",
    "    f_csv.writerows(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "c8a3b6e1-0b27-4f41-8f5a-040b7d6a2c9f",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataset = 'CM3'\n",
    "df0 = pd.read_csv(DATA_DIR/f'{dataset}.TSM_VS_VMM_all.bcr.samples.IGHV.igblast.tsv', sep='\\t')\n",
    "df = df0.loc[df0['celltype'].isin(BTYPE.keys()), :].copy(True)\n",
    "df1 = df0.loc[~df0['celltype'].isin(BTYPE.keys()), :].copy(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "49fd1f54-cbed-4fc1-b8c8-4c7597cf02e5",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stim</th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>rev_comp</th>\n",
       "      <th>productive</th>\n",
       "      <th>v_call</th>\n",
       "      <th>d_call</th>\n",
       "      <th>j_call</th>\n",
       "      <th>sequence_alignment</th>\n",
       "      <th>germline_alignment</th>\n",
       "      <th>...</th>\n",
       "      <th>c_call</th>\n",
       "      <th>consensus_count</th>\n",
       "      <th>umi_count</th>\n",
       "      <th>v_call_10x</th>\n",
       "      <th>d_call_10x</th>\n",
       "      <th>j_call_10x</th>\n",
       "      <th>junction_10x</th>\n",
       "      <th>junction_10x_aa</th>\n",
       "      <th>barcode</th>\n",
       "      <th>celltype</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>CM003-8-1</td>\n",
       "      <td>CAGAATCCAAGTCTAC-1.CM003-8-1</td>\n",
       "      <td>TTTTTCTTATATGGGGAGCTCTCAGAGAGGTGCCTTAGCCCTGGAT...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>IGHV3-48*03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IGHJ4*02</td>\n",
       "      <td>......CAACTGGTGGAGTCTGGGGGA...GGCTTGGTACAGCCTG...</td>\n",
       "      <td>GAGGTGCAGCTGGTGGAGTCTGGGGGA...GGCTTGGTACAGCCTG...</td>\n",
       "      <td>...</td>\n",
       "      <td>mus-IGHG2B</td>\n",
       "      <td>120223</td>\n",
       "      <td>11424</td>\n",
       "      <td>hsa-IGHV3-48</td>\n",
       "      <td>mus-IGHD3-1</td>\n",
       "      <td>hsa-IGHJ4</td>\n",
       "      <td>TGTGCGAGAGGCGCGGACTACTGG</td>\n",
       "      <td>CARGADYW</td>\n",
       "      <td>CAGAATCCAAGTCTAC-1.CM003-8-1</td>\n",
       "      <td>Monocytes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>CM003-3-2</td>\n",
       "      <td>AGAGTGGTCCAAATGC-1.CM003-3-2</td>\n",
       "      <td>AATGCGGCACCCTCTTTTCTTATATGGGGGACTCCTGTGCCCCACC...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>IGHV2-26*01</td>\n",
       "      <td>IGHD6-19*01</td>\n",
       "      <td>IGHJ1*01</td>\n",
       "      <td>CAGGTCACCTTGAAGGAGTCTGGTCCT...GTGCTGGTGAAACCCA...</td>\n",
       "      <td>CAGGTCACCTTGAAGGAGTCTGGTCCT...GTGCTGGTGAAACCCA...</td>\n",
       "      <td>...</td>\n",
       "      <td>mus-IGHG1</td>\n",
       "      <td>110962</td>\n",
       "      <td>10858</td>\n",
       "      <td>hsa-IGHV2-26</td>\n",
       "      <td>hsa-IGHD6-19</td>\n",
       "      <td>hsa-IGHJ1</td>\n",
       "      <td>TGTGCACGGTTAGCAGTGGCTGGTACGGCTGAATACTTCCAGCACTGG</td>\n",
       "      <td>CARLAVAGTAEYFQHW</td>\n",
       "      <td>AGAGTGGTCCAAATGC-1.CM003-3-2</td>\n",
       "      <td>Monocytes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>CM003-8-1</td>\n",
       "      <td>CATGCCTCACTTACGA-1.CM003-8-1</td>\n",
       "      <td>AGGTGTATTGGTTTCTTATATGGGGAGTGCTTTCTGAGAGTCATGG...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>IGHV4-34*01</td>\n",
       "      <td>IGHD3-10*01</td>\n",
       "      <td>IGHJ6*02</td>\n",
       "      <td>CAGGTGCAGCTACAGCAGTGGGGCGCA...GGACTGTTGAAGCCTT...</td>\n",
       "      <td>CAGGTGCAGCTACAGCAGTGGGGCGCA...GGACTGTTGAAGCCTT...</td>\n",
       "      <td>...</td>\n",
       "      <td>mus-IGHG2C</td>\n",
       "      <td>126579</td>\n",
       "      <td>10357</td>\n",
       "      <td>hsa-IGHV4-34</td>\n",
       "      <td>hsa-IGHD3-10</td>\n",
       "      <td>hsa-IGHJ6</td>\n",
       "      <td>TGTGCGAGTGGATCACTATGGTTCGGAGAATTAGCCGTCTACTACT...</td>\n",
       "      <td>CASGSLWFGELAVYYYGMDVW</td>\n",
       "      <td>CATGCCTCACTTACGA-1.CM003-8-1</td>\n",
       "      <td>Monocytes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          stim                   sequence_id  \\\n",
       "102  CM003-8-1  CAGAATCCAAGTCTAC-1.CM003-8-1   \n",
       "137  CM003-3-2  AGAGTGGTCCAAATGC-1.CM003-3-2   \n",
       "183  CM003-8-1  CATGCCTCACTTACGA-1.CM003-8-1   \n",
       "\n",
       "                                              sequence  rev_comp  productive  \\\n",
       "102  TTTTTCTTATATGGGGAGCTCTCAGAGAGGTGCCTTAGCCCTGGAT...     False        True   \n",
       "137  AATGCGGCACCCTCTTTTCTTATATGGGGGACTCCTGTGCCCCACC...     False        True   \n",
       "183  AGGTGTATTGGTTTCTTATATGGGGAGTGCTTTCTGAGAGTCATGG...     False        True   \n",
       "\n",
       "          v_call       d_call    j_call  \\\n",
       "102  IGHV3-48*03          NaN  IGHJ4*02   \n",
       "137  IGHV2-26*01  IGHD6-19*01  IGHJ1*01   \n",
       "183  IGHV4-34*01  IGHD3-10*01  IGHJ6*02   \n",
       "\n",
       "                                    sequence_alignment  \\\n",
       "102  ......CAACTGGTGGAGTCTGGGGGA...GGCTTGGTACAGCCTG...   \n",
       "137  CAGGTCACCTTGAAGGAGTCTGGTCCT...GTGCTGGTGAAACCCA...   \n",
       "183  CAGGTGCAGCTACAGCAGTGGGGCGCA...GGACTGTTGAAGCCTT...   \n",
       "\n",
       "                                    germline_alignment  ...      c_call  \\\n",
       "102  GAGGTGCAGCTGGTGGAGTCTGGGGGA...GGCTTGGTACAGCCTG...  ...  mus-IGHG2B   \n",
       "137  CAGGTCACCTTGAAGGAGTCTGGTCCT...GTGCTGGTGAAACCCA...  ...   mus-IGHG1   \n",
       "183  CAGGTGCAGCTACAGCAGTGGGGCGCA...GGACTGTTGAAGCCTT...  ...  mus-IGHG2C   \n",
       "\n",
       "    consensus_count umi_count    v_call_10x    d_call_10x  j_call_10x  \\\n",
       "102          120223     11424  hsa-IGHV3-48   mus-IGHD3-1   hsa-IGHJ4   \n",
       "137          110962     10858  hsa-IGHV2-26  hsa-IGHD6-19   hsa-IGHJ1   \n",
       "183          126579     10357  hsa-IGHV4-34  hsa-IGHD3-10   hsa-IGHJ6   \n",
       "\n",
       "                                          junction_10x        junction_10x_aa  \\\n",
       "102                           TGTGCGAGAGGCGCGGACTACTGG               CARGADYW   \n",
       "137   TGTGCACGGTTAGCAGTGGCTGGTACGGCTGAATACTTCCAGCACTGG       CARLAVAGTAEYFQHW   \n",
       "183  TGTGCGAGTGGATCACTATGGTTCGGAGAATTAGCCGTCTACTACT...  CASGSLWFGELAVYYYGMDVW   \n",
       "\n",
       "                          barcode   celltype  \n",
       "102  CAGAATCCAAGTCTAC-1.CM003-8-1  Monocytes  \n",
       "137  AGAGTGGTCCAAATGC-1.CM003-3-2  Monocytes  \n",
       "183  CATGCCTCACTTACGA-1.CM003-8-1  Monocytes  \n",
       "\n",
       "[3 rows x 60 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8bec7f23-25ae-4a2f-a7f9-4fe8c37395ac",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source: CM7_new | Cells: 116683 | Non-Bcells: 956    | Null CDR3: 0      | Non-productives: 0      | Cell-pass: 115727\n"
     ]
    }
   ],
   "source": [
    "for dataset in [f'CM{i}' for i in CURRENT_DATASET]:\n",
    "    df0 = pd.read_csv(DATA_DIR/f'{dataset}.TSM_VS_VMM_all.bcr.samples.IGHV.igblast.tsv', sep='\\t')\n",
    "\n",
    "    df = df0.loc[df0['celltype'].isin(BTYPE.keys()), :].copy(True)\n",
    "    print('Source: {:<6} | Cells: {:<6} | Non-Bcells: {:<6} '.format(dataset, df0.shape[0], df0.shape[0]-df.shape[0]), end='')\n",
    "    \n",
    "    df.reset_index(drop=True, inplace=True)\n",
    "    df['mice_index'] = df['sequence_id'].apply(lambda x: x.split('.')[-1])\n",
    "\n",
    "    droped_idx1 = df.loc[(df['cdr3']=='')|(df['cdr3']=='None')|(df['cdr3']=='NA')|(df['cdr3']==None)|(df['cdr3'].isna()), :].index\n",
    "    df.drop(index=droped_idx1, inplace=True)\n",
    "    df.reset_index(drop=True, inplace=True)\n",
    "\n",
    "    droped_idx2 = df.loc[df['productive']==False, :].index\n",
    "    df.drop(index=droped_idx2, inplace=True)\n",
    "\n",
    "    # 2: fix wrong dtype\n",
    "    df = fix_wrong_dtype(df)\n",
    "    df = df.where(pd.notna(df), None)\n",
    "\n",
    "    df.to_csv(CELL_PASS_DIR/f'{dataset}.fmt7', sep='\\t', index=None, na_rep='')\n",
    "\n",
    "    with open(CELL_PASS_DIR/f'{dataset}.fasta', 'w') as f:\n",
    "        for row in df.loc[:, ['sequence_id', 'sequence']].itertuples(False):\n",
    "            f.write(f'>{row[0]}\\n')\n",
    "            f.write(f'{row[1]}\\n')\n",
    "    \n",
    "    print('| Null CDR3: {:<6} | Non-productives: {:<6} | Cell-pass: {:<6}'.format(len(droped_idx1), len(droped_idx1), df.shape[0]))\n",
    "    statistic[dataset] = [dataset, df0.shape[0], df0.shape[0]-df.shape[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "96d4f128-2ab7-45af-9204-0ec9c52547b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'CM7_new': ['CM7_new', 116683, 956]}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "statistic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "666ad6de-0f5a-4cef-8091-8f6b75ab172b",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source: CM010  | Cells: 48743  | Cell-pass: 48403  | Valid cells: 371    | Train cells: 48032 \n",
      "Source: CM009  | Cells: 92743  | Cell-pass: 92441  | Valid cells: 663    | Train cells: 91778 \n",
      "Source: CM0    | Cells: 1906   | Cell-pass: 1770  \n",
      "Source: CM005  | Cells: 2005   | Cell-pass: 1963   | Valid cells: 160    | Train cells: 1803  \n",
      "Source: CM006  | Cells: 13157  | Cell-pass: 12783  | Valid cells: 126    | Train cells: 12657 \n",
      "Source: CM004  | Cells: 28339  | Cell-pass: 27679  | Valid cells: 880    | Train cells: 26799 \n",
      "Source: CM002  | Cells: 13310  | Cell-pass: 11992  | Valid cells: 2184   | Train cells: 9808  \n",
      "Source: CM003  | Cells: 20614  | Cell-pass: 18869  | Valid cells: 1129   | Train cells: 17740 \n",
      "Source: CM001  | Cells: 16908  | Cell-pass: 15187  | Valid cells: 1768   | Train cells: 13419 \n",
      "Source: CM00   | Cells: 18576  | Cell-pass: 17967 \n",
      "Source: CM008  | Cells: 14761  | Cell-pass: 14561  | Valid cells: 94     | Train cells: 14467 \n"
     ]
    }
   ],
   "source": [
    "for ifile in DATA_DIR.rglob('*bcr.samples.IGHV.igblast.tsv'):\n",
    "    source = str(ifile.stem).split('_')[0]\n",
    "    has_gap = True\n",
    "    \n",
    "    df0 = pd.read_csv(ifile, sep='\\t')\n",
    "    \n",
    "    # 1: cell-pass, cdr3-pass, is_productive-pass:\n",
    "    df = df0.loc[df0['celltype'].isin(BTYPE.keys()), :].copy(True)\n",
    "    df.reset_index(drop=True, inplace=True)\n",
    "    df['mice_index'] = df['sequence_id'].apply(lambda x: x.split('.')[-1])\n",
    "    \n",
    "    droped_idx = df.loc[(df['cdr3']=='')|(df['cdr3']=='None')|(df['cdr3']=='NA')|(df['cdr3']==None)|(df['cdr3'].isna()), :].index\n",
    "    df.drop(index=droped_idx, inplace=True)\n",
    "    df.reset_index(drop=True, inplace=True)\n",
    "    \n",
    "    droped_idx = df.loc[df['productive']==False, :].index\n",
    "    df.drop(index=droped_idx, inplace=True)\n",
    "    \n",
    "    # 2: fix wrong dtype\n",
    "    df = fix_wrong_dtype(df)\n",
    "    \n",
    "    df = df.where(pd.notna(df), None)\n",
    "    \n",
    "    # 3: split train:valid by mice_index\n",
    "    if source in ['CM0', 'CM00']:\n",
    "        with open(CELL_PASS_DIR/f'{source}_train.fasta', 'w') as f:\n",
    "            for row in df.loc[:, ['sequence_id', 'sequence']].itertuples(False):\n",
    "                f.write(f'>{row[0]}\\n')\n",
    "                f.write(f'{row[1]}\\n')\n",
    "        \n",
    "        df.to_csv(CELL_PASS_DIR/f'{source}_train_yao.fmt19', sep='\\t', index=None, na_rep='')\n",
    "        print('Source: {:<6} | Cells: {:<6} | Cell-pass: {:<6}'.format(source, df0.shape[0], df.shape[0]))\n",
    "        continue\n",
    "    \n",
    "    print('Source: {:<6} | Cells: {:<6d} | Cell-pass: {:<6d} | Valid cells: {:<6d} | Train cells: {:<6d}'.format(source, df0.shape[0], df.shape[0], df_v.shape[0], df_t.shape[0]))\n",
    "    \n",
    "    # 4: save data as fasta format(>sequence_id, sequence)\n",
    "#     with open(DATA_DIR2/f'{source}_train.fasta', 'w') as f:\n",
    "#         for row in df_t.loc[:, ['sequence_id', 'sequence']].itertuples(False):\n",
    "#             f.write(f'>{row[0]}\\n')\n",
    "#             f.write(f'{row[1]}\\n')\n",
    "    \n",
    "#     with open(DATA_DIR2/f'{source}_valid.fasta', 'w') as f:\n",
    "#         for row in df_v.loc[:, ['sequence_id', 'sequence']].itertuples(False):\n",
    "#             f.write(f'>{row[0]}\\n')\n",
    "#             f.write(f'{row[1]}\\n')\n",
    "    \n",
    "    with open(DATA_DIR2/f'{source}_all.fasta', 'w') as f:\n",
    "        for row in df.loc[:, ['sequence_id', 'sequence']].itertuples(False):\n",
    "            f.write(f'>{row[0]}\\n')\n",
    "            f.write(f'{row[1]}\\n')\n",
    "    \n",
    "    # 5: save as tsv: fmt7 or fmt19\n",
    "    if source in ['CM009', 'CM010']:\n",
    "        df.to_csv(CELL_PASS_DIR/f'{source}_all_yao.fmt7', sep='\\t', index=None, na_rep='')\n",
    "    else:\n",
    "        df.to_csv(CELL_PASS_DIR/f'{source}_all_yao.fmt19', sep='\\t', index=None, na_rep='')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba2c428b-185f-43d4-a8c2-47e919a38fd9",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Run igblastn(v1.17.0)\n",
    "进行igblast分析，对BCR序列进行注释"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a68c9363-aab6-4383-9011-dca57bbea737",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Submitted batch job 2\n"
     ]
    }
   ],
   "source": [
    "fasta = CELL_PASS_DIR/'CM7_new.fasta'\n",
    "run_igblast(fasta, fmt=19, threds=50)\n",
    "\n",
    "# fasta = CELL_PASS_DIR/'CM8_new.fasta'\n",
    "# run_igblast(fasta, fmt=19, threds=50)\n",
    "\n",
    "# fasta = CELL_PASS_DIR/'CM40.fasta'\n",
    "# run_igblast(fasta, fmt=19, threds=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a88973f-6e0f-45f0-96cd-11634bfea291",
   "metadata": {},
   "source": [
    "## Make DB\n",
    "将igblast分析文件转为tab分割的文本文件<br>\n",
    "注：只能转换‘-outfmt “7 std qseq sseq btop”’的igblast结果文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8e2890ce-ca20-423e-ba55-c2da988440f5",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Submitted batch job 44195\n"
     ]
    }
   ],
   "source": [
    "db = WORK_DIR / 'data/fmt7/fmt7/CM005_valid.fmt7'\n",
    "fasta = WORK_DIR / 'data/cell_pass/CM005_valid.fasta'\n",
    "make_db(db, fasta, CONTAINER, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3dc6a33e-50f3-4435-a90e-c34a9fc50805",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Submitted batch job 33828\n",
      "Submitted batch job 33829\n",
      "Submitted batch job 33830\n",
      "Submitted batch job 33831\n",
      "Submitted batch job 33832\n",
      "Submitted batch job 33833\n",
      "Submitted batch job 33834\n"
     ]
    }
   ],
   "source": [
    "for i in range(1, 11):\n",
    "    if i in [3, 7, 10]:\n",
    "        continue\n",
    "    db = WORK_DIR / f'data/fmt7/CM00{i}_all.fmt7'\n",
    "    fasta = WORK_DIR / f'data/cell_pass/CM00{i}_all.fasta'\n",
    "    make_db(db, fasta, CONTAINER, True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ca73ff3-d502-4186-bb93-5275f708e750",
   "metadata": {},
   "source": [
    "## Check db\n",
    "有些比对错误，导致germline_alignment为空  \n",
    "如果没有比对错误,则不输出*fixed.tsv文件\n",
    "\n",
    "Source: CM00       | Null germline_alignment: 0      | d_start > d_end: 0      | Passed-cells: 54663  \n",
    "Source: CM1        | Null germline_alignment: 1      | d_start > d_end: 1      | Passed-cells: 27835  \n",
    "Source: CM2        | Null germline_alignment: 8      | d_start > d_end: 8      | Passed-cells: 26441  \n",
    "Source: CM3        | Null germline_alignment: 1      | d_start > d_end: 1      | Passed-cells: 43859  \n",
    "Source: CM4        | Null germline_alignment: 3      | d_start > d_end: 3      | Passed-cells: 87914  \n",
    "Source: CM5        | Null germline_alignment: 1      | d_start > d_end: 1      | Passed-cells: 13514  \n",
    "Source: CM5_2      | Null germline_alignment: 14     | d_start > d_end: 14     | Passed-cells: 143289   \n",
    "Source: CM6        | Null germline_alignment: 3      | d_start > d_end: 3      | Passed-cells: 52137  \n",
    "Source: CM7        | Null germline_alignment: 1      | d_start > d_end: 1      | Passed-cells: 29063  \n",
    "Source: CM8        | Null germline_alignment: 2      | d_start > d_end: 2      | Passed-cells: 64831  \n",
    "Source: CM9        | Null germline_alignment: 118    | d_start > d_end: 118    | Passed-cells: 103250  \n",
    "Source: CM10       | Null germline_alignment: 167    | d_start > d_end: 167    | Passed-cells: 48236  \n",
    "Source: CM11       | Null germline_alignment: 162    | d_start > d_end: 162    | Passed-cells: 203269  \n",
    "Source: CM12       | Null germline_alignment: 0      | d_start > d_end: 0      | Passed-cells: 138174  \n",
    "Source: CM13       | Null germline_alignment: 29     | d_start > d_end: 29     | Passed-cells: 152719  \n",
    "Source: CM15       | Null germline_alignment: 1      | d_start > d_end: 1      | Passed-cells: 151562  \n",
    "Source: CM20       | Null germline_alignment: 0      | d_start > d_end: 0      | Passed-cells: 156290  \n",
    "Source: CM21       | Null germline_alignment: 9      | d_start > d_end: 9      | Passed-cells: 129709  \n",
    "Source: CM22       | Null germline_alignment: 12     | d_start > d_end: 12     | Passed-cells: 136937  \n",
    "Source: CM28       | Null germline_alignment: 3      | d_start > d_end: 3      | Passed-cells: 73044   \n",
    "Source: CM29       | Null germline_alignment: 72     | d_start > d_end: 72     | Passed-cells: 151935  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a9f15129-1c25-471c-9d4c-7bbc3c0bb369",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source: CM7_new    | Null germline_alignment: 3      | d_start > d_end: 3      | Passed-cells: 115724\n",
      "{'CM7_new': ['CM7_new', 116683, 956, 3, 115724]}\n"
     ]
    }
   ],
   "source": [
    "for dataset in [f'CM{i}' for i in CURRENT_DATASET]:\n",
    "    ifile = CELL_PASS_DIR/f'{dataset}.fmt7'\n",
    "    df = pd.read_csv(ifile, sep='\\t')\n",
    "    df = fix_wrong_dtype(df)\n",
    "    \n",
    "    droped_idx1 = df.loc[df['germline_alignment'].isna(),:].index\n",
    "    droped_idx2 = df.loc[df['d_germline_start']>=df['d_germline_end'],:].index\n",
    "    print('Source: {:<10} | Null germline_alignment: {:<6} | d_start > d_end: {:<6} | Passed-cells: {:<6}'.format(dataset, len(droped_idx1), len(droped_idx2), df.shape[0]-len(droped_idx1)))\n",
    "    statistic[dataset].extend([len(droped_idx1), df.shape[0]-len(droped_idx1)])\n",
    "    \n",
    "    if len(droped_idx1)!=0 or len(droped_idx2)!=0:\n",
    "        df.drop(index=droped_idx1,inplace=True)\n",
    "        df.to_csv(CELL_PASS_DIR/f'{dataset}_fixed.tsv', sep='\\t', index=None, na_rep='')\n",
    "\n",
    "print(statistic)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9df16f8-59f3-4b5e-bd92-5a92d9d5587e",
   "metadata": {},
   "source": [
    "## split train, valid\n",
    "\n",
    "Source: CM1    | DB-pass: 27835  | Train cells: 24200  | Valid cells: 3635  \n",
    "Source: CM2    | DB-pass: 26441  | Train cells: 23363  | Valid cells: 3078  \n",
    "Source: CM3    | DB-pass: 43859  | Train cells: 43190  | Valid cells: 669   \n",
    "Source: CM4    | DB-pass: 87914  | Train cells: 86160  | Valid cells: 1754  \n",
    "Source: CM5    | DB-pass: 13514  | Train cells: 9736   | Valid cells: 3778   \n",
    "Source: CM5_2  | DB-pass: 143289 | Train cells: 136423 | Valid cells: 6866  \n",
    "Source: CM6    | DB-pass: 52137  | Train cells: 51771  | Valid cells: 366   \n",
    "Source: CM7    | DB-pass: 29063  | Train cells: 29044  | Valid cells: 19  \n",
    "Source: CM8    | DB-pass: 64831  | Train cells: 64371  | Valid cells: 460    \n",
    "Source: CM9    | DB-pass: 103250 | Train cells: 102588 | Valid cells: 662   \n",
    "Source: CM10   | DB-pass: 48236  | Train cells: 47865  | Valid cells: 371  \n",
    "Source: CM11   | DB-pass: 203269 | Train cells: 189877 | Valid cells: 13392   \n",
    "Source: CM20   | DB-pass: 156290 | Train cells: 155597 | Valid cells: 693   \n",
    "Source: CM22   | DB-pass: 136937 | Train cells: 123434 | Valid cells: 13503  \n",
    "Source: CM28   | DB-pass: 73044  | Train cells: 71366  | Valid cells: 1678   \n",
    "Source: CM29   | DB-pass: 151935 | Train cells: 145505 | Valid cells: 6430   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6e7402f7-a55c-44c0-8620-28eedf4e3632",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CM-007-2-1-1    18168\n",
       "CM-007-2-1-6    17137\n",
       "CM-007-2-1-2    17069\n",
       "CM-007-2-1-5    16324\n",
       "CM-007-2-1-4    15306\n",
       "CM-007-2-1-7    15244\n",
       "CM-007-2-1-3    13355\n",
       "CM-007-2-2-1     4080\n",
       "Name: stim, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = 'CM7_new'\n",
    "ifile = DATA_HOME/f'ymc_copy/{dataset}.TSM_VS_VMM_all.bcr.samples.IGHV.igblast.tsv'\n",
    "df_temp = pd.read_csv(ifile, sep='\\t')\n",
    "df_temp.stim.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f3548dba-6484-4c76-924d-0335a87a2517",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source: CM7_new | DB-pass: 115724 | Train cells: 111713 | Valid cells: 4011  \n"
     ]
    }
   ],
   "source": [
    "VALID_MICE.update({\n",
    "    'CM7_new': [['CM-007-2-2-1']]\n",
    "    # 'CM39': [['CM039-2-1'], ['CM039-3-1', 'CM039-3-2', 'CM039-3-3']]\n",
    "})\n",
    "\n",
    "for dataset in [f'CM{i}' for i in CURRENT_DATASET]:\n",
    "    ifile = CELL_PASS_DIR/f'{dataset}_fixed.tsv'\n",
    "    if not ifile.exists():\n",
    "        ifile = CELL_PASS_DIR/f'{dataset}.fmt7'\n",
    "    \n",
    "    df = pd.read_csv(ifile, sep='\\t')\n",
    "    # df['mice_index'] = df['sequence_id'].apply(lambda x: x.split('.')[1])\n",
    "    df = fix_wrong_dtype(df)\n",
    "    \n",
    "    df_tp = df.loc[df['mice_index'].isin(VALID_MICE[dataset][0]), :]\n",
    "    if len(VALID_MICE[dataset])>1:\n",
    "        df_tn = df.loc[df['mice_index'].isin(VALID_MICE[dataset][1]), :]\n",
    "        df_t = df.loc[(~df.index.isin(df_tp.index))&(~df.index.isin(df_tn.index)), :]\n",
    "        df_tn.to_csv(CELL_PASS_DIR/f'{dataset}_valid_tn.tsv', sep='\\t', index=None, na_rep='')\n",
    "        df_tp.to_csv(CELL_PASS_DIR/f'{dataset}_valid_tp.tsv', sep='\\t', index=None, na_rep='')\n",
    "    else:\n",
    "        df_t = df.loc[~df.index.isin(df_tp.index), :]\n",
    "        df_tp.to_csv(CELL_PASS_DIR/f'{dataset}_valid.tsv', sep='\\t', index=None, na_rep='')\n",
    "    \n",
    "    df_t.to_csv(CELL_PASS_DIR/f'{dataset}_train.tsv', sep='\\t', index=None, na_rep='')\n",
    "    print('Source: {:<6} | DB-pass: {:<6} | Train cells: {:<6d} | Valid cells: {:<6d}'.format(dataset, df.shape[0], df_t.shape[0], df_tp.shape[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d151981a-6f7c-4c52-a00b-c0d40540e33d",
   "metadata": {},
   "source": [
    "## Find optimal distance threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "80501903-87e9-4cf2-8b16-e13e89d84173",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Submitted batch job 3\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sh_file = SCRIPT / 'find_threshold.sh'\n",
    "sh_out = SCRIPT / f'log/slurm_find_threshold.out'\n",
    "\n",
    "with open(sh_file, 'w') as f:\n",
    "    f.write('#!/bin/bash\\n\\n')\n",
    "\n",
    "    f.write('#SBATCH -n 1 # 指定核心数量\\n')\n",
    "    f.write('#SBATCH -N 1 # 指定node的数量\\n')\n",
    "    f.write('#SBATCH -t 0-02:00 # 运行总时间，天数-小时数-分钟, D-HH:MM\\n')\n",
    "    f.write('#SBATCH --mem=5G # 所有核心可以使用的内存池大小, MB为单位\\n')\n",
    "    f.write(f'#SBATCH -o {sh_out} # 把输出结果STDOUT保存在哪一个文件\\n')\n",
    "    f.write('#SBATCH --nodelist=ab03\\n\\n')\n",
    "\n",
    "    f.write(\"docker start {}\\n\".format(CONTAINER))\n",
    "    now = datetime.now().strftime('%Y%m%d%H%M%S')\n",
    "    stdout = WORK_DIR / f'data/log/findThreshold_{now}.out'\n",
    "    code = f'''docker exec {CONTAINER} /bin/bash -c \"(time Rscript findThreshold.r) > {stdout} 2>&1\"\\n'''\n",
    "    f.write('{}\\n'.format(code))\n",
    "\n",
    "os.system(f'sbatch {sh_file}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d82e7117-007d-4697-9bfe-c228268eba73",
   "metadata": {},
   "source": [
    "## Define clones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6e1ef62c-a6f6-4a07-9fb8-21ce964e1b03",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/data/data/cell_pass/CM7_new_train.tsv\n",
      "Submitted batch job 4\n",
      "/data/data/cell_pass/CM7_new_valid.tsv\n",
      "Submitted batch job 5\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(DATA_HOME/'threshold_CM_20220809.csv')\n",
    "\n",
    "for row in df.iloc[76:, :].itertuples(False):\n",
    "    if pd.isna(row[1]):\n",
    "        print(f'No threshold of {row[0]}')\n",
    "        continue\n",
    "    print(row[0])\n",
    "    define_clones(Path(row[0]), row[1], CONTAINER, True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b32a4ce9-6f94-414a-afea-9ae15e7aa8dc",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Create germline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c5aa38ae-4a09-4657-b4d8-8eb0d68cedfb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Submitted batch job 6\n",
      "Submitted batch job 7\n"
     ]
    }
   ],
   "source": [
    "for dataset in [f'CM{i}' for i in CURRENT_DATASET]:\n",
    "    dbs = (DATA_HOME/'clones').glob(f'{dataset}*')\n",
    "    for db in dbs:\n",
    "        db_in_docker = WORK_DIR/'data/clones' / db.name\n",
    "        create_germline(db_in_docker, CONTAINER, True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a01af7e-d5ee-4047-ab3b-1e0098ec7767",
   "metadata": {},
   "source": [
    "## Merge *_aa columns in fmt19 with *clone_pass_germ-pass.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f9df4271-3a14-40ff-b844-3a8e737ee8e2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source: CM7_new | File: CM7_new_valid-clone_pass_germ-pass | Source shape: 4011   | Merged shape: 4011  \n",
      "Source: CM7_new | File: CM7_new_train-clone_pass_germ-pass | Source shape: 111713 | Merged shape: 111713\n"
     ]
    }
   ],
   "source": [
    "for dataset in [f'CM{i}' for i in CURRENT_DATASET]:\n",
    "    # if dataset=='CM00':\n",
    "    #     germ_f = HOME/f'clones/{dataset}_train-db_pass_clone-pass_germ-pass.tsv'\n",
    "    # else:\n",
    "    #     germ_f = HOME/f'clones/{dataset}_valid-db_pass_clone-pass_germ-pass.tsv'\n",
    "    \n",
    "    db = DATA_HOME/f'fmt19/{dataset}.fmt19'\n",
    "    df_fmt19 = pd.read_csv(db, sep='\\t')\n",
    "    cols = [i for i in df_fmt19.columns if i.find('_aa')>=0]\n",
    "    cols.remove('junction_aa')\n",
    "    cols.append('sequence_id')\n",
    "    \n",
    "    germ_files = (DATA_HOME/'clones').glob(f'{dataset}*germ-pass*')\n",
    "    for germ_f in germ_files:\n",
    "        df = pd.read_csv(germ_f, sep='\\t')\n",
    "        df2 = pd.merge(df, df_fmt19.loc[:, cols], on='sequence_id', how='left')\n",
    "        df2 = fix_wrong_dtype(df2)\n",
    "        df2.to_csv(germ_f.parent/f'{germ_f.stem}_merged.tsv', sep='\\t', index=None, na_rep='')\n",
    "        print('Source: {:<6} | File: {:<} | Source shape: {:<6d} | Merged shape: {:<6d}'.format(dataset, germ_f.stem, df.shape[0], df2.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b21b598c-0303-4a11-9520-58fb03178974",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'CM7_new': ['CM7_new', 116683, 956, 3, 115724]}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for key, val in statistic.items():\n",
    "    statistic[key] = val[:5]\n",
    "statistic"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5ac7f70-6cbb-4bf0-8a07-6565786d4f11",
   "metadata": {},
   "source": [
    "# 统计数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ec04f5ba-af29-48c7-811e-48a2df9646fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'CM7_new': ['CM7_new', 116683, 956, 3, 115724, 111713, 111713, 45750, 111713, 45750, 4011, 4011, 2055, 4011, 2055]}\n"
     ]
    }
   ],
   "source": [
    "for dataset in [f'CM{i}' for i in CURRENT_DATASET]:\n",
    "    clone_f_train = DATA_HOME/f'clones/{dataset}_train-clone_pass.tsv'\n",
    "    germ_f_train = DATA_HOME/f'clones/{dataset}_train-clone_pass_germ-pass.tsv'\n",
    "    df_clone_train = pd.read_csv(clone_f_train, sep='\\t')\n",
    "    df_germ_train = pd.read_csv(germ_f_train, sep='\\t')\n",
    "    \n",
    "    clone_f_valid = DATA_HOME/f'clones/{dataset}_valid-clone_pass.tsv'\n",
    "    germ_f_valid = DATA_HOME/f'clones/{dataset}_valid-clone_pass_germ-pass.tsv'\n",
    "    if clone_f_valid.exists():\n",
    "        df_clone_valid = pd.read_csv(clone_f_valid, sep='\\t')\n",
    "        df_germ_valid = pd.read_csv(germ_f_valid, sep='\\t')\n",
    "        \n",
    "        statistic[dataset].extend([df_clone_train.shape[0],\n",
    "                                    df_clone_train.shape[0],\n",
    "                                    len(set(df_clone_train.sequence_alignment)),\n",
    "                                   df_germ_train.shape[0],\n",
    "                                    len(set(df_germ_train.sequence_alignment)),\n",
    "                                   df_clone_valid.shape[0],\n",
    "                                    df_clone_valid.shape[0],\n",
    "                                    len(set(df_clone_valid.sequence_alignment)),\n",
    "                                   df_germ_valid.shape[0],\n",
    "                                    len(set(df_germ_valid.sequence_alignment))])\n",
    "        \n",
    "    else:\n",
    "        clone_f_valid_tp = DATA_HOME/f'clones/{dataset}_valid_tp-clone_pass.tsv'\n",
    "        germ_f_valid_tp = DATA_HOME/f'clones/{dataset}_valid_tp-clone_pass_germ-pass.tsv'\n",
    "        clone_f_valid_tn = DATA_HOME/f'clones/{dataset}_valid_tn-clone_pass.tsv'\n",
    "        germ_f_valid_tn = DATA_HOME/f'clones/{dataset}_valid_tn-clone_pass_germ-pass.tsv'\n",
    "        \n",
    "        df_clone_valid_tp = pd.read_csv(clone_f_valid_tp, sep='\\t')\n",
    "        df_germ_valid_tp = pd.read_csv(germ_f_valid_tp, sep='\\t')\n",
    "        df_clone_valid_tn = pd.read_csv(clone_f_valid_tn, sep='\\t')\n",
    "        df_germ_valid_tn = pd.read_csv(germ_f_valid_tn, sep='\\t')\n",
    "        \n",
    "        statistic[dataset].extend([df_clone_train.shape[0],\n",
    "                                    df_clone_train.shape[0],\n",
    "                                    len(set(df_clone_train.sequence_alignment)),\n",
    "                                   df_germ_train.shape[0],\n",
    "                                    len(set(df_germ_train.sequence_alignment)),\n",
    "                                   df_clone_valid_tp.shape[0],\n",
    "                                   df_clone_valid_tp.shape[0],\n",
    "                                    len(set(df_clone_valid_tp.sequence_alignment)),\n",
    "                                   df_germ_valid_tp.shape[0],\n",
    "                                    len(set(df_germ_valid_tp.sequence_alignment)),\n",
    "                                   df_clone_valid_tn.shape[0],\n",
    "                                    df_clone_valid_tn.shape[0],\n",
    "                                    len(set(df_clone_valid_tn.sequence_alignment)),\n",
    "                                   df_germ_valid_tn.shape[0],\n",
    "                                    len(set(df_germ_valid_tn.sequence_alignment))])\n",
    "        \n",
    "print(statistic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d8112f0e-e944-45db-a444-9907bbd23aae",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(DATA_HOME/'statistic.csv', 'a+') as f:\n",
    "    f_csv = csv.writer(f)\n",
    "    # f_csv.writerow(statics_header)\n",
    "    f_csv.writerows(statistic.values())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22e41291-f679-4442-9b96-c13ee9201b8a",
   "metadata": {},
   "source": [
    "# Calculate mutation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cfeb0fb-6215-44d5-a73b-9d146557a9ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "!docker exec $CONTAINER /bin/bash -c \"Rscript mutation_statistic.r\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4760ad47-809c-4e97-ad7e-33a380272283",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "source": [
    "## Statistic\n",
    "\n",
    "Source: CM00 | Train clone: 21440/54663  | Train germ: 21440/54663   \n",
    "Source: CM1 | Train clone: 15618/24200  | Train germ: 15618/24200  | Valid germ: 2268/3635   | Valid germ: 2268/3635   \n",
    "Source: CM2 | Train clone: 17525/23363  | Train germ: 17525/23363  | Valid germ: 2404/3078   | Valid germ: 2404/3078  \n",
    "Source: CM3 | Train clone: 24016/43190  | Train germ: 24016/43190  | Valid germ: 624/669    | Valid germ: 624/669   \n",
    "Source: CM4 | Train clone: 46610/86160  | Train germ: 46610/86160  | Valid germ: 1134/1754   | Valid germ: 1134/1754  \n",
    "Source: CM5 | Train clone: 6167/9736   | Train germ: 6167/9736   | Valid germ: 1099/3778   | Valid germ: 1099/3778  \n",
    "Source: CM6 | Train clone: 22033/51771  | Train germ: 22033/51771  | Valid germ: 224/366    | Valid germ: 224/366   \n",
    "Source: CM7 | Train clone: 17321/29044  | Train germ: 17321/29044  | Valid germ: 19/19     | Valid germ: 19/19    \n",
    "Source: CM8 | Train clone: 27507/64371  | Train germ: 27507/64371  | Valid germ: 276/460    | Valid germ: 276/460   \n",
    "Source: CM9 | Train clone: 61346/102588 | Train germ: 61346/102588 | Valid germ: 564/662    | Valid germ: 564/662   \n",
    "Source: CM10 | Train clone: 28987/47865  | Train germ: 28987/47865  | Valid germ: 331/371    | Valid germ: 331/371   \n",
    "Source: CM12 | Train clone: 66969/129047 | Train germ: 66969/129047 | Valid germ: 3285/9127   | Valid germ: 3285/9127  \n",
    "Source: CM13 | Train clone: 79820/146466 | Train germ: 79815/146461 | Valid germ: 1919/6253   | Valid germ: 1919/6253  \n",
    "Source: CM15 | Train clone: 69019/141947 | Train germ: 69019/141947 | Valid germ: 4611/9615   | Valid germ: 4611/9615  \n",
    "Source: CM21 | Train clone: 75322/122619 | Train germ: 75322/122619 | Valid germ: 3443/7090   | Valid germ: 3443/7090  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6dab10e9-202e-4574-ac61-668e9b279403",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source: CM5_2 | Train clone: 65359/136423 | Train germ: 65359/136423 | Valid germ: 2842/6866   | Valid germ: 2842/6866  \n",
      "Source: CM11 | Train clone: 40730/189877 | Train germ: 40730/189877 | Valid germ: 7556/13392  | Valid germ: 7556/13392 \n",
      "Source: CM20 | Train clone: 85732/155597 | Train germ: 85732/155597 | Valid germ: 549/693    | Valid germ: 549/693   \n",
      "Source: CM22 | Train clone: 59482/123434 | Train germ: 59482/123434 | Valid germ: 4204/13503  | Valid germ: 4204/13503 \n",
      "Source: CM28 | Train clone: 41614/71366  | Train germ: 41614/71366  | Valid germ: 966/1678   | Valid germ: 966/1678  \n",
      "Source: CM29 | Train clone: 99292/145505 | Train germ: 99292/145505 | Valid germ: 3702/6430   | Valid germ: 3702/6430  \n"
     ]
    }
   ],
   "source": [
    "for dataset in [f'CM{i}' for i in ['5_2',11,20,22,28,29]]:\n",
    "    clone_f_t = DATA_HOME/f'clones/{dataset}_train-clone_pass.tsv'\n",
    "    clone_f_v = DATA_HOME/f'clones/{dataset}_valid-clone_pass.tsv'\n",
    "    germ_f_t = DATA_HOME/f'clones/{dataset}_train-clone_pass_germ-pass.tsv'\n",
    "    germ_f_v = DATA_HOME/f'clones/{dataset}_valid-clone_pass_germ-pass.tsv'\n",
    "\n",
    "    clone_df_t = pd.read_csv(clone_f_t, sep='\\t', usecols=['sequence_id', 'sequence_alignment'])\n",
    "    germ_df_t = pd.read_csv(germ_f_t, sep='\\t', usecols=['sequence_id', 'sequence_alignment'])\n",
    "    if dataset!='CM0':\n",
    "        clone_df_v = pd.read_csv(clone_f_v, sep='\\t', usecols=['sequence_id', 'sequence_alignment'])\n",
    "        germ_df_v = pd.read_csv(germ_f_v, sep='\\t', usecols=['sequence_id', 'sequence_alignment'])\n",
    "        print(f'Source: {dataset} | Train clone: {clone_df_t.sequence_alignment.unique().shape[0]}/{clone_df_t.shape[0]:<6d} | Train germ: {germ_df_t.sequence_alignment.unique().shape[0]}/{germ_df_t.shape[0]:<6d} | Valid germ: {clone_df_v.sequence_alignment.unique().shape[0]}/{clone_df_v.shape[0]:<6d} | Valid germ: {germ_df_v.sequence_alignment.unique().shape[0]}/{germ_df_v.shape[0]:<6d}')\n",
    "    else:                   \n",
    "        print(f'Source: {dataset} | Train clone: {clone_df_t.sequence_alignment.unique().shape[0]}/{clone_df_t.shape[0]:<6d} | Train germ: {germ_df_t.sequence_alignment.unique().shape[0]}/{germ_df_t.shape[0]:<6d}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4917a612-9a18-4424-b5cf-4bd1d9667632",
   "metadata": {},
   "source": [
    "## Build tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "2cf0aec0-93b1-413f-a6a1-c8c7ee7cb5ac",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Submitted batch job 66091\n",
      "Submitted batch job 66092\n",
      "Submitted batch job 66093\n",
      "Submitted batch job 66094\n",
      "Submitted batch job 66095\n",
      "Submitted batch job 66096\n",
      "Submitted batch job 66097\n",
      "Submitted batch job 66098\n",
      "Submitted batch job 66099\n",
      "Submitted batch job 66100\n",
      "Submitted batch job 66101\n",
      "Submitted batch job 66102\n",
      "Submitted batch job 66103\n",
      "Submitted batch job 66104\n",
      "Submitted batch job 66105\n",
      "Submitted batch job 66106\n",
      "Submitted batch job 66107\n",
      "Submitted batch job 66108\n",
      "Submitted batch job 66109\n",
      "Submitted batch job 66110\n",
      "Submitted batch job 66111\n",
      "Submitted batch job 66112\n",
      "Submitted batch job 66113\n",
      "Submitted batch job 66114\n",
      "Submitted batch job 66115\n",
      "Submitted batch job 66116\n",
      "Submitted batch job 66117\n",
      "Submitted batch job 66118\n",
      "Submitted batch job 66119\n",
      "Submitted batch job 66120\n",
      "Submitted batch job 66121\n",
      "Submitted batch job 66122\n",
      "Submitted batch job 66123\n",
      "Submitted batch job 66124\n",
      "Submitted batch job 66125\n",
      "Submitted batch job 66126\n",
      "Submitted batch job 66127\n"
     ]
    }
   ],
   "source": [
    "db = DATA_HOME/'clones/CM20_valid-clone_pass_germ-pass.tsv'\n",
    "build_repertoire_tree(db, CONTAINER, 3, 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "56cc6969-6215-47a7-b3cd-1dd4732975e2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rebuild Tree: CM5_2_valid-clone_pass_germ-pass.tsv\n"
     ]
    }
   ],
   "source": [
    "db = DATA_HOME/f'clones/CM5_2_valid-clone_pass_germ-pass.tsv'\n",
    "rebuild_tree(db, CONTAINER, 3, do_run=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b9f9511c-143c-4047-954f-941e49de02e9",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rebuild Tree: CM001_train-clone_pass_germ-pass.tsv\n",
      "Rebuild Tree: CM002_train-clone_pass_germ-pass.tsv\n",
      "Rebuild Tree: CM003_train-clone_pass_germ-pass.tsv\n",
      "Rebuild Tree: CM004_train-clone_pass_germ-pass.tsv\n",
      "Rebuild Tree: CM005_train-clone_pass_germ-pass.tsv\n",
      "Rebuild Tree: CM006_train-clone_pass_germ-pass.tsv\n",
      "Rebuild Tree: CM008_train-clone_pass_germ-pass.tsv\n",
      "Rebuild Tree: CM009_train-clone_pass_germ-pass.tsv\n",
      "Rebuild Tree: CM010_train-clone_pass_germ-pass.tsv\n"
     ]
    }
   ],
   "source": [
    "for i in range(1,11):\n",
    "    if i==7:\n",
    "        continue\n",
    "    db = HOME/f'clones/CM{i:0=3}_train-clone_pass_germ-pass.tsv'\n",
    "    rebuild_tree(db, CONTAINER, 3, do_run=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ae2c5268-9ea3-4021-9080-c4c040ff1519",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Submitted batch job 35073\n"
     ]
    }
   ],
   "source": [
    "germ_file = WORK_DIR/'data/clones/CM002_valid-clone-pass_germ-pass.tsv'\n",
    "\n",
    "build_lineage_tree(germ_file, CONTAINER, 99, 20, 3, True, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "e7074401-c872-4565-b6ff-b00b2b6925f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df30= pd.read_csv(DATA_HOME/'ymc_copy/CM3_all.bcr.samples.IGHV.igblast.tsv', sep='\\t')\n",
    "df31= pd.read_csv(DATA_HOME/'clones/CM3_valid_clone-pass_germ-pass_merged.tsv', sep='\\t')\n",
    "\n",
    "pos_ids = [\n",
    "    'CGTCAGGAGTCAATAG-1.VMM_1',\n",
    "    'GTACGTAGTTTACTCT-1.VMM_1',\n",
    "    'CATTCGCCAACTGCGC-1.VMM_5_9',\n",
    "    'TTCTCAATCCTGTACC-1.VMM_5_9',\n",
    "    'AACTCAGGTCCCTTGT-1.VMM_5_9',\n",
    "    'CTGGTCTCATTAACCG-1.VMM_5_9',\n",
    "    'GCATGCGTCACAGGCC-1.VMM_5_9',\n",
    "    'TGGGCGTTCCACGACG-1.VMM_5_9',\n",
    "    'CAGCCGACAAGGACAC-1.VMM_5_9',\n",
    "    'TGGGCGTTCCACGACG-1.VMM_5_9',\n",
    "    'GAAGCAGGTCATCCCT-1.VMM_1',\n",
    "    'GCTGCGAAGATATGCA-1.VMM_5_9',\n",
    "    'CCATGTCTCTCGCATC-1.VMM_5_9',\n",
    "    'GATCGCGCATCCCACT-1.VMM_5_9'\n",
    "]\n",
    "\n",
    "neg_ids = [\n",
    "    'GAGGTGATCTCAACTT-1.VMM_5_9',\n",
    "    'CGGTTAAAGAGTCTGG-1.VMM_5_9',\n",
    "    'CTACACCTCGTCCAGG-1.VMM_1',\n",
    "    'GCATGATCACGAAAGC-1.VMM_5_9',\n",
    "    'CGTTAGAGTTGCTCCT-1.VMM_5_9',\n",
    "    'TGAGCCGTCCACGAAT-1.VMM_5_9',\n",
    "    'GACACGCTCCTGCTTG-1.VMM_5_9'\n",
    "]\n",
    "\n",
    "seq_aa = {\n",
    "'QVQLQESGPGLVKPSGTLSLTCSVSGGSISNTNWWSWVRQPPGKGLEWIGEIYHSGSTKYNPSLKSRVTISVDKSKNQFSLKFYSVTAADTAVYYCARASFDILTGYYGMDVWGQGTTVTVSS':1,\n",
    "'QVQLQESGPGLVKPSGTLSLICAVSGGSISSTNWWSWVRQAPEKGLEWIGEIYHSGSTRYNPSLKSRVTISVDKSKNQFSLKLTSVTAADTAVYYCARAYYDVLTGYDGLDVWGLGTSVTVSS':0,\n",
    "'QVQLQESGPGLVKPSGTLSLTCAVSDGSISSTNWWSWVRQSPGKGLEWIGEIYHSGSTKYNPSLKSRVTILVDKSKNQFSLKLNSVTAADTAVYYCARASFDLLTGYEGLDVWGQGTTVTVSS':0.5,\n",
    "'QVQLQESGPGLVKPSGTLSLTCAVSGGSISSSNWWSWVRQSPGKGLEWIGEIYHSGSTKYNPSLKSRVTISLDKSENQFSLNLYSVTAADSAVYYCARAAFDILTGFYGLDVWGQGTTVTVSS':0,\n",
    "'QVQLQESGPELVKPSGTLSLTCAVSGGSITSTYWWTWVRQPPGKGLEWIGEIFHSGSTKYNPSLKSRVTFSVDKSKNQFSLKLTSVTDADTAVYYCARASFDILTGYSGLDVWGQGTTVTVSS':0,\n",
    "'QLQLQESGPGLVKPSETLSLTCTVSGGSINSGDYYWGWIRQPPGKGLEWIGSISYSGSTYSKPSLKSRVTISVDTSKNQFSLKLSSVTAADTAFYHCARQGRFLDWLLYFDYWGQGALVTVSS':1, \n",
    "'QLQLQASGPGLVKPSETLSLTCTVSGGSISRSDNYWGWIRQPPGKGLEWIGSISSSGTTYYNPSLKSRVTISVDTSKDQFSLNLTSVTAADTAVYYCVRQGRFLAWLLYFELWDQGTLVTVSS':1,\n",
    "'HLQLQESGPGLVKPSETLALTCNVSDGSISSGEYYWGWIRQPPGKGLEWIGSVDYGGTTYYNPSLKSQATISVDTSKNQFSLKLSSVTAADTAVYYCVRQGRFLNWLLYFEDWGRGTLVTVSS':1, \n",
    "'QLQLQESGPGPVKPSETLSLTCSVSGGSISSGNYYWGWIRQSPGKGLDWIGSIYYGGTTYYNPSLKSRVTISVDTSMNQFSLHLSSVTAADTAMYFCVKQGRFLDWLLYFDYWGQGTLVIVSS':1,\n",
    "'QLQLQESGPGLVKPSETLSLTCTVSGDSVSSGDYYWAWIRQPPGKGLEWIGTIYYSGTTYYNPSLKSRVTISVTTSKNQFSLKLNSVTATDTAVYYCARQGRFLDWLLYFENWGQGNLVTVSS':1, \n",
    "'QVQLQQWGAGLLKPSETLSLTCAVYGGSLSGNYWSWIRQSPGKGLEWIGEINHYGSTNYNPSLKSRVTISLDTPNNQFSLKLSSVTAADTAVYYCAREGYDILTGVERDFHYWGQGILVTVSS':0.5,\n",
    "'QVQLQQWGAGLLKPSETLSLTCAVYGGSFKNYYWSWIRQSPGKGLEWIGEISHYGNTNYIPSLKSRVTISLDPSTNQFSLKLTSVTAADTAVYYCARQGYDILTGVERHFDSWGQGTLVTVSS':1,\n",
    "'QVQLQQWGAGLLKPSETLSLTCAIYGGSFSGYWWTWIRQSPGKGLEWIGEISHSGNTNYNPSLKSRVTITLDTSKNQFSLNLSSVAAADTAIYYCAREGYDILTGLERHFDSWGQGILVTVSS':1,\n",
    "'QVQLQQWGAGLLKPSETLSLTCAVYDGSLSGYYWSWIRQSPGKGLEWIGEINHYGGTNYNPSLKSRVTISRDTSKNQFSLKLSSVTGADTAVYYCARQGYDILTGVERDFDSWGQGTLVTVSS':0,\n",
    "'QVQLQQWGAGLLKPSETLSLTCAVNGGSFSGYYWTWIRQSPGKGLEWIGEINHSGSITNYSPSLKSRVTISVDTSKNQFSLKLSSVTAADTAVYFCARQGYDILTGVERHFDYWGQGTVVTVSS':0,\n",
    "'QVQLLESGGGVVQPGRSLRLSCAASGFTFSSYGMHWVRQAPGKGLEWVTFIWYDQNNKYYADSVKGRFTISRDNSKKMLYLQMNSLRVEDTAVYYCARERETTFSHYYGMDVWGRGTTVTVSS':1,\n",
    "'QVQVQESGPGLVKPSGTLSLTCAVSGGSMRSSNWWSWVRQSPGKGLEWIGEIHYSGNTNYNPSLKSRVTMSVDKSENQFSLKLTSVTAADTAVYYCARVGHDILTGLNWFDPWGQGSPVIVSS':0,\n",
    "'QVQLQQSGPGLVKPSQTLSVTCVISGDSVSTSSAVWNWIRQSPSRGLEWLGRTYYRSKWYNDYAISVKSRITINPDTSKNQFSLHLTSVTPEDTAVYYCARGGSSSYDYYGFDVWGQGTTVTVSS':1,\n",
    "'QVQLQQWGAGLLKPSETLSLTCAVSGGSFSGYYWSWIRQSPGKGLEWIGEINHSGSITNYSPSLKSRVTISVDTSKNQFSLKLSSVTAADTAVYFCAREDYDILTGVERDFDYWGQGTVVTVSS':0\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "b7ec861e-bd2a-4d0b-82dc-b68f4635c5a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pos_ids)+len(neg_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "f3b8cbfa-809d-45e3-b7f8-753cdcb6f6a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_id(barcode):\n",
    "    nt = barcode.split('.')[0]\n",
    "    mouse = barcode.split('.')[-1].replace('CM003', 'VMM').replace('-', '_')\n",
    "    return f'{nt}.{mouse}'\n",
    "\n",
    "df3['new_id'] = df3['barcode'].apply(convert_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "7f70be30-859c-4838-b53b-d4af2781ddf4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stim</th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>rev_comp</th>\n",
       "      <th>productive</th>\n",
       "      <th>v_call</th>\n",
       "      <th>d_call</th>\n",
       "      <th>j_call</th>\n",
       "      <th>sequence_alignment</th>\n",
       "      <th>germline_alignment</th>\n",
       "      <th>...</th>\n",
       "      <th>consensus_count</th>\n",
       "      <th>umi_count</th>\n",
       "      <th>v_call_10x</th>\n",
       "      <th>d_call_10x</th>\n",
       "      <th>j_call_10x</th>\n",
       "      <th>junction_10x</th>\n",
       "      <th>junction_10x_aa</th>\n",
       "      <th>barcode</th>\n",
       "      <th>celltype</th>\n",
       "      <th>new_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>29625</th>\n",
       "      <td>CM003-1</td>\n",
       "      <td>GTACGTAGTTTACTCT-1.CM003-1</td>\n",
       "      <td>TGGGGATGCTTTCTGAGAGTCATGGATCTCATGTGCAAGAAAATGA...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>IGHV4-39*01</td>\n",
       "      <td>IGHD3/OR15-3a*01,IGHD3/OR15-3b*01</td>\n",
       "      <td>IGHJ4*02</td>\n",
       "      <td>CAACTACAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>CAGCTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>...</td>\n",
       "      <td>7953</td>\n",
       "      <td>14</td>\n",
       "      <td>hsa-IGHV4-39</td>\n",
       "      <td>hsa-IGHD3/OR15-3A</td>\n",
       "      <td>hsa-IGHJ4</td>\n",
       "      <td>TGTGCGAGACAAGGCCGATTTTTGGACTGGTTATTGTACTTTGAGA...</td>\n",
       "      <td>CARQGRFLDWLLYFENW</td>\n",
       "      <td>GTACGTAGTTTACTCT-1.CM003-1</td>\n",
       "      <td>Plasma cells</td>\n",
       "      <td>GTACGTAGTTTACTCT-1.VMM_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29643</th>\n",
       "      <td>CM003-1</td>\n",
       "      <td>CGTCAGGAGTCAATAG-1.CM003-1</td>\n",
       "      <td>GACCCTGCAGCTCTGGGAGAGGAGCCCAGCACTAGAAGTCGGCGGT...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>IGHV3-33*01</td>\n",
       "      <td>IGHD4-11*01,IGHD4-17*01,IGHD4-23*01</td>\n",
       "      <td>IGHJ6*02</td>\n",
       "      <td>CAGGTGCAGTTGCTGGAATCTGGGGGA...GGCGTGGTCCAGCCTG...</td>\n",
       "      <td>CAGGTGCAGCTGGTGGAGTCTGGGGGA...GGCGTGGTCCAGCCTG...</td>\n",
       "      <td>...</td>\n",
       "      <td>3504</td>\n",
       "      <td>14</td>\n",
       "      <td>hsa-IGHV3-33</td>\n",
       "      <td>mus-IGHD5-5</td>\n",
       "      <td>hsa-IGHJ6</td>\n",
       "      <td>TGTGCGAGAGAGAGAGAGACTACCTTTTCCCACTACTACGGTATGG...</td>\n",
       "      <td>CARERETTFSHYYGMDVW</td>\n",
       "      <td>CGTCAGGAGTCAATAG-1.CM003-1</td>\n",
       "      <td>Memory B-cells</td>\n",
       "      <td>CGTCAGGAGTCAATAG-1.VMM_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30822</th>\n",
       "      <td>CM003-1</td>\n",
       "      <td>GAAGCAGGTCATCCCT-1.CM003-1</td>\n",
       "      <td>GATATACTTTCTGAGACTCATGGACCTCCTGCACAAGAACATGAAA...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>IGHV4-4*02</td>\n",
       "      <td>IGHD3-9*01</td>\n",
       "      <td>IGHJ6*02</td>\n",
       "      <td>CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>...</td>\n",
       "      <td>4586</td>\n",
       "      <td>12</td>\n",
       "      <td>hsa-IGHV4-4</td>\n",
       "      <td>hsa-IGHD3-9</td>\n",
       "      <td>hsa-IGHJ6</td>\n",
       "      <td>TGTGCGAGAGCGTCTTTCGATCTTTTGACTGGTTATGAGGGATTGG...</td>\n",
       "      <td>CARASFDLLTGYEGLDVW</td>\n",
       "      <td>GAAGCAGGTCATCCCT-1.CM003-1</td>\n",
       "      <td>Class-switched memory B-cells</td>\n",
       "      <td>GAAGCAGGTCATCCCT-1.VMM_1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 61 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          stim                 sequence_id  \\\n",
       "29625  CM003-1  GTACGTAGTTTACTCT-1.CM003-1   \n",
       "29643  CM003-1  CGTCAGGAGTCAATAG-1.CM003-1   \n",
       "30822  CM003-1  GAAGCAGGTCATCCCT-1.CM003-1   \n",
       "\n",
       "                                                sequence  rev_comp  \\\n",
       "29625  TGGGGATGCTTTCTGAGAGTCATGGATCTCATGTGCAAGAAAATGA...     False   \n",
       "29643  GACCCTGCAGCTCTGGGAGAGGAGCCCAGCACTAGAAGTCGGCGGT...     False   \n",
       "30822  GATATACTTTCTGAGACTCATGGACCTCCTGCACAAGAACATGAAA...     False   \n",
       "\n",
       "       productive       v_call                               d_call    j_call  \\\n",
       "29625        True  IGHV4-39*01    IGHD3/OR15-3a*01,IGHD3/OR15-3b*01  IGHJ4*02   \n",
       "29643        True  IGHV3-33*01  IGHD4-11*01,IGHD4-17*01,IGHD4-23*01  IGHJ6*02   \n",
       "30822        True   IGHV4-4*02                           IGHD3-9*01  IGHJ6*02   \n",
       "\n",
       "                                      sequence_alignment  \\\n",
       "29625  CAACTACAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...   \n",
       "29643  CAGGTGCAGTTGCTGGAATCTGGGGGA...GGCGTGGTCCAGCCTG...   \n",
       "30822  CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...   \n",
       "\n",
       "                                      germline_alignment  ... consensus_count  \\\n",
       "29625  CAGCTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...  ...            7953   \n",
       "29643  CAGGTGCAGCTGGTGGAGTCTGGGGGA...GGCGTGGTCCAGCCTG...  ...            3504   \n",
       "30822  CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...  ...            4586   \n",
       "\n",
       "      umi_count    v_call_10x         d_call_10x j_call_10x  \\\n",
       "29625        14  hsa-IGHV4-39  hsa-IGHD3/OR15-3A  hsa-IGHJ4   \n",
       "29643        14  hsa-IGHV3-33        mus-IGHD5-5  hsa-IGHJ6   \n",
       "30822        12   hsa-IGHV4-4        hsa-IGHD3-9  hsa-IGHJ6   \n",
       "\n",
       "                                            junction_10x     junction_10x_aa  \\\n",
       "29625  TGTGCGAGACAAGGCCGATTTTTGGACTGGTTATTGTACTTTGAGA...   CARQGRFLDWLLYFENW   \n",
       "29643  TGTGCGAGAGAGAGAGAGACTACCTTTTCCCACTACTACGGTATGG...  CARERETTFSHYYGMDVW   \n",
       "30822  TGTGCGAGAGCGTCTTTCGATCTTTTGACTGGTTATGAGGGATTGG...  CARASFDLLTGYEGLDVW   \n",
       "\n",
       "                          barcode                       celltype  \\\n",
       "29625  GTACGTAGTTTACTCT-1.CM003-1                   Plasma cells   \n",
       "29643  CGTCAGGAGTCAATAG-1.CM003-1                 Memory B-cells   \n",
       "30822  GAAGCAGGTCATCCCT-1.CM003-1  Class-switched memory B-cells   \n",
       "\n",
       "                         new_id  \n",
       "29625  GTACGTAGTTTACTCT-1.VMM_1  \n",
       "29643  CGTCAGGAGTCAATAG-1.VMM_1  \n",
       "30822  GAAGCAGGTCATCCCT-1.VMM_1  \n",
       "\n",
       "[3 rows x 61 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3.loc[df3.new_id.isin(pos_ids), :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "a4cc3dcc-2b3c-49cf-a87b-1355c332ee7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stim</th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>rev_comp</th>\n",
       "      <th>productive</th>\n",
       "      <th>v_call</th>\n",
       "      <th>d_call</th>\n",
       "      <th>j_call</th>\n",
       "      <th>sequence_alignment</th>\n",
       "      <th>germline_alignment</th>\n",
       "      <th>...</th>\n",
       "      <th>consensus_count</th>\n",
       "      <th>umi_count</th>\n",
       "      <th>v_call_10x</th>\n",
       "      <th>d_call_10x</th>\n",
       "      <th>j_call_10x</th>\n",
       "      <th>junction_10x</th>\n",
       "      <th>junction_10x_aa</th>\n",
       "      <th>barcode</th>\n",
       "      <th>celltype</th>\n",
       "      <th>new_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3520</th>\n",
       "      <td>CM003-1</td>\n",
       "      <td>CTACACCTCGTCCAGG-1.CM003-1</td>\n",
       "      <td>GGGAGGGTCCAGCTCACATGGGAAATACTTTCTGAGACTCATGGAC...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>IGHV4-4*02</td>\n",
       "      <td>IGHD3-9*01</td>\n",
       "      <td>IGHJ6*02</td>\n",
       "      <td>CAGGTGCAGCTACAGGAGTCGGGTCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>...</td>\n",
       "      <td>99642</td>\n",
       "      <td>4099</td>\n",
       "      <td>hsa-IGHV4-4</td>\n",
       "      <td>hsa-IGHD3-9</td>\n",
       "      <td>hsa-IGHJ6</td>\n",
       "      <td>TGTGCGAGAGCTTATTACGATGTTTTGACTGGTTATGACGGTTTGG...</td>\n",
       "      <td>CARAYYDVLTGYDGLDVW</td>\n",
       "      <td>CTACACCTCGTCCAGG-1.CM003-1</td>\n",
       "      <td>Plasma cells</td>\n",
       "      <td>CTACACCTCGTCCAGG-1.VMM_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12119</th>\n",
       "      <td>CM003-5-9</td>\n",
       "      <td>CGTTAGAGTTGCTCCT-1.CM003-5-9</td>\n",
       "      <td>GACATTTCCTTAAATTCAGGGTCCAGCTCACATGGGAAATACTTTC...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>IGHV4-4*02</td>\n",
       "      <td>IGHD3-9*01</td>\n",
       "      <td>IGHJ6*02</td>\n",
       "      <td>CAGGTGCAGCTGCAGGAGTCGGGCCCA...GAACTGGTGAAGCCTT...</td>\n",
       "      <td>CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>...</td>\n",
       "      <td>9223</td>\n",
       "      <td>237</td>\n",
       "      <td>hsa-IGHV4-4</td>\n",
       "      <td>hsa-IGHD3-9</td>\n",
       "      <td>hsa-IGHJ6</td>\n",
       "      <td>TGTGCGAGGGCCTCTTTCGATATTTTGACTGGTTACTCCGGACTGG...</td>\n",
       "      <td>CARASFDILTGYSGLDVW</td>\n",
       "      <td>CGTTAGAGTTGCTCCT-1.CM003-5-9</td>\n",
       "      <td>Plasma cells</td>\n",
       "      <td>CGTTAGAGTTGCTCCT-1.VMM_5_9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 61 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            stim                   sequence_id  \\\n",
       "3520     CM003-1    CTACACCTCGTCCAGG-1.CM003-1   \n",
       "12119  CM003-5-9  CGTTAGAGTTGCTCCT-1.CM003-5-9   \n",
       "\n",
       "                                                sequence  rev_comp  \\\n",
       "3520   GGGAGGGTCCAGCTCACATGGGAAATACTTTCTGAGACTCATGGAC...     False   \n",
       "12119  GACATTTCCTTAAATTCAGGGTCCAGCTCACATGGGAAATACTTTC...     False   \n",
       "\n",
       "       productive      v_call      d_call    j_call  \\\n",
       "3520         True  IGHV4-4*02  IGHD3-9*01  IGHJ6*02   \n",
       "12119        True  IGHV4-4*02  IGHD3-9*01  IGHJ6*02   \n",
       "\n",
       "                                      sequence_alignment  \\\n",
       "3520   CAGGTGCAGCTACAGGAGTCGGGTCCA...GGACTGGTGAAGCCTT...   \n",
       "12119  CAGGTGCAGCTGCAGGAGTCGGGCCCA...GAACTGGTGAAGCCTT...   \n",
       "\n",
       "                                      germline_alignment  ... consensus_count  \\\n",
       "3520   CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...  ...           99642   \n",
       "12119  CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...  ...            9223   \n",
       "\n",
       "      umi_count   v_call_10x   d_call_10x j_call_10x  \\\n",
       "3520       4099  hsa-IGHV4-4  hsa-IGHD3-9  hsa-IGHJ6   \n",
       "12119       237  hsa-IGHV4-4  hsa-IGHD3-9  hsa-IGHJ6   \n",
       "\n",
       "                                            junction_10x     junction_10x_aa  \\\n",
       "3520   TGTGCGAGAGCTTATTACGATGTTTTGACTGGTTATGACGGTTTGG...  CARAYYDVLTGYDGLDVW   \n",
       "12119  TGTGCGAGGGCCTCTTTCGATATTTTGACTGGTTACTCCGGACTGG...  CARASFDILTGYSGLDVW   \n",
       "\n",
       "                            barcode      celltype                      new_id  \n",
       "3520     CTACACCTCGTCCAGG-1.CM003-1  Plasma cells    CTACACCTCGTCCAGG-1.VMM_1  \n",
       "12119  CGTTAGAGTTGCTCCT-1.CM003-5-9  Plasma cells  CGTTAGAGTTGCTCCT-1.VMM_5_9  \n",
       "\n",
       "[2 rows x 61 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3.loc[df3.new_id.isin(neg_ids), :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "dfe27d63-6073-4674-9b98-a2e0732bb1ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>rev_comp</th>\n",
       "      <th>productive</th>\n",
       "      <th>v_call</th>\n",
       "      <th>d_call</th>\n",
       "      <th>j_call</th>\n",
       "      <th>sequence_alignment</th>\n",
       "      <th>germline_alignment</th>\n",
       "      <th>junction</th>\n",
       "      <th>...</th>\n",
       "      <th>j_sequence_alignment_aa</th>\n",
       "      <th>j_germline_alignment_aa</th>\n",
       "      <th>fwr1_aa</th>\n",
       "      <th>cdr1_aa</th>\n",
       "      <th>fwr2_aa</th>\n",
       "      <th>cdr2_aa</th>\n",
       "      <th>fwr3_aa</th>\n",
       "      <th>fwr4_aa</th>\n",
       "      <th>cdr3_aa</th>\n",
       "      <th>junction_aa_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>CACTCCAAGCTACCGC-1.CM003-5-9</td>\n",
       "      <td>GGGGAGGGTCCAGCTCACATGGGAAAGACTTTCTGAGACTCATGGA...</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>IGHV4-4*02</td>\n",
       "      <td>IGHD3-9*01</td>\n",
       "      <td>IGHJ5*02</td>\n",
       "      <td>CAGGTGCAGGTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>TGTGCGAGAGTTGGTCACGATATTTTGACTGGCCTCAACTGGTTCG...</td>\n",
       "      <td>...</td>\n",
       "      <td>NWFDPWGQGSPVIVSS</td>\n",
       "      <td>NWFDPWGQGTLVTVSS</td>\n",
       "      <td>QVQVQESGPGLVKPSGTLSLTCAVS</td>\n",
       "      <td>GGSMRSSNW</td>\n",
       "      <td>WSWVRQSPGKGLEWIGE</td>\n",
       "      <td>IHYSGNT</td>\n",
       "      <td>NYNPSLKSRVTMSVDKSENQFSLKLTSVTAADTAVYYC</td>\n",
       "      <td>WGQGSPVIVSS</td>\n",
       "      <td>ARVGHDILTGLNWFDP</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>GAAGCAGGTCATCCCT-1.CM003-1</td>\n",
       "      <td>GATATACTTTCTGAGACTCATGGACCTCCTGCACAAGAACATGAAA...</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>IGHV4-4*02</td>\n",
       "      <td>IGHD3-9*01</td>\n",
       "      <td>IGHJ6*02</td>\n",
       "      <td>CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>TGTGCGAGAGCGTCTTTCGATCTTTTGACTGGTTATGAGGGATTGG...</td>\n",
       "      <td>...</td>\n",
       "      <td>DVWGQGTTVTVSS</td>\n",
       "      <td>DVWGQGTTVTVSS</td>\n",
       "      <td>QVQLQESGPGLVKPSGTLSLTCAVS</td>\n",
       "      <td>DGSISSTNW</td>\n",
       "      <td>WSWVRQSPGKGLEWIGE</td>\n",
       "      <td>IYHSGST</td>\n",
       "      <td>KYNPSLKSRVTILVDKSKNQFSLKLNSVTAADTAVYYC</td>\n",
       "      <td>WGQGTTVTVSS</td>\n",
       "      <td>ARASFDLLTGYEGLDV</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>CGTTAGAGTTGCTCCT-1.CM003-5-9</td>\n",
       "      <td>GACATTTCCTTAAATTCAGGGTCCAGCTCACATGGGAAATACTTTC...</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>IGHV4-4*02</td>\n",
       "      <td>IGHD3-9*01</td>\n",
       "      <td>IGHJ6*02</td>\n",
       "      <td>CAGGTGCAGCTGCAGGAGTCGGGCCCA...GAACTGGTGAAGCCTT...</td>\n",
       "      <td>CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>TGTGCGAGGGCCTCTTTCGATATTTTGACTGGTTACTCCGGACTGG...</td>\n",
       "      <td>...</td>\n",
       "      <td>YSGLDVWGQGTTVTVSS</td>\n",
       "      <td>YYGMDVWGQGTTVTVSS</td>\n",
       "      <td>QVQLQESGPELVKPSGTLSLTCAVS</td>\n",
       "      <td>GGSITSTYW</td>\n",
       "      <td>WTWVRQPPGKGLEWIGE</td>\n",
       "      <td>IFHSGST</td>\n",
       "      <td>KYNPSLKSRVTFSVDKSKNQFSLKLTSVTDADTAVYYC</td>\n",
       "      <td>WGQGTTVTVSS</td>\n",
       "      <td>ARASFDILTGYSGLDV</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>CTACACCTCGTCCAGG-1.CM003-1</td>\n",
       "      <td>GGGAGGGTCCAGCTCACATGGGAAATACTTTCTGAGACTCATGGAC...</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>IGHV4-4*02</td>\n",
       "      <td>IGHD3-9*01</td>\n",
       "      <td>IGHJ6*02</td>\n",
       "      <td>CAGGTGCAGCTACAGGAGTCGGGTCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>TGTGCGAGAGCTTATTACGATGTTTTGACTGGTTATGACGGTTTGG...</td>\n",
       "      <td>...</td>\n",
       "      <td>GLDVWGLGTSVTVSS</td>\n",
       "      <td>GMDVWGQGTTVTVSS</td>\n",
       "      <td>QVQLQESGPGLVKPSGTLSLICAVS</td>\n",
       "      <td>GGSISSTNW</td>\n",
       "      <td>WSWVRQAPEKGLEWIGE</td>\n",
       "      <td>IYHSGST</td>\n",
       "      <td>RYNPSLKSRVTISVDKSKNQFSLKLTSVTAADTAVYYC</td>\n",
       "      <td>WGLGTSVTVSS</td>\n",
       "      <td>ARAYYDVLTGYDGLDV</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>GTACGTAGTTTACTCT-1.CM003-1</td>\n",
       "      <td>TGGGGATGCTTTCTGAGAGTCATGGATCTCATGTGCAAGAAAATGA...</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>IGHV4-39*01</td>\n",
       "      <td>IGHD3/OR15-3a*01,IGHD3/OR15-3b*01</td>\n",
       "      <td>IGHJ4*02</td>\n",
       "      <td>CAACTACAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>CAGCTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...</td>\n",
       "      <td>TGTGCGAGACAAGGCCGATTTTTGGACTGGTTATTGTACTTTGAGA...</td>\n",
       "      <td>...</td>\n",
       "      <td>YFENWGQGNLVTVSS</td>\n",
       "      <td>YFDYWGQGTLVTVSS</td>\n",
       "      <td>QLQLQESGPGLVKPSETLSLTCTVS</td>\n",
       "      <td>GDSVSSGDYY</td>\n",
       "      <td>WAWIRQPPGKGLEWIGT</td>\n",
       "      <td>IYYSGTT</td>\n",
       "      <td>YYNPSLKSRVTISVTTSKNQFSLKLNSVTATDTAVYYC</td>\n",
       "      <td>WGQGNLVTVSS</td>\n",
       "      <td>ARQGRFLDWLLYFEN</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314</th>\n",
       "      <td>ACTATCTCAGGGAGAG-1.CM003-5-9</td>\n",
       "      <td>TGGGAGTGCTTTCTGAGAGTCATGGACCTCCTGCACAAGAACATGA...</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>IGHV4-34*01</td>\n",
       "      <td>IGHD3-9*01</td>\n",
       "      <td>IGHJ4*02</td>\n",
       "      <td>CAGGTGCAACTACAGCAGTGGGGCGCA...GGACTGTTGAAGCCTT...</td>\n",
       "      <td>CAGGTGCAGCTACAGCAGTGGGGCGCA...GGACTGTTGAAGCCTT...</td>\n",
       "      <td>TGTGCGAGACAGGGTTATGATATTTTGACTGGGGTGGAGCGACACT...</td>\n",
       "      <td>...</td>\n",
       "      <td>FDYWGQGTVVTVSS</td>\n",
       "      <td>FDYWGQGTLVTVSS</td>\n",
       "      <td>QVQLQQWGAGLLKPSETLSLTCAVN</td>\n",
       "      <td>GGSFSGYY</td>\n",
       "      <td>WTWIRQSPGKGLEWIGE</td>\n",
       "      <td>INHSGSIT</td>\n",
       "      <td>NYSPSLKSRVTISVDTSKNQFSLKLSSVTAADTAVYFC</td>\n",
       "      <td>WGQGTVVTVSS</td>\n",
       "      <td>ARQGYDILTGVERHFDY</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>659</th>\n",
       "      <td>CGTCAGGAGTCAATAG-1.CM003-1</td>\n",
       "      <td>GACCCTGCAGCTCTGGGAGAGGAGCCCAGCACTAGAAGTCGGCGGT...</td>\n",
       "      <td>F</td>\n",
       "      <td>T</td>\n",
       "      <td>IGHV3-33*01</td>\n",
       "      <td>IGHD4-11*01,IGHD4-17*01,IGHD4-23*01</td>\n",
       "      <td>IGHJ6*02</td>\n",
       "      <td>CAGGTGCAGTTGCTGGAATCTGGGGGA...GGCGTGGTCCAGCCTG...</td>\n",
       "      <td>CAGGTGCAGCTGGTGGAGTCTGGGGGA...GGCGTGGTCCAGCCTG...</td>\n",
       "      <td>TGTGCGAGAGAGAGAGAGACTACCTTTTCCCACTACTACGGTATGG...</td>\n",
       "      <td>...</td>\n",
       "      <td>YYGMDVWGRGTTVTVSS</td>\n",
       "      <td>YYGMDVWGQGTTVTVSS</td>\n",
       "      <td>QVQLLESGGGVVQPGRSLRLSCAAS</td>\n",
       "      <td>GFTFSSYG</td>\n",
       "      <td>MHWVRQAPGKGLEWVTF</td>\n",
       "      <td>IWYDQNNK</td>\n",
       "      <td>YYADSVKGRFTISRDNSKKMLYLQMNSLRVEDTAVYYC</td>\n",
       "      <td>WGRGTTVTVSS</td>\n",
       "      <td>ARERETTFSHYYGMDV</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7 rows × 82 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      sequence_id  \\\n",
       "8    CACTCCAAGCTACCGC-1.CM003-5-9   \n",
       "125    GAAGCAGGTCATCCCT-1.CM003-1   \n",
       "158  CGTTAGAGTTGCTCCT-1.CM003-5-9   \n",
       "176    CTACACCTCGTCCAGG-1.CM003-1   \n",
       "198    GTACGTAGTTTACTCT-1.CM003-1   \n",
       "314  ACTATCTCAGGGAGAG-1.CM003-5-9   \n",
       "659    CGTCAGGAGTCAATAG-1.CM003-1   \n",
       "\n",
       "                                              sequence rev_comp productive  \\\n",
       "8    GGGGAGGGTCCAGCTCACATGGGAAAGACTTTCTGAGACTCATGGA...        F          T   \n",
       "125  GATATACTTTCTGAGACTCATGGACCTCCTGCACAAGAACATGAAA...        F          T   \n",
       "158  GACATTTCCTTAAATTCAGGGTCCAGCTCACATGGGAAATACTTTC...        F          T   \n",
       "176  GGGAGGGTCCAGCTCACATGGGAAATACTTTCTGAGACTCATGGAC...        F          T   \n",
       "198  TGGGGATGCTTTCTGAGAGTCATGGATCTCATGTGCAAGAAAATGA...        F          T   \n",
       "314  TGGGAGTGCTTTCTGAGAGTCATGGACCTCCTGCACAAGAACATGA...        F          T   \n",
       "659  GACCCTGCAGCTCTGGGAGAGGAGCCCAGCACTAGAAGTCGGCGGT...        F          T   \n",
       "\n",
       "          v_call                               d_call    j_call  \\\n",
       "8     IGHV4-4*02                           IGHD3-9*01  IGHJ5*02   \n",
       "125   IGHV4-4*02                           IGHD3-9*01  IGHJ6*02   \n",
       "158   IGHV4-4*02                           IGHD3-9*01  IGHJ6*02   \n",
       "176   IGHV4-4*02                           IGHD3-9*01  IGHJ6*02   \n",
       "198  IGHV4-39*01    IGHD3/OR15-3a*01,IGHD3/OR15-3b*01  IGHJ4*02   \n",
       "314  IGHV4-34*01                           IGHD3-9*01  IGHJ4*02   \n",
       "659  IGHV3-33*01  IGHD4-11*01,IGHD4-17*01,IGHD4-23*01  IGHJ6*02   \n",
       "\n",
       "                                    sequence_alignment  \\\n",
       "8    CAGGTGCAGGTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...   \n",
       "125  CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...   \n",
       "158  CAGGTGCAGCTGCAGGAGTCGGGCCCA...GAACTGGTGAAGCCTT...   \n",
       "176  CAGGTGCAGCTACAGGAGTCGGGTCCA...GGACTGGTGAAGCCTT...   \n",
       "198  CAACTACAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...   \n",
       "314  CAGGTGCAACTACAGCAGTGGGGCGCA...GGACTGTTGAAGCCTT...   \n",
       "659  CAGGTGCAGTTGCTGGAATCTGGGGGA...GGCGTGGTCCAGCCTG...   \n",
       "\n",
       "                                    germline_alignment  \\\n",
       "8    CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...   \n",
       "125  CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...   \n",
       "158  CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...   \n",
       "176  CAGGTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...   \n",
       "198  CAGCTGCAGCTGCAGGAGTCGGGCCCA...GGACTGGTGAAGCCTT...   \n",
       "314  CAGGTGCAGCTACAGCAGTGGGGCGCA...GGACTGTTGAAGCCTT...   \n",
       "659  CAGGTGCAGCTGGTGGAGTCTGGGGGA...GGCGTGGTCCAGCCTG...   \n",
       "\n",
       "                                              junction  ...  \\\n",
       "8    TGTGCGAGAGTTGGTCACGATATTTTGACTGGCCTCAACTGGTTCG...  ...   \n",
       "125  TGTGCGAGAGCGTCTTTCGATCTTTTGACTGGTTATGAGGGATTGG...  ...   \n",
       "158  TGTGCGAGGGCCTCTTTCGATATTTTGACTGGTTACTCCGGACTGG...  ...   \n",
       "176  TGTGCGAGAGCTTATTACGATGTTTTGACTGGTTATGACGGTTTGG...  ...   \n",
       "198  TGTGCGAGACAAGGCCGATTTTTGGACTGGTTATTGTACTTTGAGA...  ...   \n",
       "314  TGTGCGAGACAGGGTTATGATATTTTGACTGGGGTGGAGCGACACT...  ...   \n",
       "659  TGTGCGAGAGAGAGAGAGACTACCTTTTCCCACTACTACGGTATGG...  ...   \n",
       "\n",
       "    j_sequence_alignment_aa j_germline_alignment_aa  \\\n",
       "8          NWFDPWGQGSPVIVSS        NWFDPWGQGTLVTVSS   \n",
       "125           DVWGQGTTVTVSS           DVWGQGTTVTVSS   \n",
       "158       YSGLDVWGQGTTVTVSS       YYGMDVWGQGTTVTVSS   \n",
       "176         GLDVWGLGTSVTVSS         GMDVWGQGTTVTVSS   \n",
       "198         YFENWGQGNLVTVSS         YFDYWGQGTLVTVSS   \n",
       "314          FDYWGQGTVVTVSS          FDYWGQGTLVTVSS   \n",
       "659       YYGMDVWGRGTTVTVSS       YYGMDVWGQGTTVTVSS   \n",
       "\n",
       "                       fwr1_aa     cdr1_aa            fwr2_aa   cdr2_aa  \\\n",
       "8    QVQVQESGPGLVKPSGTLSLTCAVS   GGSMRSSNW  WSWVRQSPGKGLEWIGE   IHYSGNT   \n",
       "125  QVQLQESGPGLVKPSGTLSLTCAVS   DGSISSTNW  WSWVRQSPGKGLEWIGE   IYHSGST   \n",
       "158  QVQLQESGPELVKPSGTLSLTCAVS   GGSITSTYW  WTWVRQPPGKGLEWIGE   IFHSGST   \n",
       "176  QVQLQESGPGLVKPSGTLSLICAVS   GGSISSTNW  WSWVRQAPEKGLEWIGE   IYHSGST   \n",
       "198  QLQLQESGPGLVKPSETLSLTCTVS  GDSVSSGDYY  WAWIRQPPGKGLEWIGT   IYYSGTT   \n",
       "314  QVQLQQWGAGLLKPSETLSLTCAVN    GGSFSGYY  WTWIRQSPGKGLEWIGE  INHSGSIT   \n",
       "659  QVQLLESGGGVVQPGRSLRLSCAAS    GFTFSSYG  MHWVRQAPGKGLEWVTF  IWYDQNNK   \n",
       "\n",
       "                                    fwr3_aa      fwr4_aa            cdr3_aa  \\\n",
       "8    NYNPSLKSRVTMSVDKSENQFSLKLTSVTAADTAVYYC  WGQGSPVIVSS   ARVGHDILTGLNWFDP   \n",
       "125  KYNPSLKSRVTILVDKSKNQFSLKLNSVTAADTAVYYC  WGQGTTVTVSS   ARASFDLLTGYEGLDV   \n",
       "158  KYNPSLKSRVTFSVDKSKNQFSLKLTSVTDADTAVYYC  WGQGTTVTVSS   ARASFDILTGYSGLDV   \n",
       "176  RYNPSLKSRVTISVDKSKNQFSLKLTSVTAADTAVYYC  WGLGTSVTVSS   ARAYYDVLTGYDGLDV   \n",
       "198  YYNPSLKSRVTISVTTSKNQFSLKLNSVTATDTAVYYC  WGQGNLVTVSS    ARQGRFLDWLLYFEN   \n",
       "314  NYSPSLKSRVTISVDTSKNQFSLKLSSVTAADTAVYFC  WGQGTVVTVSS  ARQGYDILTGVERHFDY   \n",
       "659  YYADSVKGRFTISRDNSKKMLYLQMNSLRVEDTAVYYC  WGRGTTVTVSS   ARERETTFSHYYGMDV   \n",
       "\n",
       "     junction_aa_length  \n",
       "8                    18  \n",
       "125                  18  \n",
       "158                  18  \n",
       "176                  18  \n",
       "198                  17  \n",
       "314                  19  \n",
       "659                  18  \n",
       "\n",
       "[7 rows x 82 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df31.loc[df31.sequence_alignment_aa.isin(seq_aa.keys()), :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0e65c6d-fdd1-4f7b-a2ee-22183e2e171b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:imnet]",
   "language": "python",
   "name": "conda-env-imnet-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}