lecture9.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lGA6-x1-1sSn"
   },
   "source": [
    "## Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "4oHX4JCO1sSp",
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Bad key savefig.frameon in file /home/bang/anaconda3/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle, line 421 ('savefig.frameon : True')\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "https://github.com/matplotlib/matplotlib/blob/v3.4.2/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key verbose.level in file /home/bang/anaconda3/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle, line 472 ('verbose.level  : silent      # one of silent, helpful, debug, debug-annoying')\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "https://github.com/matplotlib/matplotlib/blob/v3.4.2/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key verbose.fileo in file /home/bang/anaconda3/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle, line 473 ('verbose.fileo  : sys.stdout  # a log filename, sys.stdout or sys.stderr')\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "https://github.com/matplotlib/matplotlib/blob/v3.4.2/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "In /home/bang/anaconda3/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: \n",
      "The text.latex.preview rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.\n",
      "In /home/bang/anaconda3/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: \n",
      "The mathtext.fallback_to_cm rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.\n",
      "In /home/bang/anaconda3/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: Support for setting the 'mathtext.fallback_to_cm' rcParam is deprecated since 3.3 and will be removed two minor releases later; use 'mathtext.fallback : 'cm' instead.\n",
      "In /home/bang/anaconda3/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: \n",
      "The validate_bool_maybe_none function was deprecated in Matplotlib 3.3 and will be removed two minor releases later.\n",
      "In /home/bang/anaconda3/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: \n",
      "The savefig.jpeg_quality rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.\n",
      "In /home/bang/anaconda3/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: \n",
      "The keymap.all_axes rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.\n",
      "In /home/bang/anaconda3/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: \n",
      "The animation.avconv_path rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.\n",
      "In /home/bang/anaconda3/lib/python3.8/site-packages/matplotlib/mpl-data/stylelib/_classic_test.mplstyle: \n",
      "The animation.avconv_args rcparam was deprecated in Matplotlib 3.3 and will be removed two minor releases later.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-MIBcOvz1sSr"
   },
   "source": [
    "## Get the Data\n",
    "\n",
    "We'll use the built in breast cancer dataset from Scikit Learn. We can get with the load function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "lmHFsZJ01sSr",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_breast_cancer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "Oy4KmNEc1sSr",
    "tags": []
   },
   "outputs": [],
   "source": [
    "cancer = load_breast_cancer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ULt5f1ke1sSs"
   },
   "source": [
    "The data set is presented in a dictionary form:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "aO-znbOk1sSs",
    "outputId": "3095deb2-ba5d-4bc3-8e4e-64e26871b7b0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['data', 'target', 'frame', 'target_names', 'DESCR', 'feature_names', 'filename'])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cancer.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "J0SzCQqV1sSt"
   },
   "source": [
    "We can grab information and arrays out of this dictionary to set up our data frame and understanding of the features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "FJPM4vGb1sSt",
    "outputId": "7466094a-076a-4dec-dc85-081ba3d63a9e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".. _breast_cancer_dataset:\n",
      "\n",
      "Breast cancer wisconsin (diagnostic) dataset\n",
      "--------------------------------------------\n",
      "\n",
      "**Data Set Characteristics:**\n",
      "\n",
      "    :Number of Instances: 569\n",
      "\n",
      "    :Number of Attributes: 30 numeric, predictive attributes and the class\n",
      "\n",
      "    :Attribute Information:\n",
      "        - radius (mean of distances from center to points on the perimeter)\n",
      "        - texture (standard deviation of gray-scale values)\n",
      "        - perimeter\n",
      "        - area\n",
      "        - smoothness (local variation in radius lengths)\n",
      "        - compactness (perimeter^2 / area - 1.0)\n",
      "        - concavity (severity of concave portions of the contour)\n",
      "        - concave points (number of concave portions of the contour)\n",
      "        - symmetry\n",
      "        - fractal dimension (\"coastline approximation\" - 1)\n",
      "\n",
      "        The mean, standard error, and \"worst\" or largest (mean of the three\n",
      "        worst/largest values) of these features were computed for each image,\n",
      "        resulting in 30 features.  For instance, field 0 is Mean Radius, field\n",
      "        10 is Radius SE, field 20 is Worst Radius.\n",
      "\n",
      "        - class:\n",
      "                - WDBC-Malignant\n",
      "                - WDBC-Benign\n",
      "\n",
      "    :Summary Statistics:\n",
      "\n",
      "    ===================================== ====== ======\n",
      "                                           Min    Max\n",
      "    ===================================== ====== ======\n",
      "    radius (mean):                        6.981  28.11\n",
      "    texture (mean):                       9.71   39.28\n",
      "    perimeter (mean):                     43.79  188.5\n",
      "    area (mean):                          143.5  2501.0\n",
      "    smoothness (mean):                    0.053  0.163\n",
      "    compactness (mean):                   0.019  0.345\n",
      "    concavity (mean):                     0.0    0.427\n",
      "    concave points (mean):                0.0    0.201\n",
      "    symmetry (mean):                      0.106  0.304\n",
      "    fractal dimension (mean):             0.05   0.097\n",
      "    radius (standard error):              0.112  2.873\n",
      "    texture (standard error):             0.36   4.885\n",
      "    perimeter (standard error):           0.757  21.98\n",
      "    area (standard error):                6.802  542.2\n",
      "    smoothness (standard error):          0.002  0.031\n",
      "    compactness (standard error):         0.002  0.135\n",
      "    concavity (standard error):           0.0    0.396\n",
      "    concave points (standard error):      0.0    0.053\n",
      "    symmetry (standard error):            0.008  0.079\n",
      "    fractal dimension (standard error):   0.001  0.03\n",
      "    radius (worst):                       7.93   36.04\n",
      "    texture (worst):                      12.02  49.54\n",
      "    perimeter (worst):                    50.41  251.2\n",
      "    area (worst):                         185.2  4254.0\n",
      "    smoothness (worst):                   0.071  0.223\n",
      "    compactness (worst):                  0.027  1.058\n",
      "    concavity (worst):                    0.0    1.252\n",
      "    concave points (worst):               0.0    0.291\n",
      "    symmetry (worst):                     0.156  0.664\n",
      "    fractal dimension (worst):            0.055  0.208\n",
      "    ===================================== ====== ======\n",
      "\n",
      "    :Missing Attribute Values: None\n",
      "\n",
      "    :Class Distribution: 212 - Malignant, 357 - Benign\n",
      "\n",
      "    :Creator:  Dr. William H. Wolberg, W. Nick Street, Olvi L. Mangasarian\n",
      "\n",
      "    :Donor: Nick Street\n",
      "\n",
      "    :Date: November, 1995\n",
      "\n",
      "This is a copy of UCI ML Breast Cancer Wisconsin (Diagnostic) datasets.\n",
      "https://goo.gl/U2Uwz2\n",
      "\n",
      "Features are computed from a digitized image of a fine needle\n",
      "aspirate (FNA) of a breast mass.  They describe\n",
      "characteristics of the cell nuclei present in the image.\n",
      "\n",
      "Separating plane described above was obtained using\n",
      "Multisurface Method-Tree (MSM-T) [K. P. Bennett, \"Decision Tree\n",
      "Construction Via Linear Programming.\" Proceedings of the 4th\n",
      "Midwest Artificial Intelligence and Cognitive Science Society,\n",
      "pp. 97-101, 1992], a classification method which uses linear\n",
      "programming to construct a decision tree.  Relevant features\n",
      "were selected using an exhaustive search in the space of 1-4\n",
      "features and 1-3 separating planes.\n",
      "\n",
      "The actual linear program used to obtain the separating plane\n",
      "in the 3-dimensional space is that described in:\n",
      "[K. P. Bennett and O. L. Mangasarian: \"Robust Linear\n",
      "Programming Discrimination of Two Linearly Inseparable Sets\",\n",
      "Optimization Methods and Software 1, 1992, 23-34].\n",
      "\n",
      "This database is also available through the UW CS ftp server:\n",
      "\n",
      "ftp ftp.cs.wisc.edu\n",
      "cd math-prog/cpo-dataset/machine-learn/WDBC/\n",
      "\n",
      ".. topic:: References\n",
      "\n",
      "   - W.N. Street, W.H. Wolberg and O.L. Mangasarian. Nuclear feature extraction \n",
      "     for breast tumor diagnosis. IS&T/SPIE 1993 International Symposium on \n",
      "     Electronic Imaging: Science and Technology, volume 1905, pages 861-870,\n",
      "     San Jose, CA, 1993.\n",
      "   - O.L. Mangasarian, W.N. Street and W.H. Wolberg. Breast cancer diagnosis and \n",
      "     prognosis via linear programming. Operations Research, 43(4), pages 570-577, \n",
      "     July-August 1995.\n",
      "   - W.H. Wolberg, W.N. Street, and O.L. Mangasarian. Machine learning techniques\n",
      "     to diagnose breast cancer from fine-needle aspirates. Cancer Letters 77 (1994) \n",
      "     163-171.\n"
     ]
    }
   ],
   "source": [
    "print(cancer['DESCR'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xv0lCqh01sSu",
    "outputId": "8358ca74-b678-4e88-b7f0-9af9e4de7c48"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['mean radius', 'mean texture', 'mean perimeter', 'mean area',\n",
       "       'mean smoothness', 'mean compactness', 'mean concavity',\n",
       "       'mean concave points', 'mean symmetry', 'mean fractal dimension',\n",
       "       'radius error', 'texture error', 'perimeter error', 'area error',\n",
       "       'smoothness error', 'compactness error', 'concavity error',\n",
       "       'concave points error', 'symmetry error',\n",
       "       'fractal dimension error', 'worst radius', 'worst texture',\n",
       "       'worst perimeter', 'worst area', 'worst smoothness',\n",
       "       'worst compactness', 'worst concavity', 'worst concave points',\n",
       "       'worst symmetry', 'worst fractal dimension'], dtype='<U23')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cancer['feature_names']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_0lA2o5O1sSv"
   },
   "source": [
    "## Set up DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sxpnPtWY1sSv",
    "outputId": "224f81b3-8750-4f44-f72d-e3e2267ec10c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 569 entries, 0 to 568\n",
      "Data columns (total 30 columns):\n",
      " #   Column                   Non-Null Count  Dtype  \n",
      "---  ------                   --------------  -----  \n",
      " 0   mean radius              569 non-null    float64\n",
      " 1   mean texture             569 non-null    float64\n",
      " 2   mean perimeter           569 non-null    float64\n",
      " 3   mean area                569 non-null    float64\n",
      " 4   mean smoothness          569 non-null    float64\n",
      " 5   mean compactness         569 non-null    float64\n",
      " 6   mean concavity           569 non-null    float64\n",
      " 7   mean concave points      569 non-null    float64\n",
      " 8   mean symmetry            569 non-null    float64\n",
      " 9   mean fractal dimension   569 non-null    float64\n",
      " 10  radius error             569 non-null    float64\n",
      " 11  texture error            569 non-null    float64\n",
      " 12  perimeter error          569 non-null    float64\n",
      " 13  area error               569 non-null    float64\n",
      " 14  smoothness error         569 non-null    float64\n",
      " 15  compactness error        569 non-null    float64\n",
      " 16  concavity error          569 non-null    float64\n",
      " 17  concave points error     569 non-null    float64\n",
      " 18  symmetry error           569 non-null    float64\n",
      " 19  fractal dimension error  569 non-null    float64\n",
      " 20  worst radius             569 non-null    float64\n",
      " 21  worst texture            569 non-null    float64\n",
      " 22  worst perimeter          569 non-null    float64\n",
      " 23  worst area               569 non-null    float64\n",
      " 24  worst smoothness         569 non-null    float64\n",
      " 25  worst compactness        569 non-null    float64\n",
      " 26  worst concavity          569 non-null    float64\n",
      " 27  worst concave points     569 non-null    float64\n",
      " 28  worst symmetry           569 non-null    float64\n",
      " 29  worst fractal dimension  569 non-null    float64\n",
      "dtypes: float64(30)\n",
      "memory usage: 133.5 KB\n"
     ]
    }
   ],
   "source": [
    "df_feat = pd.DataFrame(cancer['data'], columns=cancer['feature_names'])\n",
    "df_feat.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 202
    },
    "id": "16DaSmGK1sSw",
    "outputId": "a56b7d37-bc6a-439d-e344-0623d625485e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Target\n",
       "0       0\n",
       "1       0\n",
       "2       0\n",
       "3       0\n",
       "4       0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_target = pd.DataFrame(cancer['target'], columns=['Target'])\n",
    "df_target.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_-3OAMXd1sSx"
   },
   "source": [
    "## Train Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "OJcbHsnI1sSx",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "JK92WYcw1sSx"
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    df_feat, np.ravel(df_target),\n",
    "    test_size=0.30, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qEyFojl81sSx"
   },
   "source": [
    "# Train the Support Vector Classifier\n",
    "**[TODO]** Train the SVM Classifier with `gamma='auto` using `X_train` and `y_train`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hS4bjaVa1sSx",
    "outputId": "e144e2bf-7230-4e4c-8875-7b6203f7e098",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(gamma='auto')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "model = SVC(gamma='auto')\n",
    "\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KBru00IU1sSy"
   },
   "source": [
    "## Predictions and Evaluations\n",
    "\n",
    "**[TODO]** Now let's predict using the trained model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "kkvV7kq91sSy"
   },
   "outputs": [],
   "source": [
    "predictions = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "EuFDcY4V1sSy",
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "437Y_77d1sSy",
    "outputId": "4b96b68d-3bc1-42fe-aa04-c98e702dd4ad"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  0  63]\n",
      " [  0 108]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_test, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hvrGiaCL1sSz",
    "outputId": "ed3f0381-b22f-4b3e-b080-2078ce3b2564"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00        63\n",
      "           1       0.63      1.00      0.77       108\n",
      "\n",
      "    accuracy                           0.63       171\n",
      "   macro avg       0.32      0.50      0.39       171\n",
      "weighted avg       0.40      0.63      0.49       171\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bang/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/bang/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/bang/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, predictions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ViJFWNG41sSz"
   },
   "source": [
    "Woah! Notice that we are classifying everything into a single class! This means our model needs to have it parameters adjusted (it may also help to normalize the data).\n",
    "\n",
    "We can search for parameters using a GridSearch!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wbWcgdzG1sSz"
   },
   "source": [
    "# Gridsearch\n",
    "\n",
    "Finding the right parameters (like what `C` or `gamma values` to use) is a tricky task!\n",
    "\n",
    "But luckily, we can be a little lazy and just try a bunch of combinations and see what works best! This idea of creating a `grid` of parameters and just trying out all the possible combinations is called a `Gridsearch`, this method is common enough that Scikit-learn has this functionality built in with [`GridSearchCV`](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html)! The CV stands for cross-validation.\n",
    "\n",
    "The `GridSearchCV` takes a dictionary that describes the parameters that should be tried and a model to train. The grid of parameters is defined as a dictionary, where the keys are the parameters and the values are the settings to be tested. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "5N2ATiTA1sSz",
    "tags": []
   },
   "outputs": [],
   "source": [
    "param_grid = {'C': [0.1, 1, 10, 100, 1000],\n",
    "              'gamma': [1, 0.1, 0.01, 0.001, 0.0001],\n",
    "              'kernel': ['rbf']} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fWUJ9q-O1sSz",
    "outputId": "f11bf55e-24de-44a9-a8f7-a0bf9a358613",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 25 candidates, totalling 125 fits\n",
      "[CV 1/5] END ........C=0.1, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 2/5] END ........C=0.1, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END ........C=0.1, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END ........C=0.1, gamma=1, kernel=rbf;, score=0.633 total time=   0.0s\n",
      "[CV 5/5] END ........C=0.1, gamma=1, kernel=rbf;, score=0.620 total time=   0.0s\n",
      "[CV 1/5] END ......C=0.1, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 2/5] END ......C=0.1, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END ......C=0.1, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END ......C=0.1, gamma=0.1, kernel=rbf;, score=0.633 total time=   0.0s\n",
      "[CV 5/5] END ......C=0.1, gamma=0.1, kernel=rbf;, score=0.620 total time=   0.0s\n",
      "[CV 1/5] END .....C=0.1, gamma=0.01, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 2/5] END .....C=0.1, gamma=0.01, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END .....C=0.1, gamma=0.01, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END .....C=0.1, gamma=0.01, kernel=rbf;, score=0.633 total time=   0.0s\n",
      "[CV 5/5] END .....C=0.1, gamma=0.01, kernel=rbf;, score=0.620 total time=   0.0s\n",
      "[CV 1/5] END ....C=0.1, gamma=0.001, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 2/5] END ....C=0.1, gamma=0.001, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END ....C=0.1, gamma=0.001, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END ....C=0.1, gamma=0.001, kernel=rbf;, score=0.633 total time=   0.0s\n",
      "[CV 5/5] END ....C=0.1, gamma=0.001, kernel=rbf;, score=0.620 total time=   0.0s\n",
      "[CV 1/5] END ...C=0.1, gamma=0.0001, kernel=rbf;, score=0.912 total time=   0.0s\n",
      "[CV 2/5] END ...C=0.1, gamma=0.0001, kernel=rbf;, score=0.850 total time=   0.0s\n",
      "[CV 3/5] END ...C=0.1, gamma=0.0001, kernel=rbf;, score=0.950 total time=   0.0s\n",
      "[CV 4/5] END ...C=0.1, gamma=0.0001, kernel=rbf;, score=0.899 total time=   0.0s\n",
      "[CV 5/5] END ...C=0.1, gamma=0.0001, kernel=rbf;, score=0.937 total time=   0.0s\n",
      "[CV 1/5] END ..........C=1, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 2/5] END ..........C=1, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END ..........C=1, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END ..........C=1, gamma=1, kernel=rbf;, score=0.633 total time=   0.0s\n",
      "[CV 5/5] END ..........C=1, gamma=1, kernel=rbf;, score=0.620 total time=   0.0s\n",
      "[CV 1/5] END ........C=1, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 2/5] END ........C=1, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END ........C=1, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END ........C=1, gamma=0.1, kernel=rbf;, score=0.633 total time=   0.0s\n",
      "[CV 5/5] END ........C=1, gamma=0.1, kernel=rbf;, score=0.620 total time=   0.0s\n",
      "[CV 1/5] END .......C=1, gamma=0.01, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 2/5] END .......C=1, gamma=0.01, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END .......C=1, gamma=0.01, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END .......C=1, gamma=0.01, kernel=rbf;, score=0.633 total time=   0.0s\n",
      "[CV 5/5] END .......C=1, gamma=0.01, kernel=rbf;, score=0.620 total time=   0.0s\n",
      "[CV 1/5] END ......C=1, gamma=0.001, kernel=rbf;, score=0.950 total time=   0.0s\n",
      "[CV 2/5] END ......C=1, gamma=0.001, kernel=rbf;, score=0.887 total time=   0.0s\n",
      "[CV 3/5] END ......C=1, gamma=0.001, kernel=rbf;, score=0.925 total time=   0.0s\n",
      "[CV 4/5] END ......C=1, gamma=0.001, kernel=rbf;, score=0.911 total time=   0.0s\n",
      "[CV 5/5] END ......C=1, gamma=0.001, kernel=rbf;, score=0.899 total time=   0.0s\n",
      "[CV 1/5] END .....C=1, gamma=0.0001, kernel=rbf;, score=0.950 total time=   0.0s\n",
      "[CV 2/5] END .....C=1, gamma=0.0001, kernel=rbf;, score=0.912 total time=   0.0s\n",
      "[CV 3/5] END .....C=1, gamma=0.0001, kernel=rbf;, score=0.963 total time=   0.0s\n",
      "[CV 4/5] END .....C=1, gamma=0.0001, kernel=rbf;, score=0.911 total time=   0.0s\n",
      "[CV 5/5] END .....C=1, gamma=0.0001, kernel=rbf;, score=0.949 total time=   0.0s\n",
      "[CV 1/5] END .........C=10, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 2/5] END .........C=10, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END .........C=10, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END .........C=10, gamma=1, kernel=rbf;, score=0.633 total time=   0.0s\n",
      "[CV 5/5] END .........C=10, gamma=1, kernel=rbf;, score=0.620 total time=   0.0s\n",
      "[CV 1/5] END .......C=10, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 2/5] END .......C=10, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END .......C=10, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END .......C=10, gamma=0.1, kernel=rbf;, score=0.633 total time=   0.0s\n",
      "[CV 5/5] END .......C=10, gamma=0.1, kernel=rbf;, score=0.620 total time=   0.0s\n",
      "[CV 1/5] END ......C=10, gamma=0.01, kernel=rbf;, score=0.637 total time=   0.0s\n",
      "[CV 2/5] END ......C=10, gamma=0.01, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END ......C=10, gamma=0.01, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END ......C=10, gamma=0.01, kernel=rbf;, score=0.646 total time=   0.0s\n",
      "[CV 5/5] END ......C=10, gamma=0.01, kernel=rbf;, score=0.608 total time=   0.0s\n",
      "[CV 1/5] END .....C=10, gamma=0.001, kernel=rbf;, score=0.950 total time=   0.0s\n",
      "[CV 2/5] END .....C=10, gamma=0.001, kernel=rbf;, score=0.887 total time=   0.0s\n",
      "[CV 3/5] END .....C=10, gamma=0.001, kernel=rbf;, score=0.887 total time=   0.0s\n",
      "[CV 4/5] END .....C=10, gamma=0.001, kernel=rbf;, score=0.937 total time=   0.0s\n",
      "[CV 5/5] END .....C=10, gamma=0.001, kernel=rbf;, score=0.924 total time=   0.0s\n",
      "[CV 1/5] END ....C=10, gamma=0.0001, kernel=rbf;, score=0.963 total time=   0.0s\n",
      "[CV 2/5] END ....C=10, gamma=0.0001, kernel=rbf;, score=0.900 total time=   0.0s\n",
      "[CV 3/5] END ....C=10, gamma=0.0001, kernel=rbf;, score=0.963 total time=   0.0s\n",
      "[CV 4/5] END ....C=10, gamma=0.0001, kernel=rbf;, score=0.924 total time=   0.0s\n",
      "[CV 5/5] END ....C=10, gamma=0.0001, kernel=rbf;, score=0.911 total time=   0.0s\n",
      "[CV 1/5] END ........C=100, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 2/5] END ........C=100, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END ........C=100, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END ........C=100, gamma=1, kernel=rbf;, score=0.633 total time=   0.0s\n",
      "[CV 5/5] END ........C=100, gamma=1, kernel=rbf;, score=0.620 total time=   0.0s\n",
      "[CV 1/5] END ......C=100, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 2/5] END ......C=100, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END ......C=100, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END ......C=100, gamma=0.1, kernel=rbf;, score=0.633 total time=   0.0s\n",
      "[CV 5/5] END ......C=100, gamma=0.1, kernel=rbf;, score=0.620 total time=   0.0s\n",
      "[CV 1/5] END .....C=100, gamma=0.01, kernel=rbf;, score=0.637 total time=   0.0s\n",
      "[CV 2/5] END .....C=100, gamma=0.01, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END .....C=100, gamma=0.01, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END .....C=100, gamma=0.01, kernel=rbf;, score=0.646 total time=   0.0s\n",
      "[CV 5/5] END .....C=100, gamma=0.01, kernel=rbf;, score=0.608 total time=   0.0s\n",
      "[CV 1/5] END ....C=100, gamma=0.001, kernel=rbf;, score=0.950 total time=   0.0s\n",
      "[CV 2/5] END ....C=100, gamma=0.001, kernel=rbf;, score=0.887 total time=   0.0s\n",
      "[CV 3/5] END ....C=100, gamma=0.001, kernel=rbf;, score=0.887 total time=   0.0s\n",
      "[CV 4/5] END ....C=100, gamma=0.001, kernel=rbf;, score=0.937 total time=   0.0s\n",
      "[CV 5/5] END ....C=100, gamma=0.001, kernel=rbf;, score=0.924 total time=   0.0s\n",
      "[CV 1/5] END ...C=100, gamma=0.0001, kernel=rbf;, score=0.950 total time=   0.0s\n",
      "[CV 2/5] END ...C=100, gamma=0.0001, kernel=rbf;, score=0.900 total time=   0.0s\n",
      "[CV 3/5] END ...C=100, gamma=0.0001, kernel=rbf;, score=0.938 total time=   0.0s\n",
      "[CV 4/5] END ...C=100, gamma=0.0001, kernel=rbf;, score=0.924 total time=   0.0s\n",
      "[CV 5/5] END ...C=100, gamma=0.0001, kernel=rbf;, score=0.924 total time=   0.0s\n",
      "[CV 1/5] END .......C=1000, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 2/5] END .......C=1000, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END .......C=1000, gamma=1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END .......C=1000, gamma=1, kernel=rbf;, score=0.633 total time=   0.0s\n",
      "[CV 5/5] END .......C=1000, gamma=1, kernel=rbf;, score=0.620 total time=   0.0s\n",
      "[CV 1/5] END .....C=1000, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 2/5] END .....C=1000, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END .....C=1000, gamma=0.1, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END .....C=1000, gamma=0.1, kernel=rbf;, score=0.633 total time=   0.0s\n",
      "[CV 5/5] END .....C=1000, gamma=0.1, kernel=rbf;, score=0.620 total time=   0.0s\n",
      "[CV 1/5] END ....C=1000, gamma=0.01, kernel=rbf;, score=0.637 total time=   0.0s\n",
      "[CV 2/5] END ....C=1000, gamma=0.01, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 3/5] END ....C=1000, gamma=0.01, kernel=rbf;, score=0.625 total time=   0.0s\n",
      "[CV 4/5] END ....C=1000, gamma=0.01, kernel=rbf;, score=0.646 total time=   0.0s\n",
      "[CV 5/5] END ....C=1000, gamma=0.01, kernel=rbf;, score=0.608 total time=   0.0s\n",
      "[CV 1/5] END ...C=1000, gamma=0.001, kernel=rbf;, score=0.950 total time=   0.0s\n",
      "[CV 2/5] END ...C=1000, gamma=0.001, kernel=rbf;, score=0.887 total time=   0.0s\n",
      "[CV 3/5] END ...C=1000, gamma=0.001, kernel=rbf;, score=0.887 total time=   0.0s\n",
      "[CV 4/5] END ...C=1000, gamma=0.001, kernel=rbf;, score=0.937 total time=   0.0s\n",
      "[CV 5/5] END ...C=1000, gamma=0.001, kernel=rbf;, score=0.924 total time=   0.0s\n",
      "[CV 1/5] END ..C=1000, gamma=0.0001, kernel=rbf;, score=0.950 total time=   0.0s\n",
      "[CV 2/5] END ..C=1000, gamma=0.0001, kernel=rbf;, score=0.875 total time=   0.0s\n",
      "[CV 3/5] END ..C=1000, gamma=0.0001, kernel=rbf;, score=0.938 total time=   0.0s\n",
      "[CV 4/5] END ..C=1000, gamma=0.0001, kernel=rbf;, score=0.911 total time=   0.0s\n",
      "[CV 5/5] END ..C=1000, gamma=0.0001, kernel=rbf;, score=0.899 total time=   0.0s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=SVC(),\n",
       "             param_grid={'C': [0.1, 1, 10, 100, 1000],\n",
       "                         'gamma': [1, 0.1, 0.01, 0.001, 0.0001],\n",
       "                         'kernel': ['rbf']},\n",
       "             verbose=3)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "grid = GridSearchCV(SVC(), param_grid, refit=True, verbose=3)\n",
    "\n",
    "# May take awhile!\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "C_3c0GXn1sS0"
   },
   "source": [
    "You can inspect the best parameters found by GridSearchCV in the best_params_ attribute, and the best estimator in the best\\_estimator_ attribute:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yIcaKkYK1sS0",
    "outputId": "e989604e-3ad6-4bc5-f2f6-69de131fc8be"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 1, 'gamma': 0.0001, 'kernel': 'rbf'}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dbSaiZkb1sS1",
    "outputId": "f37f49b9-1c86-4bb2-cd24-d4f43493808b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1, gamma=0.0001)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-TttgAlY1sS1"
   },
   "source": [
    "Then you can re-run predictions on this grid object just like you would with a normal model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "Rfs9ThDm1sS1"
   },
   "outputs": [],
   "source": [
    "grid_predictions = grid.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "NlW2hZXc1sS1",
    "outputId": "ced2657f-99a5-48c0-91a9-32ee37cb593b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 59   4]\n",
      " [  3 105]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_test, grid_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "XzTwjpef1sS1",
    "outputId": "90ec5a00-451d-4b5b-f3dd-532472525418"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.94      0.94        63\n",
      "           1       0.96      0.97      0.97       108\n",
      "\n",
      "    accuracy                           0.96       171\n",
      "   macro avg       0.96      0.95      0.96       171\n",
      "weighted avg       0.96      0.96      0.96       171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,grid_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Lecture 9.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}