Lesson 16, Optimizing Network Training (Optimized).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "j9Gv6-w0oS1G"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os\n",
    "import cv2\n",
    "import zipfile\n",
    "from functools import partial\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "tf.config.experimental.set_memory_growth(gpus[0], True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "kr_PD5lezYvz"
   },
   "outputs": [],
   "source": [
    "#! mkdir ~/.kaggle\n",
    "#! cp kaggle.json ~/.kaggle/\n",
    "#! chmod 600 ~/.kaggle/kaggle.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kx8WjSkWziAi",
    "outputId": "e2719cbe-3de6-40c0-eeb2-895c60b113b7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "401 - Unauthorized\n"
     ]
    }
   ],
   "source": [
    "! kaggle datasets download -d puneet6060/intel-image-classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "jE6IMoy9ziDB"
   },
   "outputs": [],
   "source": [
    "if not os.path.exists('./intel-image-classification/'):\n",
    "    with zipfile.ZipFile(\"intel-image-classification.zip\",\"r\") as zip_ref:\n",
    "        zip_ref.extractall(\"intel-image-classification\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "5UUJEE57ziG2"
   },
   "outputs": [],
   "source": [
    "ROOT_PATH = 'intel-image-classification'\n",
    "\n",
    "config = {\n",
    "\n",
    "    'TRAIN_PATH': os.path.join(ROOT_PATH, 'seg_train', 'seg_train'),\n",
    "    'TEST_PATH': os.path.join(ROOT_PATH, 'seg_test', 'seg_test'),\n",
    "    'VALID_PATH': os.path.join(ROOT_PATH, 'seg_pred', 'seg_pred'),\n",
    "    'BATCH_SIZE' :16,\n",
    "    'IMG_SIZE' : 224\n",
    "}\n",
    "\n",
    "classnames=['buildings', 'forest', 'glacier', 'mountain', 'sea', 'street']\n",
    "\n",
    "train_paths =  glob(os.path.join(config['TRAIN_PATH'], '*', '*'), recursive=True)\n",
    "test_paths =  glob(os.path.join(config['TEST_PATH'], '*', '*'), recursive=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "RYUeUQuD7bGz"
   },
   "outputs": [],
   "source": [
    "def get_labels(pathlist):\n",
    "    labels = []\n",
    "    for path in pathlist:\n",
    "        labels.append(classnames.index(path.split(os.sep)[3]))\n",
    "    labels = np.array(labels)\n",
    "    return labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "Ve9G6Ck77qb1"
   },
   "outputs": [],
   "source": [
    "train_labels = get_labels(train_paths)\n",
    "test_labels = get_labels(test_paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "NwNL4NaVziJT"
   },
   "outputs": [],
   "source": [
    "import albumentations as A\n",
    "\n",
    "def preprocess(image_path, label):\n",
    "    img = tf.io.read_file(image_path)\n",
    "    img = tf.image.decode_jpeg(img, channels=3)\n",
    "    img = tf.image.resize(img, size=[config['IMG_SIZE'], config['IMG_SIZE']])\n",
    "    img = tf.cast(img, tf.float32)\n",
    "\n",
    "    return img, label\n",
    "\n",
    "def albumentations(img):\n",
    "    transform = A.Compose([\n",
    "        A.HorizontalFlip(p=0.5),\n",
    "        A.RandomBrightnessContrast(p=0.3),\n",
    "        A.VerticalFlip(p=0.5),\n",
    "        A.CoarseDropout(p=0.5)\n",
    "    ])\n",
    "    \n",
    "    transformed_image = transform(image=img)['image']\n",
    "    return transformed_image\n",
    "\n",
    "def apply_albumentations(img, label):\n",
    "    aug_img = tf.numpy_function(func=albumentations, inp=[img], Tout=tf.float32)\n",
    "    return aug_img, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "qRsTy2aNziMP"
   },
   "outputs": [],
   "source": [
    "def create_dataset(images, labels):\n",
    "    dataset = tf.data.Dataset.from_tensor_slices((images, labels)).shuffle(len(images))\n",
    "    dataset = dataset.map(preprocess, num_parallel_calls=tf.data.AUTOTUNE).map(apply_albumentations).batch(config['BATCH_SIZE'], drop_remainder=True).prefetch(tf.data.AUTOTUNE)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "Xe5G9beN0YI_"
   },
   "outputs": [],
   "source": [
    "train_set = create_dataset(train_paths, train_labels)\n",
    "test_set = create_dataset(test_paths, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "sQ4xmI7ZkPfC"
   },
   "outputs": [],
   "source": [
    "def make_model(jit=False, separable=False, mixed_precision=False, fuse_steps=1):\n",
    "    if mixed_precision:\n",
    "        tf.keras.mixed_precision.set_global_policy('mixed_float16')\n",
    "    elif mixed_precision == False:\n",
    "        tf.keras.mixed_precision.set_global_policy('float32')\n",
    "    \n",
    "    if separable:\n",
    "        model = keras.models.Sequential([\n",
    "          keras.layers.SeparableConv2D(64, (3, 3), activation='relu', padding='same', input_shape=[224, 224, 3]),\n",
    "          keras.layers.SeparableConv2D(64, 3, activation='relu', padding='same'),\n",
    "          keras.layers.MaxPooling2D((2, 2), (2, 2)),\n",
    "          keras.layers.BatchNormalization(),\n",
    "\n",
    "          keras.layers.SeparableConv2D(128, 3, activation='relu', padding='same'),\n",
    "          keras.layers.SeparableConv2D(128, 3, activation='relu', padding='same'),\n",
    "          keras.layers.MaxPooling2D(2, 2),\n",
    "          keras.layers.BatchNormalization(),\n",
    "\n",
    "          keras.layers.SeparableConv2D(256, 3, activation='relu', padding='same'),\n",
    "          keras.layers.SeparableConv2D(256, 3, activation='relu', padding='same'),\n",
    "          keras.layers.MaxPooling2D(2, 2),\n",
    "          keras.layers.BatchNormalization(),\n",
    "\n",
    "          keras.layers.SeparableConv2D(256, 3, activation='relu', padding='same'),\n",
    "          keras.layers.SeparableConv2D(256, 3, activation='relu', padding='same'),\n",
    "          keras.layers.MaxPooling2D(2, 2),\n",
    "          keras.layers.BatchNormalization(),\n",
    "\n",
    "          keras.layers.SeparableConv2D(512, 3, activation='relu', padding='same'),\n",
    "          keras.layers.SeparableConv2D(512, 3, activation='relu', padding='same'),\n",
    "          keras.layers.MaxPooling2D(2, 2),\n",
    "          keras.layers.BatchNormalization(),\n",
    "\n",
    "          keras.layers.GlobalAveragePooling2D(),    \n",
    "          keras.layers.Dropout(0.3, seed=2),\n",
    "          keras.layers.Dense(6, activation='softmax')\n",
    "      ])\n",
    "      else:\n",
    "        model = keras.models.Sequential([\n",
    "          keras.layers.Conv2D(64, (3, 3), activation='relu', padding='same', input_shape=[224, 224, 3]),\n",
    "          keras.layers.Conv2D(64, 3, activation='relu', padding='same'),\n",
    "          keras.layers.MaxPooling2D((2, 2), (2, 2)),\n",
    "          keras.layers.BatchNormalization(),\n",
    "\n",
    "          keras.layers.Conv2D(128, 3, activation='relu', padding='same'),\n",
    "          keras.layers.Conv2D(128, 3, activation='relu', padding='same'),\n",
    "          keras.layers.MaxPooling2D(2, 2),\n",
    "          keras.layers.BatchNormalization(),\n",
    "\n",
    "          keras.layers.Conv2D(256, 3, activation='relu', padding='same'),\n",
    "          keras.layers.Conv2D(256, 3, activation='relu', padding='same'),\n",
    "          keras.layers.MaxPooling2D(2, 2),\n",
    "          keras.layers.BatchNormalization(),\n",
    "\n",
    "          keras.layers.Conv2D(256, 3, activation='relu', padding='same'),\n",
    "          keras.layers.Conv2D(256, 3, activation='relu', padding='same'),\n",
    "          keras.layers.MaxPooling2D(2, 2),\n",
    "          keras.layers.BatchNormalization(),\n",
    "\n",
    "          keras.layers.Conv2D(512, 3, activation='relu', padding='same'),\n",
    "          keras.layers.Conv2D(512, 3, activation='relu', padding='same'),\n",
    "          keras.layers.MaxPooling2D(2, 2),\n",
    "          keras.layers.BatchNormalization(),\n",
    "\n",
    "          keras.layers.GlobalAveragePooling2D(),    \n",
    "          keras.layers.Dropout(0.3, seed=2),\n",
    "          keras.layers.Dense(6, activation='softmax')\n",
    "      ])\n",
    "  \n",
    "\n",
    "      model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "                  optimizer=keras.optimizers.Adam(),\n",
    "                  jit_compile=jit,\n",
    "                  steps_per_execution=fuse_steps,\n",
    "                  metrics=['accuracy',\n",
    "                           keras.metrics.SparseTopKCategoricalAccuracy(k=2)])\n",
    "  \n",
    "  return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "reducelr = keras.callbacks.ReduceLROnPlateau(patience=5, factor=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "iGSowBd84NQn",
    "outputId": "6e2baae4-64b5-4639-ec7e-dcf1a8754376"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " conv2d (Conv2D)             (None, 224, 224, 64)      1792      \n",
      "                                                                 \n",
      " conv2d_1 (Conv2D)           (None, 224, 224, 64)      36928     \n",
      "                                                                 \n",
      " max_pooling2d (MaxPooling2D  (None, 112, 112, 64)     0         \n",
      " )                                                               \n",
      "                                                                 \n",
      " batch_normalization (BatchN  (None, 112, 112, 64)     256       \n",
      " ormalization)                                                   \n",
      "                                                                 \n",
      " conv2d_2 (Conv2D)           (None, 112, 112, 128)     73856     \n",
      "                                                                 \n",
      " conv2d_3 (Conv2D)           (None, 112, 112, 128)     147584    \n",
      "                                                                 \n",
      " max_pooling2d_1 (MaxPooling  (None, 56, 56, 128)      0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " batch_normalization_1 (Batc  (None, 56, 56, 128)      512       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " conv2d_4 (Conv2D)           (None, 56, 56, 256)       295168    \n",
      "                                                                 \n",
      " conv2d_5 (Conv2D)           (None, 56, 56, 256)       590080    \n",
      "                                                                 \n",
      " max_pooling2d_2 (MaxPooling  (None, 28, 28, 256)      0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " batch_normalization_2 (Batc  (None, 28, 28, 256)      1024      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " conv2d_6 (Conv2D)           (None, 28, 28, 256)       590080    \n",
      "                                                                 \n",
      " conv2d_7 (Conv2D)           (None, 28, 28, 256)       590080    \n",
      "                                                                 \n",
      " max_pooling2d_3 (MaxPooling  (None, 14, 14, 256)      0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " batch_normalization_3 (Batc  (None, 14, 14, 256)      1024      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " conv2d_8 (Conv2D)           (None, 14, 14, 512)       1180160   \n",
      "                                                                 \n",
      " conv2d_9 (Conv2D)           (None, 14, 14, 512)       2359808   \n",
      "                                                                 \n",
      " max_pooling2d_4 (MaxPooling  (None, 7, 7, 512)        0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " batch_normalization_4 (Batc  (None, 7, 7, 512)        2048      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " global_average_pooling2d (G  (None, 512)              0         \n",
      " lobalAveragePooling2D)                                          \n",
      "                                                                 \n",
      " dropout (Dropout)           (None, 512)               0         \n",
      "                                                                 \n",
      " dense (Dense)               (None, 6)                 3078      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 5,873,478\n",
      "Trainable params: 5,871,046\n",
      "Non-trainable params: 2,432\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "plain = make_model()\n",
    "plain.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Wvns_VDF4P4I",
    "outputId": "cd0367b4-75fd-4c3b-b862-ffb4808f7db6",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "877/877 [==============================] - 171s 189ms/step - loss: 1.4848 - accuracy: 0.3813 - sparse_top_k_categorical_accuracy: 0.6031 - val_loss: 1.6316 - val_accuracy: 0.4044 - val_sparse_top_k_categorical_accuracy: 0.6083 - lr: 0.0010\n",
      "Epoch 2/20\n",
      "877/877 [==============================] - 165s 188ms/step - loss: 1.2100 - accuracy: 0.4991 - sparse_top_k_categorical_accuracy: 0.7025 - val_loss: 1.8141 - val_accuracy: 0.3636 - val_sparse_top_k_categorical_accuracy: 0.5832 - lr: 0.0010\n",
      "Epoch 3/20\n",
      "877/877 [==============================] - 165s 188ms/step - loss: 1.1293 - accuracy: 0.5342 - sparse_top_k_categorical_accuracy: 0.7191 - val_loss: 1.1630 - val_accuracy: 0.5231 - val_sparse_top_k_categorical_accuracy: 0.7062 - lr: 0.0010\n",
      "Epoch 4/20\n",
      "877/877 [==============================] - 165s 188ms/step - loss: 1.0832 - accuracy: 0.5523 - sparse_top_k_categorical_accuracy: 0.7318 - val_loss: 1.0257 - val_accuracy: 0.5538 - val_sparse_top_k_categorical_accuracy: 0.7510 - lr: 0.0010\n",
      "Epoch 5/20\n",
      "877/877 [==============================] - 165s 188ms/step - loss: 1.0618 - accuracy: 0.5609 - sparse_top_k_categorical_accuracy: 0.7392 - val_loss: 0.9574 - val_accuracy: 0.5912 - val_sparse_top_k_categorical_accuracy: 0.7614 - lr: 0.0010\n",
      "Epoch 6/20\n",
      "877/877 [==============================] - 165s 188ms/step - loss: 1.0322 - accuracy: 0.5733 - sparse_top_k_categorical_accuracy: 0.7424 - val_loss: 1.2259 - val_accuracy: 0.5201 - val_sparse_top_k_categorical_accuracy: 0.7253 - lr: 0.0010\n",
      "Epoch 7/20\n",
      "877/877 [==============================] - 166s 189ms/step - loss: 1.0000 - accuracy: 0.5903 - sparse_top_k_categorical_accuracy: 0.7516 - val_loss: 1.0323 - val_accuracy: 0.5802 - val_sparse_top_k_categorical_accuracy: 0.7640 - lr: 0.0010\n",
      "Epoch 8/20\n",
      "877/877 [==============================] - 165s 189ms/step - loss: 0.9793 - accuracy: 0.5930 - sparse_top_k_categorical_accuracy: 0.7551 - val_loss: 1.0279 - val_accuracy: 0.5779 - val_sparse_top_k_categorical_accuracy: 0.7443 - lr: 0.0010\n",
      "Epoch 9/20\n",
      "877/877 [==============================] - 165s 188ms/step - loss: 0.9478 - accuracy: 0.6090 - sparse_top_k_categorical_accuracy: 0.7605 - val_loss: 1.1018 - val_accuracy: 0.5394 - val_sparse_top_k_categorical_accuracy: 0.7320 - lr: 0.0010\n",
      "Epoch 10/20\n",
      "877/877 [==============================] - 166s 190ms/step - loss: 0.9508 - accuracy: 0.6048 - sparse_top_k_categorical_accuracy: 0.7572 - val_loss: 0.8971 - val_accuracy: 0.6173 - val_sparse_top_k_categorical_accuracy: 0.7707 - lr: 0.0010\n",
      "Epoch 11/20\n",
      "877/877 [==============================] - 167s 190ms/step - loss: 0.9221 - accuracy: 0.6187 - sparse_top_k_categorical_accuracy: 0.7687 - val_loss: 0.9830 - val_accuracy: 0.5872 - val_sparse_top_k_categorical_accuracy: 0.7540 - lr: 0.0010\n",
      "Epoch 12/20\n",
      "877/877 [==============================] - 166s 189ms/step - loss: 0.9143 - accuracy: 0.6178 - sparse_top_k_categorical_accuracy: 0.7672 - val_loss: 0.9115 - val_accuracy: 0.6213 - val_sparse_top_k_categorical_accuracy: 0.7654 - lr: 0.0010\n",
      "Epoch 13/20\n",
      "877/877 [==============================] - 165s 189ms/step - loss: 0.8976 - accuracy: 0.6262 - sparse_top_k_categorical_accuracy: 0.7699 - val_loss: 0.9875 - val_accuracy: 0.5695 - val_sparse_top_k_categorical_accuracy: 0.7289 - lr: 0.0010\n",
      "Epoch 14/20\n",
      "877/877 [==============================] - 166s 189ms/step - loss: 0.8902 - accuracy: 0.6283 - sparse_top_k_categorical_accuracy: 0.7728 - val_loss: 0.8858 - val_accuracy: 0.6230 - val_sparse_top_k_categorical_accuracy: 0.7684 - lr: 0.0010\n",
      "Epoch 15/20\n",
      "877/877 [==============================] - 165s 189ms/step - loss: 0.8735 - accuracy: 0.6345 - sparse_top_k_categorical_accuracy: 0.7741 - val_loss: 1.0020 - val_accuracy: 0.5896 - val_sparse_top_k_categorical_accuracy: 0.7406 - lr: 0.0010\n",
      "Epoch 16/20\n",
      "877/877 [==============================] - 165s 189ms/step - loss: 0.8817 - accuracy: 0.6323 - sparse_top_k_categorical_accuracy: 0.7688 - val_loss: 0.8736 - val_accuracy: 0.6367 - val_sparse_top_k_categorical_accuracy: 0.7744 - lr: 0.0010\n",
      "Epoch 17/20\n",
      "877/877 [==============================] - 165s 188ms/step - loss: 0.8633 - accuracy: 0.6406 - sparse_top_k_categorical_accuracy: 0.7712 - val_loss: 0.9967 - val_accuracy: 0.6136 - val_sparse_top_k_categorical_accuracy: 0.7507 - lr: 0.0010\n",
      "Epoch 18/20\n",
      "877/877 [==============================] - 164s 187ms/step - loss: 0.8611 - accuracy: 0.6423 - sparse_top_k_categorical_accuracy: 0.7739 - val_loss: 0.9254 - val_accuracy: 0.5916 - val_sparse_top_k_categorical_accuracy: 0.7423 - lr: 0.0010\n",
      "Epoch 19/20\n",
      "877/877 [==============================] - 164s 187ms/step - loss: 0.8467 - accuracy: 0.6466 - sparse_top_k_categorical_accuracy: 0.7773 - val_loss: 0.8445 - val_accuracy: 0.6394 - val_sparse_top_k_categorical_accuracy: 0.7711 - lr: 0.0010\n",
      "Epoch 20/20\n",
      "877/877 [==============================] - 164s 187ms/step - loss: 0.8304 - accuracy: 0.6530 - sparse_top_k_categorical_accuracy: 0.7824 - val_loss: 0.8227 - val_accuracy: 0.6444 - val_sparse_top_k_categorical_accuracy: 0.7828 - lr: 0.0010\n"
     ]
    }
   ],
   "source": [
    "plain_history = plain.fit(train_set,\n",
    "                    validation_data = test_set,\n",
    "                    callbacks=[reducelr],\n",
    "                    epochs = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DFchSS1Hfalx",
    "outputId": "a211d3cd-2ef6-41b6-e3f7-01eef352ee43"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " separable_conv2d (Separable  (None, 224, 224, 64)     283       \n",
      " Conv2D)                                                         \n",
      "                                                                 \n",
      " separable_conv2d_1 (Separab  (None, 224, 224, 64)     4736      \n",
      " leConv2D)                                                       \n",
      "                                                                 \n",
      " max_pooling2d_5 (MaxPooling  (None, 112, 112, 64)     0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " batch_normalization_5 (Batc  (None, 112, 112, 64)     256       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " separable_conv2d_2 (Separab  (None, 112, 112, 128)    8896      \n",
      " leConv2D)                                                       \n",
      "                                                                 \n",
      " separable_conv2d_3 (Separab  (None, 112, 112, 128)    17664     \n",
      " leConv2D)                                                       \n",
      "                                                                 \n",
      " max_pooling2d_6 (MaxPooling  (None, 56, 56, 128)      0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " batch_normalization_6 (Batc  (None, 56, 56, 128)      512       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " separable_conv2d_4 (Separab  (None, 56, 56, 256)      34176     \n",
      " leConv2D)                                                       \n",
      "                                                                 \n",
      " separable_conv2d_5 (Separab  (None, 56, 56, 256)      68096     \n",
      " leConv2D)                                                       \n",
      "                                                                 \n",
      " max_pooling2d_7 (MaxPooling  (None, 28, 28, 256)      0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " batch_normalization_7 (Batc  (None, 28, 28, 256)      1024      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " separable_conv2d_6 (Separab  (None, 28, 28, 256)      68096     \n",
      " leConv2D)                                                       \n",
      "                                                                 \n",
      " separable_conv2d_7 (Separab  (None, 28, 28, 256)      68096     \n",
      " leConv2D)                                                       \n",
      "                                                                 \n",
      " max_pooling2d_8 (MaxPooling  (None, 14, 14, 256)      0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " batch_normalization_8 (Batc  (None, 14, 14, 256)      1024      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " separable_conv2d_8 (Separab  (None, 14, 14, 512)      133888    \n",
      " leConv2D)                                                       \n",
      "                                                                 \n",
      " separable_conv2d_9 (Separab  (None, 14, 14, 512)      267264    \n",
      " leConv2D)                                                       \n",
      "                                                                 \n",
      " max_pooling2d_9 (MaxPooling  (None, 7, 7, 512)        0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " batch_normalization_9 (Batc  (None, 7, 7, 512)        2048      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " global_average_pooling2d_1   (None, 512)              0         \n",
      " (GlobalAveragePooling2D)                                        \n",
      "                                                                 \n",
      " dropout_1 (Dropout)         (None, 512)               0         \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 6)                 3078      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 679,137\n",
      "Trainable params: 676,705\n",
      "Non-trainable params: 2,432\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "separable = make_model(separable=True)\n",
    "separable.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WMzVCdY34dRR",
    "outputId": "6893c482-6edc-4c73-f4c9-c79cfcec7cc9",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "877/877 [==============================] - 253s 285ms/step - loss: 1.3489 - accuracy: 0.4065 - sparse_top_k_categorical_accuracy: 0.6484 - val_loss: 1.4346 - val_accuracy: 0.4281 - val_sparse_top_k_categorical_accuracy: 0.6624 - lr: 0.0010\n",
      "Epoch 2/20\n",
      "877/877 [==============================] - 251s 286ms/step - loss: 1.1321 - accuracy: 0.5315 - sparse_top_k_categorical_accuracy: 0.7264 - val_loss: 1.9086 - val_accuracy: 0.4047 - val_sparse_top_k_categorical_accuracy: 0.5979 - lr: 0.0010\n",
      "Epoch 3/20\n",
      "877/877 [==============================] - 249s 284ms/step - loss: 1.0105 - accuracy: 0.5864 - sparse_top_k_categorical_accuracy: 0.7647 - val_loss: 0.9210 - val_accuracy: 0.6240 - val_sparse_top_k_categorical_accuracy: 0.8055 - lr: 0.0010\n",
      "Epoch 4/20\n",
      "877/877 [==============================] - 249s 284ms/step - loss: 0.8997 - accuracy: 0.6418 - sparse_top_k_categorical_accuracy: 0.8277 - val_loss: 0.8957 - val_accuracy: 0.6661 - val_sparse_top_k_categorical_accuracy: 0.8469 - lr: 0.0010\n",
      "Epoch 5/20\n",
      "877/877 [==============================] - 249s 284ms/step - loss: 0.8680 - accuracy: 0.6593 - sparse_top_k_categorical_accuracy: 0.8392 - val_loss: 0.8224 - val_accuracy: 0.6872 - val_sparse_top_k_categorical_accuracy: 0.8570 - lr: 0.0010\n",
      "Epoch 6/20\n",
      "877/877 [==============================] - 248s 283ms/step - loss: 0.8131 - accuracy: 0.6816 - sparse_top_k_categorical_accuracy: 0.8531 - val_loss: 1.0206 - val_accuracy: 0.6247 - val_sparse_top_k_categorical_accuracy: 0.8386 - lr: 0.0010\n",
      "Epoch 7/20\n",
      "877/877 [==============================] - 249s 284ms/step - loss: 0.7941 - accuracy: 0.6916 - sparse_top_k_categorical_accuracy: 0.8568 - val_loss: 0.7463 - val_accuracy: 0.7273 - val_sparse_top_k_categorical_accuracy: 0.8723 - lr: 0.0010\n",
      "Epoch 8/20\n",
      "877/877 [==============================] - 247s 282ms/step - loss: 0.7888 - accuracy: 0.6955 - sparse_top_k_categorical_accuracy: 0.8595 - val_loss: 0.7284 - val_accuracy: 0.7186 - val_sparse_top_k_categorical_accuracy: 0.8760 - lr: 0.0010\n",
      "Epoch 9/20\n",
      "877/877 [==============================] - 249s 284ms/step - loss: 0.7692 - accuracy: 0.7046 - sparse_top_k_categorical_accuracy: 0.8597 - val_loss: 0.7627 - val_accuracy: 0.7146 - val_sparse_top_k_categorical_accuracy: 0.8553 - lr: 0.0010\n",
      "Epoch 10/20\n",
      "877/877 [==============================] - 250s 285ms/step - loss: 0.7533 - accuracy: 0.7091 - sparse_top_k_categorical_accuracy: 0.8682 - val_loss: 0.7857 - val_accuracy: 0.6949 - val_sparse_top_k_categorical_accuracy: 0.8770 - lr: 0.0010\n",
      "Epoch 11/20\n",
      "877/877 [==============================] - 248s 283ms/step - loss: 0.7346 - accuracy: 0.7202 - sparse_top_k_categorical_accuracy: 0.8667 - val_loss: 0.9239 - val_accuracy: 0.6544 - val_sparse_top_k_categorical_accuracy: 0.8616 - lr: 0.0010\n",
      "Epoch 12/20\n",
      "877/877 [==============================] - 250s 285ms/step - loss: 0.7176 - accuracy: 0.7281 - sparse_top_k_categorical_accuracy: 0.8759 - val_loss: 0.7847 - val_accuracy: 0.7149 - val_sparse_top_k_categorical_accuracy: 0.8753 - lr: 0.0010\n",
      "Epoch 13/20\n",
      "877/877 [==============================] - 249s 284ms/step - loss: 0.7045 - accuracy: 0.7319 - sparse_top_k_categorical_accuracy: 0.8746 - val_loss: 0.7040 - val_accuracy: 0.7249 - val_sparse_top_k_categorical_accuracy: 0.8803 - lr: 0.0010\n",
      "Epoch 14/20\n",
      "877/877 [==============================] - 250s 284ms/step - loss: 0.6981 - accuracy: 0.7346 - sparse_top_k_categorical_accuracy: 0.8775 - val_loss: 0.6969 - val_accuracy: 0.7356 - val_sparse_top_k_categorical_accuracy: 0.8810 - lr: 0.0010\n",
      "Epoch 15/20\n",
      "877/877 [==============================] - 250s 285ms/step - loss: 0.6841 - accuracy: 0.7361 - sparse_top_k_categorical_accuracy: 0.8816 - val_loss: 0.6508 - val_accuracy: 0.7574 - val_sparse_top_k_categorical_accuracy: 0.8844 - lr: 0.0010\n",
      "Epoch 16/20\n",
      "877/877 [==============================] - 249s 284ms/step - loss: 0.6785 - accuracy: 0.7417 - sparse_top_k_categorical_accuracy: 0.8811 - val_loss: 0.6643 - val_accuracy: 0.7507 - val_sparse_top_k_categorical_accuracy: 0.8847 - lr: 0.0010\n",
      "Epoch 17/20\n",
      "877/877 [==============================] - 249s 284ms/step - loss: 0.6700 - accuracy: 0.7432 - sparse_top_k_categorical_accuracy: 0.8814 - val_loss: 0.6329 - val_accuracy: 0.7600 - val_sparse_top_k_categorical_accuracy: 0.8920 - lr: 0.0010\n",
      "Epoch 18/20\n",
      "877/877 [==============================] - 250s 286ms/step - loss: 0.6673 - accuracy: 0.7477 - sparse_top_k_categorical_accuracy: 0.8865 - val_loss: 0.7093 - val_accuracy: 0.7286 - val_sparse_top_k_categorical_accuracy: 0.8767 - lr: 0.0010\n",
      "Epoch 19/20\n",
      "877/877 [==============================] - 249s 284ms/step - loss: 0.6747 - accuracy: 0.7442 - sparse_top_k_categorical_accuracy: 0.8753 - val_loss: 0.7812 - val_accuracy: 0.6935 - val_sparse_top_k_categorical_accuracy: 0.8459 - lr: 0.0010\n",
      "Epoch 20/20\n",
      "877/877 [==============================] - 250s 285ms/step - loss: 0.6721 - accuracy: 0.7437 - sparse_top_k_categorical_accuracy: 0.8783 - val_loss: 0.6921 - val_accuracy: 0.7433 - val_sparse_top_k_categorical_accuracy: 0.8810 - lr: 0.0010\n"
     ]
    }
   ],
   "source": [
    "separable_history = separable.fit(train_set,\n",
    "                    validation_data = test_set,\n",
    "                    callbacks=[reducelr],\n",
    "                    epochs = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xPpzMWuz4ebK",
    "outputId": "d322fb26-1c10-4248-cbaa-08c97e809843"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " separable_conv2d_10 (Separa  (None, 224, 224, 64)     283       \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_11 (Separa  (None, 224, 224, 64)     4736      \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_10 (MaxPoolin  (None, 112, 112, 64)     0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_10 (Bat  (None, 112, 112, 64)     256       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " separable_conv2d_12 (Separa  (None, 112, 112, 128)    8896      \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_13 (Separa  (None, 112, 112, 128)    17664     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_11 (MaxPoolin  (None, 56, 56, 128)      0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_11 (Bat  (None, 56, 56, 128)      512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " separable_conv2d_14 (Separa  (None, 56, 56, 256)      34176     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_15 (Separa  (None, 56, 56, 256)      68096     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_12 (MaxPoolin  (None, 28, 28, 256)      0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_12 (Bat  (None, 28, 28, 256)      1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " separable_conv2d_16 (Separa  (None, 28, 28, 256)      68096     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_17 (Separa  (None, 28, 28, 256)      68096     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_13 (MaxPoolin  (None, 14, 14, 256)      0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_13 (Bat  (None, 14, 14, 256)      1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " separable_conv2d_18 (Separa  (None, 14, 14, 512)      133888    \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_19 (Separa  (None, 14, 14, 512)      267264    \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_14 (MaxPoolin  (None, 7, 7, 512)        0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_14 (Bat  (None, 7, 7, 512)        2048      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " global_average_pooling2d_2   (None, 512)              0         \n",
      " (GlobalAveragePooling2D)                                        \n",
      "                                                                 \n",
      " dropout_2 (Dropout)         (None, 512)               0         \n",
      "                                                                 \n",
      " dense_2 (Dense)             (None, 6)                 3078      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 679,137\n",
      "Trainable params: 676,705\n",
      "Non-trainable params: 2,432\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "separable_jit = make_model(separable=True, jit=True)\n",
    "separable_jit.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvcc: NVIDIA (R) Cuda compiler driver\n",
      "Copyright (c) 2005-2020 NVIDIA Corporation\n",
      "Built on Mon_Nov_30_19:15:10_Pacific_Standard_Time_2020\n",
      "Cuda compilation tools, release 11.2, V11.2.67\n",
      "Build cuda_11.2.r11.2/compiler.29373293_0\n"
     ]
    }
   ],
   "source": [
    "!nvcc --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "slCDuPdL4ed1",
    "outputId": "5fe04661-7618-44c1-b91b-37eb7e40b979",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "877/877 [==============================] - 124s 102ms/step - loss: 1.3210 - accuracy: 0.4247 - sparse_top_k_categorical_accuracy: 0.6578 - val_loss: 1.2304 - val_accuracy: 0.4763 - val_sparse_top_k_categorical_accuracy: 0.7142 - lr: 0.0010\n",
      "Epoch 2/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 1.1045 - accuracy: 0.5437 - sparse_top_k_categorical_accuracy: 0.7318 - val_loss: 1.0270 - val_accuracy: 0.5739 - val_sparse_top_k_categorical_accuracy: 0.7376 - lr: 0.0010\n",
      "Epoch 3/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 1.0339 - accuracy: 0.5745 - sparse_top_k_categorical_accuracy: 0.7437 - val_loss: 1.1088 - val_accuracy: 0.5361 - val_sparse_top_k_categorical_accuracy: 0.7383 - lr: 0.0010\n",
      "Epoch 4/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.9958 - accuracy: 0.5900 - sparse_top_k_categorical_accuracy: 0.7568 - val_loss: 0.9926 - val_accuracy: 0.5943 - val_sparse_top_k_categorical_accuracy: 0.7577 - lr: 0.0010\n",
      "Epoch 5/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.9617 - accuracy: 0.6011 - sparse_top_k_categorical_accuracy: 0.7564 - val_loss: 0.9262 - val_accuracy: 0.6156 - val_sparse_top_k_categorical_accuracy: 0.7620 - lr: 0.0010\n",
      "Epoch 6/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.9211 - accuracy: 0.6251 - sparse_top_k_categorical_accuracy: 0.7705 - val_loss: 1.0324 - val_accuracy: 0.5672 - val_sparse_top_k_categorical_accuracy: 0.7533 - lr: 0.0010\n",
      "Epoch 7/20\n",
      "877/877 [==============================] - 89s 102ms/step - loss: 0.9158 - accuracy: 0.6244 - sparse_top_k_categorical_accuracy: 0.7687 - val_loss: 0.8859 - val_accuracy: 0.6237 - val_sparse_top_k_categorical_accuracy: 0.7717 - lr: 0.0010\n",
      "Epoch 8/20\n",
      "877/877 [==============================] - 89s 102ms/step - loss: 0.8764 - accuracy: 0.6449 - sparse_top_k_categorical_accuracy: 0.7876 - val_loss: 0.8435 - val_accuracy: 0.6614 - val_sparse_top_k_categorical_accuracy: 0.8202 - lr: 0.0010\n",
      "Epoch 9/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.8229 - accuracy: 0.6762 - sparse_top_k_categorical_accuracy: 0.8347 - val_loss: 0.8647 - val_accuracy: 0.6668 - val_sparse_top_k_categorical_accuracy: 0.8459 - lr: 0.0010\n",
      "Epoch 10/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.7847 - accuracy: 0.6949 - sparse_top_k_categorical_accuracy: 0.8532 - val_loss: 0.7284 - val_accuracy: 0.7166 - val_sparse_top_k_categorical_accuracy: 0.8660 - lr: 0.0010\n",
      "Epoch 11/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.7629 - accuracy: 0.7038 - sparse_top_k_categorical_accuracy: 0.8540 - val_loss: 0.8042 - val_accuracy: 0.6915 - val_sparse_top_k_categorical_accuracy: 0.8519 - lr: 0.0010\n",
      "Epoch 12/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.7437 - accuracy: 0.7126 - sparse_top_k_categorical_accuracy: 0.8635 - val_loss: 0.7381 - val_accuracy: 0.7203 - val_sparse_top_k_categorical_accuracy: 0.8673 - lr: 0.0010\n",
      "Epoch 13/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.7329 - accuracy: 0.7166 - sparse_top_k_categorical_accuracy: 0.8609 - val_loss: 0.7910 - val_accuracy: 0.7072 - val_sparse_top_k_categorical_accuracy: 0.8713 - lr: 0.0010\n",
      "Epoch 14/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.7127 - accuracy: 0.7274 - sparse_top_k_categorical_accuracy: 0.8697 - val_loss: 0.7061 - val_accuracy: 0.7273 - val_sparse_top_k_categorical_accuracy: 0.8824 - lr: 0.0010\n",
      "Epoch 15/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.7115 - accuracy: 0.7288 - sparse_top_k_categorical_accuracy: 0.8689 - val_loss: 0.6874 - val_accuracy: 0.7373 - val_sparse_top_k_categorical_accuracy: 0.8783 - lr: 0.0010\n",
      "Epoch 16/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.6916 - accuracy: 0.7392 - sparse_top_k_categorical_accuracy: 0.8766 - val_loss: 0.7013 - val_accuracy: 0.7139 - val_sparse_top_k_categorical_accuracy: 0.8743 - lr: 0.0010\n",
      "Epoch 17/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.6771 - accuracy: 0.7390 - sparse_top_k_categorical_accuracy: 0.8803 - val_loss: 0.6729 - val_accuracy: 0.7584 - val_sparse_top_k_categorical_accuracy: 0.8820 - lr: 0.0010\n",
      "Epoch 18/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.6709 - accuracy: 0.7425 - sparse_top_k_categorical_accuracy: 0.8838 - val_loss: 0.7234 - val_accuracy: 0.7293 - val_sparse_top_k_categorical_accuracy: 0.8650 - lr: 0.0010\n",
      "Epoch 19/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.6724 - accuracy: 0.7402 - sparse_top_k_categorical_accuracy: 0.8775 - val_loss: 0.7893 - val_accuracy: 0.6878 - val_sparse_top_k_categorical_accuracy: 0.8422 - lr: 0.0010\n",
      "Epoch 20/20\n",
      "877/877 [==============================] - 89s 101ms/step - loss: 0.6731 - accuracy: 0.7409 - sparse_top_k_categorical_accuracy: 0.8809 - val_loss: 0.8489 - val_accuracy: 0.6768 - val_sparse_top_k_categorical_accuracy: 0.8666 - lr: 0.0010\n"
     ]
    }
   ],
   "source": [
    "separable_jit_history = separable_jit.fit(train_set,\n",
    "                    validation_data = test_set,\n",
    "                    callbacks=[reducelr],\n",
    "                    epochs = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "zM_df7Hp4s0O",
    "outputId": "b78662f2-4352-46e3-d589-3a2af0366ed0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Mixed precision compatibility check (mixed_float16): OK\n",
      "Your GPU will likely run quickly with dtype policy mixed_float16 as it has compute capability of at least 7.0. Your GPU: NVIDIA GeForce GTX 1660 SUPER, compute capability 7.5\n",
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " separable_conv2d_20 (Separa  (None, 224, 224, 64)     283       \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_21 (Separa  (None, 224, 224, 64)     4736      \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_15 (MaxPoolin  (None, 112, 112, 64)     0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_15 (Bat  (None, 112, 112, 64)     256       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " separable_conv2d_22 (Separa  (None, 112, 112, 128)    8896      \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_23 (Separa  (None, 112, 112, 128)    17664     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_16 (MaxPoolin  (None, 56, 56, 128)      0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_16 (Bat  (None, 56, 56, 128)      512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " separable_conv2d_24 (Separa  (None, 56, 56, 256)      34176     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_25 (Separa  (None, 56, 56, 256)      68096     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_17 (MaxPoolin  (None, 28, 28, 256)      0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_17 (Bat  (None, 28, 28, 256)      1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " separable_conv2d_26 (Separa  (None, 28, 28, 256)      68096     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_27 (Separa  (None, 28, 28, 256)      68096     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_18 (MaxPoolin  (None, 14, 14, 256)      0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_18 (Bat  (None, 14, 14, 256)      1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " separable_conv2d_28 (Separa  (None, 14, 14, 512)      133888    \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_29 (Separa  (None, 14, 14, 512)      267264    \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_19 (MaxPoolin  (None, 7, 7, 512)        0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_19 (Bat  (None, 7, 7, 512)        2048      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " global_average_pooling2d_3   (None, 512)              0         \n",
      " (GlobalAveragePooling2D)                                        \n",
      "                                                                 \n",
      " dropout_3 (Dropout)         (None, 512)               0         \n",
      "                                                                 \n",
      " dense_3 (Dense)             (None, 6)                 3078      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 679,137\n",
      "Trainable params: 676,705\n",
      "Non-trainable params: 2,432\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "separable_jit_amp = make_model(separable=True, jit=True, mixed_precision=True)\n",
    "separable_jit_amp.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "wsnHAsjA4xOQ",
    "outputId": "00c46f74-4c4b-4922-ee24-92e278617ccc",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "877/877 [==============================] - 213s 77ms/step - loss: 1.3460 - accuracy: 0.4116 - sparse_top_k_categorical_accuracy: 0.6440 - val_loss: 1.1263 - val_accuracy: 0.5130 - val_sparse_top_k_categorical_accuracy: 0.7299 - lr: 0.0010\n",
      "Epoch 2/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 1.1402 - accuracy: 0.5228 - sparse_top_k_categorical_accuracy: 0.7200 - val_loss: 1.0441 - val_accuracy: 0.5588 - val_sparse_top_k_categorical_accuracy: 0.7467 - lr: 0.0010\n",
      "Epoch 3/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 1.0539 - accuracy: 0.5693 - sparse_top_k_categorical_accuracy: 0.7385 - val_loss: 1.2315 - val_accuracy: 0.5057 - val_sparse_top_k_categorical_accuracy: 0.7086 - lr: 0.0010\n",
      "Epoch 4/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.9955 - accuracy: 0.5919 - sparse_top_k_categorical_accuracy: 0.7535 - val_loss: 0.9769 - val_accuracy: 0.5852 - val_sparse_top_k_categorical_accuracy: 0.7590 - lr: 0.0010\n",
      "Epoch 5/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.9529 - accuracy: 0.6134 - sparse_top_k_categorical_accuracy: 0.7623 - val_loss: 1.0592 - val_accuracy: 0.5668 - val_sparse_top_k_categorical_accuracy: 0.7279 - lr: 0.0010\n",
      "Epoch 6/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.9355 - accuracy: 0.6155 - sparse_top_k_categorical_accuracy: 0.7615 - val_loss: 0.9438 - val_accuracy: 0.6036 - val_sparse_top_k_categorical_accuracy: 0.7537 - lr: 0.0010\n",
      "Epoch 7/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.9108 - accuracy: 0.6243 - sparse_top_k_categorical_accuracy: 0.7701 - val_loss: 1.0763 - val_accuracy: 0.5678 - val_sparse_top_k_categorical_accuracy: 0.7296 - lr: 0.0010\n",
      "Epoch 8/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.8991 - accuracy: 0.6298 - sparse_top_k_categorical_accuracy: 0.7695 - val_loss: 1.0175 - val_accuracy: 0.5785 - val_sparse_top_k_categorical_accuracy: 0.7654 - lr: 0.0010\n",
      "Epoch 9/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.8622 - accuracy: 0.6475 - sparse_top_k_categorical_accuracy: 0.7936 - val_loss: 0.8032 - val_accuracy: 0.6825 - val_sparse_top_k_categorical_accuracy: 0.8376 - lr: 0.0010\n",
      "Epoch 10/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.7996 - accuracy: 0.6861 - sparse_top_k_categorical_accuracy: 0.8393 - val_loss: 0.7344 - val_accuracy: 0.7249 - val_sparse_top_k_categorical_accuracy: 0.8670 - lr: 0.0010\n",
      "Epoch 11/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.7645 - accuracy: 0.7042 - sparse_top_k_categorical_accuracy: 0.8530 - val_loss: 0.8701 - val_accuracy: 0.6815 - val_sparse_top_k_categorical_accuracy: 0.8583 - lr: 0.0010\n",
      "Epoch 12/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.7457 - accuracy: 0.7122 - sparse_top_k_categorical_accuracy: 0.8605 - val_loss: 0.7902 - val_accuracy: 0.7179 - val_sparse_top_k_categorical_accuracy: 0.8620 - lr: 0.0010\n",
      "Epoch 13/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.7287 - accuracy: 0.7208 - sparse_top_k_categorical_accuracy: 0.8664 - val_loss: 0.7438 - val_accuracy: 0.7092 - val_sparse_top_k_categorical_accuracy: 0.8620 - lr: 0.0010\n",
      "Epoch 14/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.7197 - accuracy: 0.7242 - sparse_top_k_categorical_accuracy: 0.8678 - val_loss: 0.8282 - val_accuracy: 0.6865 - val_sparse_top_k_categorical_accuracy: 0.8509 - lr: 0.0010\n",
      "Epoch 15/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.7012 - accuracy: 0.7288 - sparse_top_k_categorical_accuracy: 0.8744 - val_loss: 0.6859 - val_accuracy: 0.7216 - val_sparse_top_k_categorical_accuracy: 0.8783 - lr: 0.0010\n",
      "Epoch 16/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.6982 - accuracy: 0.7342 - sparse_top_k_categorical_accuracy: 0.8729 - val_loss: 0.7808 - val_accuracy: 0.7062 - val_sparse_top_k_categorical_accuracy: 0.8620 - lr: 0.0010\n",
      "Epoch 17/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.6964 - accuracy: 0.7328 - sparse_top_k_categorical_accuracy: 0.8754 - val_loss: 0.6920 - val_accuracy: 0.7413 - val_sparse_top_k_categorical_accuracy: 0.8807 - lr: 0.0010\n",
      "Epoch 18/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.6832 - accuracy: 0.7386 - sparse_top_k_categorical_accuracy: 0.8726 - val_loss: 0.7519 - val_accuracy: 0.7099 - val_sparse_top_k_categorical_accuracy: 0.8606 - lr: 0.0010\n",
      "Epoch 19/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.6818 - accuracy: 0.7365 - sparse_top_k_categorical_accuracy: 0.8745 - val_loss: 0.6856 - val_accuracy: 0.7386 - val_sparse_top_k_categorical_accuracy: 0.8760 - lr: 0.0010\n",
      "Epoch 20/20\n",
      "877/877 [==============================] - 67s 77ms/step - loss: 0.6608 - accuracy: 0.7494 - sparse_top_k_categorical_accuracy: 0.8793 - val_loss: 0.7750 - val_accuracy: 0.7159 - val_sparse_top_k_categorical_accuracy: 0.8770 - lr: 0.0010\n"
     ]
    }
   ],
   "source": [
    "separable_jit_amp_history = separable_jit_amp.fit(train_set,\n",
    "                    validation_data = test_set,\n",
    "                    callbacks=[reducelr],\n",
    "                    epochs = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Nk3q2br3Yj6O",
    "outputId": "408cf1da-7984-440a-b1c5-a9e22aaf1b33",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " separable_conv2d_30 (Separa  (None, 224, 224, 64)     283       \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_31 (Separa  (None, 224, 224, 64)     4736      \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_20 (MaxPoolin  (None, 112, 112, 64)     0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_20 (Bat  (None, 112, 112, 64)     256       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " separable_conv2d_32 (Separa  (None, 112, 112, 128)    8896      \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_33 (Separa  (None, 112, 112, 128)    17664     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_21 (MaxPoolin  (None, 56, 56, 128)      0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_21 (Bat  (None, 56, 56, 128)      512       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " separable_conv2d_34 (Separa  (None, 56, 56, 256)      34176     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_35 (Separa  (None, 56, 56, 256)      68096     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_22 (MaxPoolin  (None, 28, 28, 256)      0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_22 (Bat  (None, 28, 28, 256)      1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " separable_conv2d_36 (Separa  (None, 28, 28, 256)      68096     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_37 (Separa  (None, 28, 28, 256)      68096     \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_23 (MaxPoolin  (None, 14, 14, 256)      0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_23 (Bat  (None, 14, 14, 256)      1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " separable_conv2d_38 (Separa  (None, 14, 14, 512)      133888    \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " separable_conv2d_39 (Separa  (None, 14, 14, 512)      267264    \n",
      " bleConv2D)                                                      \n",
      "                                                                 \n",
      " max_pooling2d_24 (MaxPoolin  (None, 7, 7, 512)        0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " batch_normalization_24 (Bat  (None, 7, 7, 512)        2048      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " global_average_pooling2d_4   (None, 512)              0         \n",
      " (GlobalAveragePooling2D)                                        \n",
      "                                                                 \n",
      " dropout_4 (Dropout)         (None, 512)               0         \n",
      "                                                                 \n",
      " dense_4 (Dense)             (None, 6)                 3078      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 679,137\n",
      "Trainable params: 676,705\n",
      "Non-trainable params: 2,432\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "separable_jit_amp_stepf = make_model(separable=True, jit=True, mixed_precision=True, fuse_steps=3)\n",
    "separable_jit_amp_stepf.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3J6JeSjWYhEm",
    "outputId": "17dd9bf5-b5c1-4256-c017-453bc60bc74e",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "877/877 [==============================] - 74s 84ms/step - loss: 1.2912 - accuracy: 0.4457 - sparse_top_k_categorical_accuracy: 0.6690 - val_loss: 1.1696 - val_accuracy: 0.5221 - val_sparse_top_k_categorical_accuracy: 0.7059 - lr: 0.0010\n",
      "Epoch 2/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 1.0694 - accuracy: 0.5564 - sparse_top_k_categorical_accuracy: 0.7449 - val_loss: 1.1186 - val_accuracy: 0.5328 - val_sparse_top_k_categorical_accuracy: 0.7373 - lr: 0.0010\n",
      "Epoch 3/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 1.0271 - accuracy: 0.5766 - sparse_top_k_categorical_accuracy: 0.7500 - val_loss: 1.2138 - val_accuracy: 0.5321 - val_sparse_top_k_categorical_accuracy: 0.7330 - lr: 0.0010\n",
      "Epoch 4/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.9821 - accuracy: 0.6042 - sparse_top_k_categorical_accuracy: 0.7592 - val_loss: 1.0868 - val_accuracy: 0.5665 - val_sparse_top_k_categorical_accuracy: 0.7320 - lr: 0.0010\n",
      "Epoch 5/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.9270 - accuracy: 0.6271 - sparse_top_k_categorical_accuracy: 0.7871 - val_loss: 0.9775 - val_accuracy: 0.6257 - val_sparse_top_k_categorical_accuracy: 0.7754 - lr: 0.0010\n",
      "Epoch 6/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.8697 - accuracy: 0.6512 - sparse_top_k_categorical_accuracy: 0.8173 - val_loss: 0.9070 - val_accuracy: 0.6367 - val_sparse_top_k_categorical_accuracy: 0.8142 - lr: 0.0010\n",
      "Epoch 7/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.8166 - accuracy: 0.6876 - sparse_top_k_categorical_accuracy: 0.8445 - val_loss: 0.7896 - val_accuracy: 0.6868 - val_sparse_top_k_categorical_accuracy: 0.8549 - lr: 0.0010\n",
      "Epoch 8/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.7843 - accuracy: 0.6933 - sparse_top_k_categorical_accuracy: 0.8498 - val_loss: 1.0402 - val_accuracy: 0.5922 - val_sparse_top_k_categorical_accuracy: 0.8272 - lr: 0.0010\n",
      "Epoch 9/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.7763 - accuracy: 0.7065 - sparse_top_k_categorical_accuracy: 0.8537 - val_loss: 0.7187 - val_accuracy: 0.7293 - val_sparse_top_k_categorical_accuracy: 0.8723 - lr: 0.0010\n",
      "Epoch 10/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.7660 - accuracy: 0.7036 - sparse_top_k_categorical_accuracy: 0.8635 - val_loss: 0.7350 - val_accuracy: 0.7323 - val_sparse_top_k_categorical_accuracy: 0.8623 - lr: 0.0010\n",
      "Epoch 11/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.7482 - accuracy: 0.7134 - sparse_top_k_categorical_accuracy: 0.8617 - val_loss: 0.7347 - val_accuracy: 0.7182 - val_sparse_top_k_categorical_accuracy: 0.8650 - lr: 0.0010\n",
      "Epoch 12/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.7258 - accuracy: 0.7231 - sparse_top_k_categorical_accuracy: 0.8724 - val_loss: 0.8377 - val_accuracy: 0.6811 - val_sparse_top_k_categorical_accuracy: 0.8476 - lr: 0.0010\n",
      "Epoch 13/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.7150 - accuracy: 0.7295 - sparse_top_k_categorical_accuracy: 0.8729 - val_loss: 0.7563 - val_accuracy: 0.7166 - val_sparse_top_k_categorical_accuracy: 0.8653 - lr: 0.0010\n",
      "Epoch 14/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.6941 - accuracy: 0.7344 - sparse_top_k_categorical_accuracy: 0.8768 - val_loss: 0.9386 - val_accuracy: 0.6451 - val_sparse_top_k_categorical_accuracy: 0.8416 - lr: 0.0010\n",
      "Epoch 15/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.6386 - accuracy: 0.7577 - sparse_top_k_categorical_accuracy: 0.8881 - val_loss: 0.6015 - val_accuracy: 0.7761 - val_sparse_top_k_categorical_accuracy: 0.8997 - lr: 2.0000e-04\n",
      "Epoch 16/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.6260 - accuracy: 0.7578 - sparse_top_k_categorical_accuracy: 0.8913 - val_loss: 0.6098 - val_accuracy: 0.7754 - val_sparse_top_k_categorical_accuracy: 0.8934 - lr: 2.0000e-04\n",
      "Epoch 17/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.6056 - accuracy: 0.7711 - sparse_top_k_categorical_accuracy: 0.8942 - val_loss: 0.6038 - val_accuracy: 0.7744 - val_sparse_top_k_categorical_accuracy: 0.8997 - lr: 2.0000e-04\n",
      "Epoch 18/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.6004 - accuracy: 0.7760 - sparse_top_k_categorical_accuracy: 0.8979 - val_loss: 0.6143 - val_accuracy: 0.7670 - val_sparse_top_k_categorical_accuracy: 0.8937 - lr: 2.0000e-04\n",
      "Epoch 19/20\n",
      "877/877 [==============================] - 67s 76ms/step - loss: 0.5913 - accuracy: 0.7722 - sparse_top_k_categorical_accuracy: 0.8950 - val_loss: 0.5950 - val_accuracy: 0.7717 - val_sparse_top_k_categorical_accuracy: 0.9001 - lr: 2.0000e-04\n",
      "Epoch 20/20\n",
      "877/877 [==============================] - 66s 76ms/step - loss: 0.5943 - accuracy: 0.7729 - sparse_top_k_categorical_accuracy: 0.8971 - val_loss: 0.5821 - val_accuracy: 0.7854 - val_sparse_top_k_categorical_accuracy: 0.9024 - lr: 2.0000e-04\n"
     ]
    }
   ],
   "source": [
    "separable_jit_amp_stepf_history = separable_jit_amp_stepf.fit(train_set,\n",
    "                    validation_data = test_set,\n",
    "                    callbacks=[reducelr],\n",
    "                    epochs = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jMwRgdJrBYLw",
    "outputId": "21418a05-24d9-47c3-95f6-a9d976e99fe5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "187/187 [==============================] - 10s 51ms/step - loss: 0.8324 - accuracy: 0.6397 - sparse_top_k_categorical_accuracy: 0.7811\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.8324045538902283, 0.6397058963775635, 0.7810828685760498]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plain.evaluate(test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "YXnDXtllNCu7",
    "outputId": "9c81ffbf-7e66-46ac-df16-8a6d0492e066"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "187/187 [==============================] - 7s 36ms/step - loss: 0.6741 - accuracy: 0.7433 - sparse_top_k_categorical_accuracy: 0.8877\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.6740912199020386, 0.7433155179023743, 0.8877005577087402]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "separable.evaluate(test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-SCpe6fBnNhH",
    "outputId": "3f55ed87-fd9e-4aae-ebf4-8457d07dfe9c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "187/187 [==============================] - 4s 19ms/step - loss: 0.8206 - accuracy: 0.6835 - sparse_top_k_categorical_accuracy: 0.8693\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.8206043243408203, 0.6834893226623535, 0.8693181872367859]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "separable_jit.evaluate(test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8jVxmMYLnNyI",
    "outputId": "d7576e09-c212-4145-d2e2-fb242d837368"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "187/187 [==============================] - 4s 22ms/step - loss: 0.7690 - accuracy: 0.7172 - sparse_top_k_categorical_accuracy: 0.8760\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.7690488696098328, 0.7172459959983826, 0.8760026693344116]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "separable_jit_amp.evaluate(test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rKwFCG93nSia",
    "outputId": "ae0aeba6-0318-4e4e-f4e7-493f7a89594b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "187/187 [==============================] - 4s 22ms/step - loss: 0.5980 - accuracy: 0.7714 - sparse_top_k_categorical_accuracy: 0.8934\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.597995400428772, 0.7713903784751892, 0.8933823704719543]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "separable_jit_amp_stepf.evaluate(test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WCsFeSeAapn3"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}