effnet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import glob\n",
    "import cv2 as cv\n",
    "import random\n",
    "import os\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import random\n",
    "from PIL import Image\n",
    "import PIL.ImageOps    \n",
    "\n",
    "import torchvision\n",
    "import torchvision.datasets as datasets\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "import torchvision.utils\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "import albumentations as A\n",
    "from albumentations.pytorch.transforms import ToTensorV2\n",
    "from sklearn.metrics import f1_score, accuracy_score\n",
    "\n",
    "\n",
    "from torch.cuda.amp import autocast\n",
    "from torch.cuda.amp import GradScaler\n",
    "from tqdm.auto import tqdm\n",
    "import timm\n",
    "import math\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pytorch_model_summary\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SplitDataset(image_dir, val_size, seed = 42):\n",
    "    fake_images = glob( image_dir + 'fake_images/*.png')\n",
    "    real_images = glob( image_dir + 'real_images/*.png')\n",
    "    labels = [1] * len(fake_images) + [0] * len(real_images)\n",
    "\n",
    "    X_train, X_val, y_train, y_val = train_test_split(fake_images + real_images, labels, test_size=val_size, random_state=seed, shuffle=True)\n",
    "\n",
    "    return X_train, X_val, y_train, y_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val, Y_train, Y_val = SplitDataset(img_dir = './DATA/train/',\n",
    "                val_size = 0.3,\n",
    "                seed = 41,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomDataset(Dataset):\n",
    "    def __init__(self, X, y):\n",
    "        '''\n",
    "        X: list of image path\n",
    "        y: list of label (0->real, 1->fake)\n",
    "        '''\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.transforms = transforms.Compose([\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Resize((256, 256)),\n",
    "            transforms.Normalize([0.485, 0.456,0.406],[0.229,0.224,0.225])\n",
    "        ])\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.X)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        impath = self.X[index]\n",
    "        fname = os.path.basename(impath)\n",
    "        img = Image.open(impath).convert('RGB')\n",
    "        img = self.transforms(img)\n",
    "        target = self.y[index]\n",
    "\n",
    "        return img, target, fname"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = CustomDataset(X = X_train, y = Y_train) # + CustomDataset2(X = X_train, y = Y_train)\n",
    "val_dataset = CustomDataset(X = X_val, y = Y_val)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "CFG = {\n",
    "    'IMG_SIZE':224,\n",
    "    'EPOCHS':100,\n",
    "    'LEARNING_RATE':3e-4,\n",
    "    # 'LEARNING_RATE':10,\n",
    "    'BATCH_SIZE':8,\n",
    "    'SEED':42,\n",
    "    'NUM_WORKERS':1,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seed_everything(seed):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = 'a'\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "\n",
    "seed_everything(CFG['SEED']) # Seed 고정"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataloader = DataLoader(dataset = train_dataset,\n",
    "                              batch_size = CFG['BATCH_SIZE'],\n",
    "                              num_workers = CFG['NUM_WORKERS'],\n",
    "                              shuffle = True,\n",
    "                              pin_memory = True,\n",
    "                              drop_last = True,)\n",
    "\n",
    "val_dataloader = DataLoader(dataset = val_dataset,\n",
    "                            batch_size = CFG['BATCH_SIZE'],\n",
    "                            num_workers = CFG['NUM_WORKERS'], \n",
    "                            shuffle = False,\n",
    "                            pin_memory = True,\n",
    "                            drop_last = True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set model\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['adv_inception_v3', 'bat_resnext26ts', 'beit_base_patch16_224', 'beit_base_patch16_224_in22k', 'beit_base_patch16_384', 'beit_large_patch16_224', 'beit_large_patch16_224_in22k', 'beit_large_patch16_384', 'beit_large_patch16_512', 'beitv2_base_patch16_224', 'beitv2_base_patch16_224_in22k', 'beitv2_large_patch16_224', 'beitv2_large_patch16_224_in22k', 'botnet26t_256', 'cait_m36_384', 'cait_m48_448', 'cait_s24_224', 'cait_s24_384', 'cait_s36_384', 'cait_xs24_384', 'cait_xxs24_224', 'cait_xxs24_384', 'cait_xxs36_224', 'cait_xxs36_384', 'coat_lite_mini', 'coat_lite_small', 'coat_lite_tiny', 'coat_mini', 'coat_tiny', 'coatnet_0_rw_224', 'coatnet_1_rw_224', 'coatnet_bn_0_rw_224', 'coatnet_nano_rw_224', 'coatnet_rmlp_1_rw_224', 'coatnet_rmlp_2_rw_224', 'coatnet_rmlp_nano_rw_224', 'coatnext_nano_rw_224', 'convit_base', 'convit_small', 'convit_tiny', 'convmixer_768_32', 'convmixer_1024_20_ks9_p14', 'convmixer_1536_20', 'convnext_atto', 'convnext_atto_ols', 'convnext_base', 'convnext_base_384_in22ft1k', 'convnext_base_in22ft1k', 'convnext_base_in22k', 'convnext_femto', 'convnext_femto_ols', 'convnext_large', 'convnext_large_384_in22ft1k', 'convnext_large_in22ft1k', 'convnext_large_in22k', 'convnext_nano', 'convnext_nano_ols', 'convnext_pico', 'convnext_pico_ols', 'convnext_small', 'convnext_small_384_in22ft1k', 'convnext_small_in22ft1k', 'convnext_small_in22k', 'convnext_tiny', 'convnext_tiny_384_in22ft1k', 'convnext_tiny_hnf', 'convnext_tiny_in22ft1k', 'convnext_tiny_in22k', 'convnext_xlarge_384_in22ft1k', 'convnext_xlarge_in22ft1k', 'convnext_xlarge_in22k', 'crossvit_9_240', 'crossvit_9_dagger_240', 'crossvit_15_240', 'crossvit_15_dagger_240', 'crossvit_15_dagger_408', 'crossvit_18_240', 'crossvit_18_dagger_240', 'crossvit_18_dagger_408', 'crossvit_base_240', 'crossvit_small_240', 'crossvit_tiny_240', 'cs3darknet_focus_l', 'cs3darknet_focus_m', 'cs3darknet_l', 'cs3darknet_m', 'cs3darknet_x', 'cs3edgenet_x', 'cs3se_edgenet_x', 'cs3sedarknet_l', 'cs3sedarknet_x', 'cspdarknet53', 'cspresnet50', 'cspresnext50', 'darknet53', 'darknetaa53', 'deit3_base_patch16_224', 'deit3_base_patch16_224_in21ft1k', 'deit3_base_patch16_384', 'deit3_base_patch16_384_in21ft1k', 'deit3_huge_patch14_224', 'deit3_huge_patch14_224_in21ft1k', 'deit3_large_patch16_224', 'deit3_large_patch16_224_in21ft1k', 'deit3_large_patch16_384', 'deit3_large_patch16_384_in21ft1k', 'deit3_medium_patch16_224', 'deit3_medium_patch16_224_in21ft1k', 'deit3_small_patch16_224', 'deit3_small_patch16_224_in21ft1k', 'deit3_small_patch16_384', 'deit3_small_patch16_384_in21ft1k', 'deit_base_distilled_patch16_224', 'deit_base_distilled_patch16_384', 'deit_base_patch16_224', 'deit_base_patch16_384', 'deit_small_distilled_patch16_224', 'deit_small_patch16_224', 'deit_tiny_distilled_patch16_224', 'deit_tiny_patch16_224', 'densenet121', 'densenet161', 'densenet169', 'densenet201', 'densenetblur121d', 'dla34', 'dla46_c', 'dla46x_c', 'dla60', 'dla60_res2net', 'dla60_res2next', 'dla60x', 'dla60x_c', 'dla102', 'dla102x', 'dla102x2', 'dla169', 'dm_nfnet_f0', 'dm_nfnet_f1', 'dm_nfnet_f2', 'dm_nfnet_f3', 'dm_nfnet_f4', 'dm_nfnet_f5', 'dm_nfnet_f6', 'dpn68', 'dpn68b', 'dpn92', 'dpn98', 'dpn107', 'dpn131', 'eca_botnext26ts_256', 'eca_halonext26ts', 'eca_nfnet_l0', 'eca_nfnet_l1', 'eca_nfnet_l2', 'eca_resnet33ts', 'eca_resnext26ts', 'ecaresnet26t', 'ecaresnet50d', 'ecaresnet50d_pruned', 'ecaresnet50t', 'ecaresnet101d', 'ecaresnet101d_pruned', 'ecaresnet269d', 'ecaresnetlight', 'edgenext_base', 'edgenext_small', 'edgenext_small_rw', 'edgenext_x_small', 'edgenext_xx_small', 'efficientformer_l1', 'efficientformer_l3', 'efficientformer_l7', 'efficientnet_b0', 'efficientnet_b1', 'efficientnet_b1_pruned', 'efficientnet_b2', 'efficientnet_b2_pruned', 'efficientnet_b3', 'efficientnet_b3_pruned', 'efficientnet_b4', 'efficientnet_el', 'efficientnet_el_pruned', 'efficientnet_em', 'efficientnet_es', 'efficientnet_es_pruned', 'efficientnet_lite0', 'efficientnetv2_rw_m', 'efficientnetv2_rw_s', 'efficientnetv2_rw_t', 'ens_adv_inception_resnet_v2', 'ese_vovnet19b_dw', 'ese_vovnet39b', 'fbnetc_100', 'fbnetv3_b', 'fbnetv3_d', 'fbnetv3_g', 'gc_efficientnetv2_rw_t', 'gcresnet33ts', 'gcresnet50t', 'gcresnext26ts', 'gcresnext50ts', 'gcvit_base', 'gcvit_small', 'gcvit_tiny', 'gcvit_xtiny', 'gcvit_xxtiny', 'gernet_l', 'gernet_m', 'gernet_s', 'ghostnet_100', 'gluon_inception_v3', 'gluon_resnet18_v1b', 'gluon_resnet34_v1b', 'gluon_resnet50_v1b', 'gluon_resnet50_v1c', 'gluon_resnet50_v1d', 'gluon_resnet50_v1s', 'gluon_resnet101_v1b', 'gluon_resnet101_v1c', 'gluon_resnet101_v1d', 'gluon_resnet101_v1s', 'gluon_resnet152_v1b', 'gluon_resnet152_v1c', 'gluon_resnet152_v1d', 'gluon_resnet152_v1s', 'gluon_resnext50_32x4d', 'gluon_resnext101_32x4d', 'gluon_resnext101_64x4d', 'gluon_senet154', 'gluon_seresnext50_32x4d', 'gluon_seresnext101_32x4d', 'gluon_seresnext101_64x4d', 'gluon_xception65', 'gmixer_24_224', 'gmlp_s16_224', 'halo2botnet50ts_256', 'halonet26t', 'halonet50ts', 'haloregnetz_b', 'hardcorenas_a', 'hardcorenas_b', 'hardcorenas_c', 'hardcorenas_d', 'hardcorenas_e', 'hardcorenas_f', 'hrnet_w18', 'hrnet_w18_small', 'hrnet_w18_small_v2', 'hrnet_w30', 'hrnet_w32', 'hrnet_w40', 'hrnet_w44', 'hrnet_w48', 'hrnet_w64', 'ig_resnext101_32x8d', 'ig_resnext101_32x16d', 'ig_resnext101_32x32d', 'ig_resnext101_32x48d', 'inception_resnet_v2', 'inception_v3', 'inception_v4', 'jx_nest_base', 'jx_nest_small', 'jx_nest_tiny', 'lambda_resnet26rpt_256', 'lambda_resnet26t', 'lambda_resnet50ts', 'lamhalobotnet50ts_256', 'lcnet_050', 'lcnet_075', 'lcnet_100', 'legacy_senet154', 'legacy_seresnet18', 'legacy_seresnet34', 'legacy_seresnet50', 'legacy_seresnet101', 'legacy_seresnet152', 'legacy_seresnext26_32x4d', 'legacy_seresnext50_32x4d', 'legacy_seresnext101_32x4d', 'levit_128', 'levit_128s', 'levit_192', 'levit_256', 'levit_384', 'maxvit_nano_rw_256', 'maxvit_rmlp_nano_rw_256', 'maxvit_rmlp_pico_rw_256', 'maxvit_rmlp_small_rw_224', 'maxvit_rmlp_tiny_rw_256', 'maxvit_tiny_rw_224', 'maxxvit_rmlp_nano_rw_256', 'maxxvit_rmlp_small_rw_256', 'mixer_b16_224', 'mixer_b16_224_in21k', 'mixer_b16_224_miil', 'mixer_b16_224_miil_in21k', 'mixer_l16_224', 'mixer_l16_224_in21k', 'mixnet_l', 'mixnet_m', 'mixnet_s', 'mixnet_xl', 'mnasnet_100', 'mnasnet_small', 'mobilenetv2_050', 'mobilenetv2_100', 'mobilenetv2_110d', 'mobilenetv2_120d', 'mobilenetv2_140', 'mobilenetv3_large_100', 'mobilenetv3_large_100_miil', 'mobilenetv3_large_100_miil_in21k', 'mobilenetv3_rw', 'mobilenetv3_small_050', 'mobilenetv3_small_075', 'mobilenetv3_small_100', 'mobilevit_s', 'mobilevit_xs', 'mobilevit_xxs', 'mobilevitv2_050', 'mobilevitv2_075', 'mobilevitv2_100', 'mobilevitv2_125', 'mobilevitv2_150', 'mobilevitv2_150_384_in22ft1k', 'mobilevitv2_150_in22ft1k', 'mobilevitv2_175', 'mobilevitv2_175_384_in22ft1k', 'mobilevitv2_175_in22ft1k', 'mobilevitv2_200', 'mobilevitv2_200_384_in22ft1k', 'mobilevitv2_200_in22ft1k', 'mvitv2_base', 'mvitv2_large', 'mvitv2_small', 'mvitv2_tiny', 'nasnetalarge', 'nf_regnet_b1', 'nf_resnet50', 'nfnet_l0', 'pit_b_224', 'pit_b_distilled_224', 'pit_s_224', 'pit_s_distilled_224', 'pit_ti_224', 'pit_ti_distilled_224', 'pit_xs_224', 'pit_xs_distilled_224', 'pnasnet5large', 'poolformer_m36', 'poolformer_m48', 'poolformer_s12', 'poolformer_s24', 'poolformer_s36', 'pvt_v2_b0', 'pvt_v2_b1', 'pvt_v2_b2', 'pvt_v2_b2_li', 'pvt_v2_b3', 'pvt_v2_b4', 'pvt_v2_b5', 'regnetv_040', 'regnetv_064', 'regnetx_002', 'regnetx_004', 'regnetx_006', 'regnetx_008', 'regnetx_016', 'regnetx_032', 'regnetx_040', 'regnetx_064', 'regnetx_080', 'regnetx_120', 'regnetx_160', 'regnetx_320', 'regnety_002', 'regnety_004', 'regnety_006', 'regnety_008', 'regnety_016', 'regnety_032', 'regnety_040', 'regnety_064', 'regnety_080', 'regnety_120', 'regnety_160', 'regnety_320', 'regnetz_040', 'regnetz_040h', 'regnetz_b16', 'regnetz_c16', 'regnetz_c16_evos', 'regnetz_d8', 'regnetz_d8_evos', 'regnetz_d32', 'regnetz_e8', 'repvgg_a2', 'repvgg_b0', 'repvgg_b1', 'repvgg_b1g4', 'repvgg_b2', 'repvgg_b2g4', 'repvgg_b3', 'repvgg_b3g4', 'res2net50_14w_8s', 'res2net50_26w_4s', 'res2net50_26w_6s', 'res2net50_26w_8s', 'res2net50_48w_2s', 'res2net101_26w_4s', 'res2next50', 'resmlp_12_224', 'resmlp_12_224_dino', 'resmlp_12_distilled_224', 'resmlp_24_224', 'resmlp_24_224_dino', 'resmlp_24_distilled_224', 'resmlp_36_224', 'resmlp_36_distilled_224', 'resmlp_big_24_224', 'resmlp_big_24_224_in22ft1k', 'resmlp_big_24_distilled_224', 'resnest14d', 'resnest26d', 'resnest50d', 'resnest50d_1s4x24d', 'resnest50d_4s2x40d', 'resnest101e', 'resnest200e', 'resnest269e', 'resnet10t', 'resnet14t', 'resnet18', 'resnet18d', 'resnet26', 'resnet26d', 'resnet26t', 'resnet32ts', 'resnet33ts', 'resnet34', 'resnet34d', 'resnet50', 'resnet50_gn', 'resnet50d', 'resnet51q', 'resnet61q', 'resnet101', 'resnet101d', 'resnet152', 'resnet152d', 'resnet200d', 'resnetaa50', 'resnetblur50', 'resnetrs50', 'resnetrs101', 'resnetrs152', 'resnetrs200', 'resnetrs270', 'resnetrs350', 'resnetrs420', 'resnetv2_50', 'resnetv2_50d_evos', 'resnetv2_50d_gn', 'resnetv2_50x1_bit_distilled', 'resnetv2_50x1_bitm', 'resnetv2_50x1_bitm_in21k', 'resnetv2_50x3_bitm', 'resnetv2_50x3_bitm_in21k', 'resnetv2_101', 'resnetv2_101x1_bitm', 'resnetv2_101x1_bitm_in21k', 'resnetv2_101x3_bitm', 'resnetv2_101x3_bitm_in21k', 'resnetv2_152x2_bit_teacher', 'resnetv2_152x2_bit_teacher_384', 'resnetv2_152x2_bitm', 'resnetv2_152x2_bitm_in21k', 'resnetv2_152x4_bitm', 'resnetv2_152x4_bitm_in21k', 'resnext26ts', 'resnext50_32x4d', 'resnext50d_32x4d', 'resnext101_32x8d', 'resnext101_64x4d', 'rexnet_100', 'rexnet_130', 'rexnet_150', 'rexnet_200', 'sebotnet33ts_256', 'sehalonet33ts', 'selecsls42b', 'selecsls60', 'selecsls60b', 'semnasnet_075', 'semnasnet_100', 'sequencer2d_l', 'sequencer2d_m', 'sequencer2d_s', 'seresnet33ts', 'seresnet50', 'seresnet152d', 'seresnext26d_32x4d', 'seresnext26t_32x4d', 'seresnext26ts', 'seresnext50_32x4d', 'seresnext101_32x8d', 'seresnext101d_32x8d', 'seresnextaa101d_32x8d', 'skresnet18', 'skresnet34', 'skresnext50_32x4d', 'spnasnet_100', 'ssl_resnet18', 'ssl_resnet50', 'ssl_resnext50_32x4d', 'ssl_resnext101_32x4d', 'ssl_resnext101_32x8d', 'ssl_resnext101_32x16d', 'swin_base_patch4_window7_224', 'swin_base_patch4_window7_224_in22k', 'swin_base_patch4_window12_384', 'swin_base_patch4_window12_384_in22k', 'swin_large_patch4_window7_224', 'swin_large_patch4_window7_224_in22k', 'swin_large_patch4_window12_384', 'swin_large_patch4_window12_384_in22k', 'swin_s3_base_224', 'swin_s3_small_224', 'swin_s3_tiny_224', 'swin_small_patch4_window7_224', 'swin_tiny_patch4_window7_224', 'swinv2_base_window8_256', 'swinv2_base_window12_192_22k', 'swinv2_base_window12to16_192to256_22kft1k', 'swinv2_base_window12to24_192to384_22kft1k', 'swinv2_base_window16_256', 'swinv2_cr_small_224', 'swinv2_cr_small_ns_224', 'swinv2_cr_tiny_ns_224', 'swinv2_large_window12_192_22k', 'swinv2_large_window12to16_192to256_22kft1k', 'swinv2_large_window12to24_192to384_22kft1k', 'swinv2_small_window8_256', 'swinv2_small_window16_256', 'swinv2_tiny_window8_256', 'swinv2_tiny_window16_256', 'swsl_resnet18', 'swsl_resnet50', 'swsl_resnext50_32x4d', 'swsl_resnext101_32x4d', 'swsl_resnext101_32x8d', 'swsl_resnext101_32x16d', 'tf_efficientnet_b0', 'tf_efficientnet_b0_ap', 'tf_efficientnet_b0_ns', 'tf_efficientnet_b1', 'tf_efficientnet_b1_ap', 'tf_efficientnet_b1_ns', 'tf_efficientnet_b2', 'tf_efficientnet_b2_ap', 'tf_efficientnet_b2_ns', 'tf_efficientnet_b3', 'tf_efficientnet_b3_ap', 'tf_efficientnet_b3_ns', 'tf_efficientnet_b4', 'tf_efficientnet_b4_ap', 'tf_efficientnet_b4_ns', 'tf_efficientnet_b5', 'tf_efficientnet_b5_ap', 'tf_efficientnet_b5_ns', 'tf_efficientnet_b6', 'tf_efficientnet_b6_ap', 'tf_efficientnet_b6_ns', 'tf_efficientnet_b7', 'tf_efficientnet_b7_ap', 'tf_efficientnet_b7_ns', 'tf_efficientnet_b8', 'tf_efficientnet_b8_ap', 'tf_efficientnet_cc_b0_4e', 'tf_efficientnet_cc_b0_8e', 'tf_efficientnet_cc_b1_8e', 'tf_efficientnet_el', 'tf_efficientnet_em', 'tf_efficientnet_es', 'tf_efficientnet_l2_ns', 'tf_efficientnet_l2_ns_475', 'tf_efficientnet_lite0', 'tf_efficientnet_lite1', 'tf_efficientnet_lite2', 'tf_efficientnet_lite3', 'tf_efficientnet_lite4', 'tf_efficientnetv2_b0', 'tf_efficientnetv2_b1', 'tf_efficientnetv2_b2', 'tf_efficientnetv2_b3', 'tf_efficientnetv2_l', 'tf_efficientnetv2_l_in21ft1k', 'tf_efficientnetv2_l_in21k', 'tf_efficientnetv2_m', 'tf_efficientnetv2_m_in21ft1k', 'tf_efficientnetv2_m_in21k', 'tf_efficientnetv2_s', 'tf_efficientnetv2_s_in21ft1k', 'tf_efficientnetv2_s_in21k', 'tf_efficientnetv2_xl_in21ft1k', 'tf_efficientnetv2_xl_in21k', 'tf_inception_v3', 'tf_mixnet_l', 'tf_mixnet_m', 'tf_mixnet_s', 'tf_mobilenetv3_large_075', 'tf_mobilenetv3_large_100', 'tf_mobilenetv3_large_minimal_100', 'tf_mobilenetv3_small_075', 'tf_mobilenetv3_small_100', 'tf_mobilenetv3_small_minimal_100', 'tinynet_a', 'tinynet_b', 'tinynet_c', 'tinynet_d', 'tinynet_e', 'tnt_s_patch16_224', 'tresnet_l', 'tresnet_l_448', 'tresnet_m', 'tresnet_m_448', 'tresnet_m_miil_in21k', 'tresnet_v2_l', 'tresnet_xl', 'tresnet_xl_448', 'tv_densenet121', 'tv_resnet34', 'tv_resnet50', 'tv_resnet101', 'tv_resnet152', 'tv_resnext50_32x4d', 'twins_pcpvt_base', 'twins_pcpvt_large', 'twins_pcpvt_small', 'twins_svt_base', 'twins_svt_large', 'twins_svt_small', 'vgg11', 'vgg11_bn', 'vgg13', 'vgg13_bn', 'vgg16', 'vgg16_bn', 'vgg19', 'vgg19_bn', 'visformer_small', 'vit_base_patch8_224', 'vit_base_patch8_224_dino', 'vit_base_patch8_224_in21k', 'vit_base_patch16_224', 'vit_base_patch16_224_dino', 'vit_base_patch16_224_in21k', 'vit_base_patch16_224_miil', 'vit_base_patch16_224_miil_in21k', 'vit_base_patch16_224_sam', 'vit_base_patch16_384', 'vit_base_patch16_rpn_224', 'vit_base_patch32_224', 'vit_base_patch32_224_clip_laion2b', 'vit_base_patch32_224_in21k', 'vit_base_patch32_224_sam', 'vit_base_patch32_384', 'vit_base_r50_s16_224_in21k', 'vit_base_r50_s16_384', 'vit_giant_patch14_224_clip_laion2b', 'vit_huge_patch14_224_clip_laion2b', 'vit_huge_patch14_224_in21k', 'vit_large_patch14_224_clip_laion2b', 'vit_large_patch16_224', 'vit_large_patch16_224_in21k', 'vit_large_patch16_384', 'vit_large_patch32_224_in21k', 'vit_large_patch32_384', 'vit_large_r50_s32_224', 'vit_large_r50_s32_224_in21k', 'vit_large_r50_s32_384', 'vit_relpos_base_patch16_224', 'vit_relpos_base_patch16_clsgap_224', 'vit_relpos_base_patch32_plus_rpn_256', 'vit_relpos_medium_patch16_224', 'vit_relpos_medium_patch16_cls_224', 'vit_relpos_medium_patch16_rpn_224', 'vit_relpos_small_patch16_224', 'vit_small_patch8_224_dino', 'vit_small_patch16_224', 'vit_small_patch16_224_dino', 'vit_small_patch16_224_in21k', 'vit_small_patch16_384', 'vit_small_patch32_224', 'vit_small_patch32_224_in21k', 'vit_small_patch32_384', 'vit_small_r26_s32_224', 'vit_small_r26_s32_224_in21k', 'vit_small_r26_s32_384', 'vit_srelpos_medium_patch16_224', 'vit_srelpos_small_patch16_224', 'vit_tiny_patch16_224', 'vit_tiny_patch16_224_in21k', 'vit_tiny_patch16_384', 'vit_tiny_r_s16_p8_224', 'vit_tiny_r_s16_p8_224_in21k', 'vit_tiny_r_s16_p8_384', 'volo_d1_224', 'volo_d1_384', 'volo_d2_224', 'volo_d2_384', 'volo_d3_224', 'volo_d3_448', 'volo_d4_224', 'volo_d4_448', 'volo_d5_224', 'volo_d5_448', 'volo_d5_512', 'wide_resnet50_2', 'wide_resnet101_2', 'xception', 'xception41', 'xception41p', 'xception65', 'xception65p', 'xception71', 'xcit_large_24_p8_224', 'xcit_large_24_p8_224_dist', 'xcit_large_24_p8_384_dist', 'xcit_large_24_p16_224', 'xcit_large_24_p16_224_dist', 'xcit_large_24_p16_384_dist', 'xcit_medium_24_p8_224', 'xcit_medium_24_p8_224_dist', 'xcit_medium_24_p8_384_dist', 'xcit_medium_24_p16_224', 'xcit_medium_24_p16_224_dist', 'xcit_medium_24_p16_384_dist', 'xcit_nano_12_p8_224', 'xcit_nano_12_p8_224_dist', 'xcit_nano_12_p8_384_dist', 'xcit_nano_12_p16_224', 'xcit_nano_12_p16_224_dist', 'xcit_nano_12_p16_384_dist', 'xcit_small_12_p8_224', 'xcit_small_12_p8_224_dist', 'xcit_small_12_p8_384_dist', 'xcit_small_12_p16_224', 'xcit_small_12_p16_224_dist', 'xcit_small_12_p16_384_dist', 'xcit_small_24_p8_224', 'xcit_small_24_p8_224_dist', 'xcit_small_24_p8_384_dist', 'xcit_small_24_p16_224', 'xcit_small_24_p16_224_dist', 'xcit_small_24_p16_384_dist', 'xcit_tiny_12_p8_224', 'xcit_tiny_12_p8_224_dist', 'xcit_tiny_12_p8_384_dist', 'xcit_tiny_12_p16_224', 'xcit_tiny_12_p16_224_dist', 'xcit_tiny_12_p16_384_dist', 'xcit_tiny_24_p8_224', 'xcit_tiny_24_p8_224_dist', 'xcit_tiny_24_p8_384_dist', 'xcit_tiny_24_p16_224', 'xcit_tiny_24_p16_224_dist', 'xcit_tiny_24_p16_384_dist']\n"
     ]
    }
   ],
   "source": [
    "# import timm \n",
    "# avail_pretrained_models = timm.list_models(pretrained=True)\n",
    "# print(avail_pretrained_models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BaseModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(BaseModel, self).__init__()\n",
    "        self.backbone = timm.create_model('efficientnet_b0', pretrained=False)\n",
    "        self.classifier = nn.Linear(1000, 2)\n",
    "        # self.sigmoid = nn.Sigmoid()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.model(x)\n",
    "        x = nn.Sigmoid()(x)\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validation(model, criterion, val_loader, device):\n",
    "    model.eval()\n",
    "    val_loss = []\n",
    "    result = []\n",
    "    pp = []\n",
    "    pn = []\n",
    "    nn = []\n",
    "    n_p = []\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for review_img, product_img, labels in tqdm(iter(val_loader)):\n",
    "            review_img = review_img.float().to(device)\n",
    "            product_img = product_img.float().to(device)\n",
    "\n",
    "            labels = labels.type(torch.LongTensor).to(device)\n",
    "            \n",
    "            pred = model(review_img, product_img)\n",
    "            # print(pred.shape)\n",
    "            \n",
    "            loss = criterion(pred, labels)\n",
    "            val_loss.append(loss.item())\n",
    "            \n",
    "            pred = pred.detach().cpu().numpy().tolist()\n",
    "            labels = labels.detach().cpu().numpy().tolist()\n",
    "            \n",
    "            for i in range(len(pred)):\n",
    "                if labels[i] == 0:\n",
    "                    if pred[i][0] < 0.5:\n",
    "                        result.append(1) #맞게 예측하면 1\n",
    "                        pp.append(1)\n",
    "                    else:\n",
    "                        result.append(0)\n",
    "                        pn.append(1)\n",
    "                else:\n",
    "                    if pred[i][0] > 0.5:\n",
    "                        result.append(1) #맞게 예측하면 1\n",
    "                        nn.append(1)\n",
    "                    else:\n",
    "                        result.append(0)\n",
    "                        n_p.append(1)\n",
    "            # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html\n",
    "            \n",
    "        result_len = len(result)\n",
    "        print(sum(pp)/result_len)\n",
    "        print(sum(pn)/result_len)\n",
    "        print(sum(nn)/result_len)\n",
    "        print(sum(n_p)/result_len)\n",
    "        \n",
    "        _val_loss = np.mean(val_loss)\n",
    "        _val_score = (sum(result)/result_len) * 100 \n",
    "        \n",
    "        \n",
    "    \n",
    "    return _val_loss, _val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the Contrastive Loss Function\n",
    "class ContrastiveLoss(torch.nn.Module):\n",
    "    def __init__(self, margin=1.0):\n",
    "        super(ContrastiveLoss, self).__init__()\n",
    "        self.margin = margin\n",
    "\n",
    "    def forward(self, euclidean_distance, label):\n",
    "      # Calculate the euclidian distance and calculate the contrastive loss\n",
    "        # loss_contrastive = torch.mean((1-label) * torch.pow(euclidean_distance, 2) +\n",
    "        #                             (label) * torch.pow(torch.clamp(self.margin - euclidean_distance, min=0.0), 2))\n",
    "        # loss_contrastive = ((1-label) * torch.pow(euclidean_distance, 2) +\n",
    "        #                             (label) * torch.pow(torch.clamp(self.margin - euclidean_distance, min=0.0), 2))\n",
    "        \n",
    "        loss_contrastive = []\n",
    "        \n",
    "        for i in range(len(label)):\n",
    "              loss_contrastive.append((1-label[i]) * pow(euclidean_distance[i][0], 2) +\n",
    "                                    (label[i]) * pow(max(self.margin - euclidean_distance[i][0],0), 2))\n",
    "        return sum(loss_contrastive)/len(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, optimizer, train_loader, val_loader, scheduler, device):\n",
    "    model = model.to(device)\n",
    "    # criterion = nn.NLLLoss(reduction=\"sum\").to(device)\n",
    "    criterion = ContrastiveLoss().to(device)\n",
    "    \n",
    "    best_score = 0\n",
    "    best_model = None\n",
    "    \n",
    "    for epoch in range(0, CFG['EPOCHS']):\n",
    "        model.train()\n",
    "        train_loss = []\n",
    "        flag_list = []\n",
    "        \n",
    "        for review_img, product_img, labels in tqdm(iter(train_loader)):\n",
    "            \n",
    "            review_img = review_img.float().to(device)\n",
    "            product_img = product_img.float().to(device)\n",
    "                  \n",
    "            labels = labels.type(torch.LongTensor).to(device)\n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            output = model(review_img, product_img)\n",
    "            # print(output)\n",
    "            # print(output[3][0])\n",
    "            \n",
    "            # print(labels)\n",
    "            # print(labels[3])\n",
    "            \n",
    "            loss = criterion(output, labels)\n",
    "            # print(loss)\n",
    "            \n",
    "            # output = output.detach().cpu().numpy().tolist()\n",
    "            # labels = labels.detach().cpu().numpy().tolist()\n",
    "            # print(output)\n",
    "            # print(output[3][0])\n",
    "            # print(labels)\n",
    "            # print(labels[1])\n",
    "            # print(loss)\n",
    "            \n",
    "            # print(loss.item())\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            train_loss.append(loss.item())\n",
    "                    \n",
    "                \n",
    "        _val_loss, _val_score = validation(model, criterion, val_loader, device)\n",
    "        \n",
    "        _train_loss = np.mean(train_loss)\n",
    "        print(f'Epoch [{epoch}], Train Loss : [{_train_loss:.5f}] Val Loss : [{_val_loss:.5f}] Val accuracy score : [{_val_score:.5f}]')\n",
    "       \n",
    "        if scheduler is not None:\n",
    "            scheduler.step(_val_score)\n",
    "            \n",
    "        if best_score < _val_score:\n",
    "            best_score = _val_score\n",
    "            best_model = model\n",
    "            torch.save(best_model.state_dict(), \"./distance_EffNetBase_E_Contra.pt\")\n",
    "            \n",
    "            \n",
    "    return best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BaseModel(\n",
       "  (backbone): EfficientNet(\n",
       "    (conv_stem): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "    (bn1): BatchNormAct2d(\n",
       "      32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "      (drop): Identity()\n",
       "      (act): SiLU(inplace=True)\n",
       "    )\n",
       "    (blocks): Sequential(\n",
       "      (0): Sequential(\n",
       "        (0): DepthwiseSeparableConv(\n",
       "          (conv_dw): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(32, 8, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(8, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pw): Conv2d(32, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "      )\n",
       "      (1): Sequential(\n",
       "        (0): InvertedResidual(\n",
       "          (conv_pw): Conv2d(16, 96, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(96, 4, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(4, 96, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "        (1): InvertedResidual(\n",
       "          (conv_pw): Conv2d(24, 144, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(144, 144, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=144, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(144, 6, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(6, 144, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(144, 24, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "      )\n",
       "      (2): Sequential(\n",
       "        (0): InvertedResidual(\n",
       "          (conv_pw): Conv2d(24, 144, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(144, 144, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=144, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(144, 6, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(6, 144, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(144, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "        (1): InvertedResidual(\n",
       "          (conv_pw): Conv2d(40, 240, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(240, 240, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=240, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(240, 10, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(10, 240, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(240, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "      )\n",
       "      (3): Sequential(\n",
       "        (0): InvertedResidual(\n",
       "          (conv_pw): Conv2d(40, 240, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(240, 240, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=240, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(240, 10, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(10, 240, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(240, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            80, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "        (1): InvertedResidual(\n",
       "          (conv_pw): Conv2d(80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(480, 480, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=480, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(480, 20, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(20, 480, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(480, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            80, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "        (2): InvertedResidual(\n",
       "          (conv_pw): Conv2d(80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(480, 480, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=480, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(480, 20, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(20, 480, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(480, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            80, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "      )\n",
       "      (4): Sequential(\n",
       "        (0): InvertedResidual(\n",
       "          (conv_pw): Conv2d(80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(480, 480, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=480, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(480, 20, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(20, 480, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(480, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            112, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "        (1): InvertedResidual(\n",
       "          (conv_pw): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(672, 672, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=672, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            112, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "        (2): InvertedResidual(\n",
       "          (conv_pw): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(672, 672, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=672, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            112, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "      )\n",
       "      (5): Sequential(\n",
       "        (0): InvertedResidual(\n",
       "          (conv_pw): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(672, 672, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=672, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(672, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "        (1): InvertedResidual(\n",
       "          (conv_pw): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(1152, 1152, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1152, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(1152, 48, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(48, 1152, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "        (2): InvertedResidual(\n",
       "          (conv_pw): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(1152, 1152, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1152, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(1152, 48, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(48, 1152, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "        (3): InvertedResidual(\n",
       "          (conv_pw): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(1152, 1152, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1152, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(1152, 48, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(48, 1152, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "      )\n",
       "      (6): Sequential(\n",
       "        (0): InvertedResidual(\n",
       "          (conv_pw): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn1): BatchNormAct2d(\n",
       "            1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (conv_dw): Conv2d(1152, 1152, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1152, bias=False)\n",
       "          (bn2): BatchNormAct2d(\n",
       "            1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): SiLU(inplace=True)\n",
       "          )\n",
       "          (se): SqueezeExcite(\n",
       "            (conv_reduce): Conv2d(1152, 48, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (act1): SiLU(inplace=True)\n",
       "            (conv_expand): Conv2d(48, 1152, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (gate): Sigmoid()\n",
       "          )\n",
       "          (conv_pwl): Conv2d(1152, 320, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (bn3): BatchNormAct2d(\n",
       "            320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "            (drop): Identity()\n",
       "            (act): Identity()\n",
       "          )\n",
       "          (drop_path): Identity()\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (conv_head): Conv2d(320, 1280, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "    (bn2): BatchNormAct2d(\n",
       "      1280, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True\n",
       "      (drop): Identity()\n",
       "      (act): SiLU(inplace=True)\n",
       "    )\n",
       "    (global_pool): SelectAdaptivePool2d (pool_type=avg, flatten=Flatten(start_dim=1, end_dim=-1))\n",
       "    (classifier): Linear(in_features=1280, out_features=1000, bias=True)\n",
       "  )\n",
       "  (classifier): Linear(in_features=1000, out_features=50, bias=True)\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (ReLU): ReLU()\n",
       ")"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = BaseModel()\n",
    "model.load_state_dict(torch.load('./distance_VGGBase_E_Contra.pt'))\n",
    "model.eval()\n",
    "# print(pytorch_model_summary.summary(model, torch.zeros(8,3,256,256),max_depth=None, show_parent_layers=True, show_input=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fef0843f5ea7440d964e21e0725f1e19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1641 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2dffb6fb5ba4cc991ed7e5256b5719d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/704 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.37137777777777775\n",
      "0.1248\n",
      "0.35502222222222224\n",
      "0.1488\n",
      "Epoch [0], Train Loss : [0.19432] Val Loss : [0.18476] Val accuracy score : [72.64000]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eedea57f51e541edae2a9daa55bd3839",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1641 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "446638266cf444e59680241151894cf0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/704 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2928\n",
      "0.20337777777777777\n",
      "0.3848888888888889\n",
      "0.11893333333333334\n",
      "Epoch [1], Train Loss : [0.12688] Val Loss : [0.22306] Val accuracy score : [67.76889]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1de63398fa24cceb88bf49157011a5b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1641 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "280191360baa46a4b47c2374363a53f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/704 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.24124444444444446\n",
      "0.25493333333333335\n",
      "0.4176\n",
      "0.08622222222222223\n",
      "Epoch [2], Train Loss : [0.07122] Val Loss : [0.24801] Val accuracy score : [65.88444]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12530e7511b8467f81f3dc75e1a1dbd1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1641 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65536671f676412bbeb0516f1724e417",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/704 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.26222222222222225\n",
      "0.23395555555555556\n",
      "0.4106666666666667\n",
      "0.09315555555555556\n",
      "Epoch [3], Train Loss : [0.05439] Val Loss : [0.24023] Val accuracy score : [67.28889]\n",
      "Epoch 00004: reducing learning rate of group 0 to 1.5000e-04.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86c941250fa14966b363a75696e762fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1641 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d58d507554c2480088edf59411a09541",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/704 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.29457777777777777\n",
      "0.2016\n",
      "0.39982222222222225\n",
      "0.104\n",
      "Epoch [4], Train Loss : [0.04123] Val Loss : [0.20478] Val accuracy score : [69.44000]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c05601d35b14d979a9e5bb577cbe72d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1641 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[20], line 4\u001b[0m\n\u001b[0;32m      1\u001b[0m optimizer \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39moptim\u001b[39m.\u001b[39mAdam(params \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39mparameters(), lr \u001b[39m=\u001b[39m CFG[\u001b[39m\"\u001b[39m\u001b[39mLEARNING_RATE\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m      2\u001b[0m scheduler \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39moptim\u001b[39m.\u001b[39mlr_scheduler\u001b[39m.\u001b[39mReduceLROnPlateau(optimizer, mode\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mmax\u001b[39m\u001b[39m'\u001b[39m, factor\u001b[39m=\u001b[39m\u001b[39m0.5\u001b[39m, patience\u001b[39m=\u001b[39m\u001b[39m2\u001b[39m, threshold_mode\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mabs\u001b[39m\u001b[39m'\u001b[39m, min_lr\u001b[39m=\u001b[39m\u001b[39m1e-8\u001b[39m, verbose\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m----> 4\u001b[0m infer_model \u001b[39m=\u001b[39m train(model, optimizer, train_loader, val_loader, scheduler, device)\n",
      "Cell \u001b[1;32mIn[18], line 41\u001b[0m, in \u001b[0;36mtrain\u001b[1;34m(model, optimizer, train_loader, val_loader, scheduler, device)\u001b[0m\n\u001b[0;32m     29\u001b[0m loss \u001b[39m=\u001b[39m criterion(output, labels)\n\u001b[0;32m     30\u001b[0m \u001b[39m# print(loss)\u001b[39;00m\n\u001b[0;32m     31\u001b[0m \n\u001b[0;32m     32\u001b[0m \u001b[39m# output = output.detach().cpu().numpy().tolist()\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     39\u001b[0m \n\u001b[0;32m     40\u001b[0m \u001b[39m# print(loss.item())\u001b[39;00m\n\u001b[1;32m---> 41\u001b[0m loss\u001b[39m.\u001b[39;49mbackward()\n\u001b[0;32m     42\u001b[0m optimizer\u001b[39m.\u001b[39mstep()\n\u001b[0;32m     43\u001b[0m train_loss\u001b[39m.\u001b[39mappend(loss\u001b[39m.\u001b[39mitem())\n",
      "File \u001b[1;32mc:\\Users\\EHmin\\anaconda3\\envs\\EHmin\\lib\\site-packages\\torch\\_tensor.py:488\u001b[0m, in \u001b[0;36mTensor.backward\u001b[1;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[0;32m    478\u001b[0m \u001b[39mif\u001b[39;00m has_torch_function_unary(\u001b[39mself\u001b[39m):\n\u001b[0;32m    479\u001b[0m     \u001b[39mreturn\u001b[39;00m handle_torch_function(\n\u001b[0;32m    480\u001b[0m         Tensor\u001b[39m.\u001b[39mbackward,\n\u001b[0;32m    481\u001b[0m         (\u001b[39mself\u001b[39m,),\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    486\u001b[0m         inputs\u001b[39m=\u001b[39minputs,\n\u001b[0;32m    487\u001b[0m     )\n\u001b[1;32m--> 488\u001b[0m torch\u001b[39m.\u001b[39;49mautograd\u001b[39m.\u001b[39;49mbackward(\n\u001b[0;32m    489\u001b[0m     \u001b[39mself\u001b[39;49m, gradient, retain_graph, create_graph, inputs\u001b[39m=\u001b[39;49minputs\n\u001b[0;32m    490\u001b[0m )\n",
      "File \u001b[1;32mc:\\Users\\EHmin\\anaconda3\\envs\\EHmin\\lib\\site-packages\\torch\\autograd\\__init__.py:197\u001b[0m, in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[0;32m    192\u001b[0m     retain_graph \u001b[39m=\u001b[39m create_graph\n\u001b[0;32m    194\u001b[0m \u001b[39m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[0;32m    195\u001b[0m \u001b[39m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[0;32m    196\u001b[0m \u001b[39m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[1;32m--> 197\u001b[0m Variable\u001b[39m.\u001b[39;49m_execution_engine\u001b[39m.\u001b[39;49mrun_backward(  \u001b[39m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[0;32m    198\u001b[0m     tensors, grad_tensors_, retain_graph, create_graph, inputs,\n\u001b[0;32m    199\u001b[0m     allow_unreachable\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, accumulate_grad\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "optimizer = torch.optim.Adam(params = model.parameters(), lr = CFG[\"LEARNING_RATE\"])\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=2, threshold_mode='abs', min_lr=1e-8, verbose=True)\n",
    "\n",
    "infer_model = train(model, optimizer, train_loader, val_loader, scheduler, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "SAM",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}