BERT SC.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a8ebeab2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in ./venv/lib/python3.9/site-packages (4.20.1)\n",
      "Requirement already satisfied: requests in ./venv/lib/python3.9/site-packages (from transformers) (2.28.1)\n",
      "Requirement already satisfied: tqdm>=4.27 in ./venv/lib/python3.9/site-packages (from transformers) (4.64.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in ./venv/lib/python3.9/site-packages (from transformers) (0.8.1)\n",
      "Requirement already satisfied: numpy>=1.17 in ./venv/lib/python3.9/site-packages (from transformers) (1.23.0)\n",
      "Requirement already satisfied: packaging>=20.0 in ./venv/lib/python3.9/site-packages (from transformers) (21.3)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in ./venv/lib/python3.9/site-packages (from transformers) (0.12.1)\n",
      "Requirement already satisfied: filelock in ./venv/lib/python3.9/site-packages (from transformers) (3.7.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in ./venv/lib/python3.9/site-packages (from transformers) (2022.6.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in ./venv/lib/python3.9/site-packages (from transformers) (6.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./venv/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.3.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in ./venv/lib/python3.9/site-packages (from packaging>=20.0->transformers) (3.0.9)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in ./venv/lib/python3.9/site-packages (from requests->transformers) (2.1.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./venv/lib/python3.9/site-packages (from requests->transformers) (1.26.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in ./venv/lib/python3.9/site-packages (from requests->transformers) (2022.6.15)\n",
      "Requirement already satisfied: idna<4,>=2.5 in ./venv/lib/python3.9/site-packages (from requests->transformers) (3.3)\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "550ed923",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import torch\n",
    "import numpy as np\n",
    "from transformers import BertTokenizer, BertModel\n",
    "from torch import nn\n",
    "from torch.optim import Adam\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ac129d7e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>CASE</th>\n",
       "      <th>Q4 ANSWER</th>\n",
       "      <th>Q4 Score</th>\n",
       "      <th>Q9 ANSWER</th>\n",
       "      <th>Q9 Score</th>\n",
       "      <th>Score 1-4 Q4</th>\n",
       "      <th>Ashray Score</th>\n",
       "      <th>Score 1-4 Q9</th>\n",
       "      <th>Ashray Score.1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Case 2a</td>\n",
       "      <td>Yes, offering the discounts will help increase...</td>\n",
       "      <td>44</td>\n",
       "      <td>In a normal scenario, the client would pay 100...</td>\n",
       "      <td>44</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Case 2a</td>\n",
       "      <td>It depends. If they want to promote the use of...</td>\n",
       "      <td>33</td>\n",
       "      <td>If for every $10 increase the possibility of a...</td>\n",
       "      <td>78</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Case 2a</td>\n",
       "      <td>the introduction is very profitable from Vivit...</td>\n",
       "      <td>0</td>\n",
       "      <td>the increase in premiums corresponds to an inc...</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Case 2a</td>\n",
       "      <td>To decide whether this is a good idea, I would...</td>\n",
       "      <td>0</td>\n",
       "      <td>The expected cost increase by 150$ per opt-out...</td>\n",
       "      <td>44</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Case 2a</td>\n",
       "      <td>Ich würde empfehlen das Angebot auch den beste...</td>\n",
       "      <td>6</td>\n",
       "      <td>`-</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0     CASE                                          Q4 ANSWER  \\\n",
       "0         NaN  Case 2a  Yes, offering the discounts will help increase...   \n",
       "1         NaN  Case 2a  It depends. If they want to promote the use of...   \n",
       "2         NaN  Case 2a  the introduction is very profitable from Vivit...   \n",
       "3         NaN  Case 2a  To decide whether this is a good idea, I would...   \n",
       "4         NaN  Case 2a  Ich würde empfehlen das Angebot auch den beste...   \n",
       "\n",
       "   Q4 Score                                          Q9 ANSWER  Q9 Score  \\\n",
       "0        44  In a normal scenario, the client would pay 100...        44   \n",
       "1        33  If for every $10 increase the possibility of a...        78   \n",
       "2         0  the increase in premiums corresponds to an inc...        33   \n",
       "3         0  The expected cost increase by 150$ per opt-out...        44   \n",
       "4         6                                                 `-         0   \n",
       "\n",
       "   Score 1-4 Q4  Ashray Score  Score 1-4 Q9  Ashray Score.1  \n",
       "0             2           NaN           2.0             2.0  \n",
       "1             2           NaN           4.0             2.0  \n",
       "2             1           NaN           2.0             4.0  \n",
       "3             1           NaN           2.0             1.0  \n",
       "4             1           NaN           1.0             1.0  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_excel('data.xlsx')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "a456b90c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Q4 ANSWER</th>\n",
       "      <th>Q4 Score</th>\n",
       "      <th>Score 1-4 Q4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Yes, offering the discounts will help increase...</td>\n",
       "      <td>44</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>It depends. If they want to promote the use of...</td>\n",
       "      <td>33</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>the introduction is very profitable from Vivit...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>To decide whether this is a good idea, I would...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Existing customers might feel neglected if a p...</td>\n",
       "      <td>33</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Q4 ANSWER  Q4 Score  Score 1-4 Q4\n",
       "0  Yes, offering the discounts will help increase...        44             2\n",
       "1  It depends. If they want to promote the use of...        33             2\n",
       "2  the introduction is very profitable from Vivit...         0             1\n",
       "3  To decide whether this is a good idea, I would...         0             1\n",
       "5  Existing customers might feel neglected if a p...        33             2"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q4_df = df[[\"Q4 ANSWER\",\"Q4 Score\",\"Score 1-4 Q4\"]]\n",
    "#dropping one entry in German\n",
    "q4_df = q4_df.drop(4)\n",
    "q4_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7860f4f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Q9 ANSWER</th>\n",
       "      <th>Q9 Score</th>\n",
       "      <th>Score 1-4 Q9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>In a normal scenario, the client would pay 100...</td>\n",
       "      <td>44</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>If for every $10 increase the possibility of a...</td>\n",
       "      <td>78</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>the increase in premiums corresponds to an inc...</td>\n",
       "      <td>33</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The expected cost increase by 150$ per opt-out...</td>\n",
       "      <td>44</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>`-</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Q9 ANSWER  Q9 Score  Score 1-4 Q9\n",
       "0  In a normal scenario, the client would pay 100...        44           2.0\n",
       "1  If for every $10 increase the possibility of a...        78           4.0\n",
       "2  the increase in premiums corresponds to an inc...        33           2.0\n",
       "3  The expected cost increase by 150$ per opt-out...        44           2.0\n",
       "4                                                 `-         0           1.0"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q9_df = df[[\"Q9 ANSWER\",\"Q9 Score\",\"Score 1-4 Q9\"]]\n",
    "q9_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "291ad9a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "q4_df = q4_df.rename(columns = {\"Q4 ANSWER\":\"text\",\"Q4 Score\":\"score_100\",\"Score 1-4 Q4\":\"score_4\"})\n",
    "q9_df = q9_df.rename(columns = {\"Q9 ANSWER\":\"text\",\"Q9 Score\":\"score_100\",\"Score 1-4 Q9\":\"score_4\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "9a246ea9",
   "metadata": {},
   "outputs": [],
   "source": [
    "q4_df = q4_df[q4_df.text != \"`-\"]\n",
    "q9_df = q9_df[q9_df.text != \"`-\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "8ffc83e4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of enteries for Q9  196\n",
      "Number of enteries for Q4  301\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of enteries for Q9 \",q9_df.shape[0])\n",
    "print(\"Number of enteries for Q4 \",q4_df.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "64b0e230",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /Users/uday/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "stop_words = set(stopwords.words('english'))\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "st = SnowballStemmer('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "569ce7e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to clean data\n",
    "def clean_data(df, col, clean_col):\n",
    "\n",
    "    # change to lower and remove spaces on either side\n",
    "    #df[clean_col] = df[col].apply(lambda x: x.lower().strip())\n",
    "\n",
    "    # remove extra spaces in between\n",
    "    #df[clean_col] = df[clean_col].apply(lambda x: re.sub(' +', ' ', x))\n",
    "\n",
    "    # remove punctuation\n",
    "    df[clean_col] = df[col].apply(lambda x: re.sub('[^a-zA-Z]', ' ', x))\n",
    "\n",
    "    # remove stopwords and get the stem\n",
    "    df[clean_col] = df[clean_col].apply(lambda x: ' '.join(st.stem(text) for text in x.split() if text not in stop_words))\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "de35134c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(df, col):\n",
    "    # min-max feature scaling\n",
    "    df[col] = (df[col] - df[col].min()) / (df[col].max() - df[col].min())\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "57ec093c",
   "metadata": {},
   "outputs": [],
   "source": [
    "q4_df = clean_data(q4_df, 'text', 'text')\n",
    "q9_df = clean_data(q9_df, 'text', 'text')\n",
    "\n",
    "# q4_df = normalize(q4_df,'score_4')\n",
    "#q4_df = normalize(q4_df,'score_100')\n",
    "# q9_df = normalize(q9_df,'score_4')\n",
    "#q9_df = normalize(q9_df,'score_100')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e90c3a28",
   "metadata": {},
   "outputs": [],
   "source": [
    "q9_df[\"category\"] = ['correct' if x>=.5 else 'wrong' for x in list(q9_df['score_100'])]\n",
    "q4_df[\"category\"] = ['correct' if x>=.5 else 'wrong' for x in list(q4_df['score_100'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "96b1698e",
   "metadata": {},
   "outputs": [],
   "source": [
    "q4 = q4_df[[\"text\",\"category\"]]\n",
    "q9 = q9_df[[\"text\",\"category\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "ae58adb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-cased')\n",
    "labels = {'wrong':0,\n",
    "          'correct':1\n",
    "          }\n",
    "\n",
    "class Dataset(torch.utils.data.Dataset):\n",
    "\n",
    "    def __init__(self, df):\n",
    "\n",
    "        self.labels = [labels[label] for label in df['category']]\n",
    "        self.texts = [tokenizer(text, \n",
    "                               padding='max_length', max_length = 512, truncation=True,\n",
    "                                return_tensors=\"pt\") for text in df['text']]\n",
    "\n",
    "    def classes(self):\n",
    "        return self.labels\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "    def get_batch_labels(self, idx):\n",
    "        # Fetch a batch of labels\n",
    "        return np.array(self.labels[idx])\n",
    "\n",
    "    def get_batch_texts(self, idx):\n",
    "        # Fetch a batch of inputs\n",
    "        return self.texts[idx]\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        batch_texts = self.get_batch_texts(idx)\n",
    "        batch_y = self.get_batch_labels(idx)\n",
    "\n",
    "        return batch_texts, batch_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "c1405eaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BertClassifier(nn.Module):\n",
    "\n",
    "    def __init__(self, dropout=0.5):\n",
    "\n",
    "        super(BertClassifier, self).__init__()\n",
    "\n",
    "        self.bert = BertModel.from_pretrained('bert-base-cased')\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.linear = nn.Linear(768, 2)\n",
    "        self.relu = nn.ReLU()\n",
    "\n",
    "    def forward(self, input_id, mask):\n",
    "\n",
    "        _, pooled_output = self.bert(input_ids= input_id, attention_mask=mask,return_dict=False)\n",
    "        dropout_output = self.dropout(pooled_output)\n",
    "        linear_output = self.linear(dropout_output)\n",
    "        final_layer = self.relu(linear_output)\n",
    "\n",
    "        return final_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "afa0b870",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, train_data, val_data, learning_rate, epochs):\n",
    "\n",
    "    train, val = Dataset(train_data), Dataset(val_data)\n",
    "\n",
    "    train_dataloader = torch.utils.data.DataLoader(train, batch_size=32, shuffle=True)\n",
    "    val_dataloader = torch.utils.data.DataLoader(val, batch_size=32)\n",
    "\n",
    "    use_cuda = torch.cuda.is_available()\n",
    "    device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    optimizer = Adam(model.parameters(), lr= learning_rate)\n",
    "\n",
    "    if use_cuda:\n",
    "\n",
    "            model = model.cuda()\n",
    "            criterion = criterion.cuda()\n",
    "\n",
    "    for epoch_num in range(epochs):\n",
    "\n",
    "            total_acc_train = 0\n",
    "            total_loss_train = 0\n",
    "\n",
    "            for train_input, train_label in tqdm(train_dataloader):\n",
    "\n",
    "                train_label = train_label.to(device)\n",
    "                mask = train_input['attention_mask'].to(device)\n",
    "                input_id = train_input['input_ids'].squeeze(1).to(device)\n",
    "\n",
    "                output = model(input_id, mask)\n",
    "                \n",
    "                batch_loss = criterion(output, train_label.long())\n",
    "                total_loss_train += batch_loss.item()\n",
    "                \n",
    "                acc = (output.argmax(dim=1) == train_label).sum().item()\n",
    "                total_acc_train += acc\n",
    "\n",
    "                model.zero_grad()\n",
    "                batch_loss.backward()\n",
    "                optimizer.step()\n",
    "            \n",
    "            total_acc_val = 0\n",
    "            total_loss_val = 0\n",
    "\n",
    "            with torch.no_grad():\n",
    "\n",
    "                for val_input, val_label in val_dataloader:\n",
    "\n",
    "                    val_label = val_label.to(device)\n",
    "                    mask = val_input['attention_mask'].to(device)\n",
    "                    input_id = val_input['input_ids'].squeeze(1).to(device)\n",
    "\n",
    "                    output = model(input_id, mask)\n",
    "\n",
    "                    batch_loss = criterion(output, val_label.long())\n",
    "                    total_loss_val += batch_loss.item()\n",
    "                    \n",
    "                    acc = (output.argmax(dim=1) == val_label).sum().item()\n",
    "                    total_acc_val += acc\n",
    "            \n",
    "            print(\n",
    "                f'Epochs: {epoch_num + 1} | Train Loss: {total_loss_train / len(train_data): .3f} | Train Accuracy: {total_acc_train / len(train_data): .3f} | Val Loss: {total_loss_val / len(val_data): .3f} | Val Accuracy: {total_acc_val / len(val_data): .3f}')\n",
    "                  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "631b0e25",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "\n",
    "def evaluate(model, test_data):\n",
    "\n",
    "    test = Dataset(test_data)\n",
    "\n",
    "    test_dataloader = torch.utils.data.DataLoader(test, batch_size=2)\n",
    "\n",
    "    use_cuda = torch.cuda.is_available()\n",
    "    device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "\n",
    "    if use_cuda:\n",
    "\n",
    "        model = model.cuda()\n",
    "\n",
    "    total_acc_test = 0\n",
    "\n",
    "    predictions =[]\n",
    "    class_labels = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for test_input, test_label in test_dataloader:\n",
    "\n",
    "            test_label = test_label.to(device)\n",
    "            mask = test_input['attention_mask'].to(device)\n",
    "            input_id = test_input['input_ids'].squeeze(1).to(device)\n",
    "\n",
    "            output = model(input_id, mask)\n",
    "\n",
    "            predictions.append(output.argmax(dim=1))\n",
    "            class_labels.append(test_label)\n",
    "\n",
    "            acc = (output.argmax(dim=1) == test_label).sum().item()\n",
    "            total_acc_test += acc\n",
    "\n",
    "    \n",
    "    print(f'Test Accuracy: {total_acc_test / len(test_data): .3f}')\n",
    "\n",
    "    # Printing reports\n",
    "    print(classification_report(class_labels, predictions, labels=[0,1])) #classification report from sklearn\n",
    "    cnf_matrix = confusion_matrix(class_labels, predictions, labels=[0,1])\n",
    "    plt.imshow(cnf_matrix, cmap=plt.cm.Blues) #plot confusion matrix grid\n",
    "    threshold = cnf_matrix.max() / 2 #threshold to define text color\n",
    "    for i in range(cnf_matrix.shape[0]): #print text in grid\n",
    "        for j in range(cnf_matrix.shape[1]): \n",
    "            plt.text(j, i, cnf_matrix[i,j], color=\"w\" if cnf_matrix[i,j] > threshold else 'black')\n",
    "    tick_marks = np.arange(len(labels)) #define labeling spacing based on number of classes\n",
    "    plt.xticks(tick_marks, labels, rotation=45)\n",
    "    plt.yticks(tick_marks, labels)\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.colorbar()\n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "dfb78680",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "137 59\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(112)\n",
    "df_train, df_val = np.split(q9.sample(frac=1, random_state=42), \n",
    "                                     [int(.7*len(q9))])\n",
    "\n",
    "print(len(df_train),len(df_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "df618191",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertModel: ['cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:25<00:00, 53.10s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 1 | Train Loss:  0.025 | Train Accuracy:  0.387 | Val Loss:  0.022 | Val Accuracy:  0.525\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:13<00:00, 50.67s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 2 | Train Loss:  0.025 | Train Accuracy:  0.453 | Val Loss:  0.023 | Val Accuracy:  0.508\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:07<00:00, 49.58s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 3 | Train Loss:  0.024 | Train Accuracy:  0.569 | Val Loss:  0.023 | Val Accuracy:  0.508\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:37<00:00, 55.47s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 4 | Train Loss:  0.022 | Train Accuracy:  0.657 | Val Loss:  0.021 | Val Accuracy:  0.644\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:52<00:00, 58.47s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 5 | Train Loss:  0.020 | Train Accuracy:  0.774 | Val Loss:  0.019 | Val Accuracy:  0.695\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:18<00:00, 51.77s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 6 | Train Loss:  0.019 | Train Accuracy:  0.810 | Val Loss:  0.019 | Val Accuracy:  0.712\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:27<00:00, 53.54s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 7 | Train Loss:  0.017 | Train Accuracy:  0.803 | Val Loss:  0.021 | Val Accuracy:  0.729\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [03:48<00:00, 45.68s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 8 | Train Loss:  0.019 | Train Accuracy:  0.818 | Val Loss:  0.022 | Val Accuracy:  0.729\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:28<00:00, 53.75s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 9 | Train Loss:  0.019 | Train Accuracy:  0.810 | Val Loss:  0.020 | Val Accuracy:  0.712\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [04:51<00:00, 58.31s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 10 | Train Loss:  0.015 | Train Accuracy:  0.825 | Val Loss:  0.022 | Val Accuracy:  0.729\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 10\n",
    "model = BertClassifier()\n",
    "LR = 1e-6\n",
    "\n",
    "train(model, df_train, df_val, LR, EPOCHS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "c97b0acf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "210 91\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(112)\n",
    "df_train, df_val = np.split(q4.sample(frac=1, random_state=42), \n",
    "                                     [int(.7*len(q4))])\n",
    "\n",
    "print(len(df_train),len(df_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "f98902a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertModel: ['cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [07:09<00:00, 61.33s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 1 | Train Loss:  0.026 | Train Accuracy:  0.248 | Val Loss:  0.024 | Val Accuracy:  0.374\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [07:33<00:00, 64.75s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 2 | Train Loss:  0.024 | Train Accuracy:  0.352 | Val Loss:  0.023 | Val Accuracy:  0.440\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [10:43<00:00, 91.94s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 3 | Train Loss:  0.024 | Train Accuracy:  0.381 | Val Loss:  0.023 | Val Accuracy:  0.341\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [10:27<00:00, 89.61s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 4 | Train Loss:  0.022 | Train Accuracy:  0.414 | Val Loss:  0.022 | Val Accuracy:  0.451\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [12:34<00:00, 107.76s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 5 | Train Loss:  0.022 | Train Accuracy:  0.490 | Val Loss:  0.021 | Val Accuracy:  0.571\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [18:27<00:00, 158.25s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 6 | Train Loss:  0.021 | Train Accuracy:  0.490 | Val Loss:  0.021 | Val Accuracy:  0.582\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [08:11<00:00, 70.24s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 7 | Train Loss:  0.021 | Train Accuracy:  0.567 | Val Loss:  0.020 | Val Accuracy:  0.626\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [07:09<00:00, 61.31s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 8 | Train Loss:  0.021 | Train Accuracy:  0.643 | Val Loss:  0.020 | Val Accuracy:  0.582\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [06:41<00:00, 57.35s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 9 | Train Loss:  0.019 | Train Accuracy:  0.681 | Val Loss:  0.020 | Val Accuracy:  0.692\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [06:24<00:00, 54.92s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 10 | Train Loss:  0.020 | Train Accuracy:  0.686 | Val Loss:  0.020 | Val Accuracy:  0.670\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 10\n",
    "model = BertClassifier()\n",
    "LR = 1e-6\n",
    "\n",
    "train(model, df_train, df_val, LR, EPOCHS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3607f766",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc91a169",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}