perturbation_data_generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69ceb5fc-0154-4182-9390-741441c28f7a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from openai import AzureOpenAI\n",
    "import pandas as pd\n",
    "import re\n",
    "import time\n",
    "import csv\n",
    "import json\n",
    "import os\n",
    "import random\n",
    "import string\n",
    "import os\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed6a0fdd-6bc1-4f2b-8d04-091a688656b2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "client = AzureOpenAI(\n",
    "    azure_endpoint = \" \",\n",
    "    api_key=\"<your_api_key>\",\n",
    "    api_version=\"<date>\"\n",
    ") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bdbc600-26ea-48e5-a1ec-138ab891f6ac",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "1714ef25-d877-4c03-89d5-b6a5b4af1418",
   "metadata": {},
   "source": [
    "# Read the source data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47a8fb59-c816-44b4-b975-eccf6e44f2fa",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#df = pd.read_csv(\"NumerSense.tsv\", delimiter='\\t', names=[\"Question\", \"Answer\"])\n",
    "\n",
    "#df = pd.read_csv(\"CosmosQA.csv\")\n",
    "\n",
    "#df = pd.read_json('Social_IQA/dev.jsonl', lines=True)\n",
    "\n",
    "#df = pd.read_json('RiddleSense.jsonl', lines=True)\n",
    "#df['question_stem'] = df['question'].apply(lambda x: x['stem'])\n",
    "\n",
    "#df = pd.read_csv(\"ETHICS_test_hard.csv\")\n",
    "\n",
    "#df = pd.read_json('CommonsenseQA.jsonl', lines=True)\n",
    "#df['question_stem'] = df['question'].apply(lambda x: x['stem'])\n",
    "\n",
    "#df = pd.read_json('LogiQA/MRC.txt', lines=True)\n",
    "\n",
    "#df = pd.read_json('LogiQA/NLI.txt', lines=True)\n",
    "\n",
    "#df = pd.read_json('QASC.jsonl', lines=True)\n",
    "#df['question_stem'] = df['question'].apply(lambda x: x['stem'])\n",
    "\n",
    "#df = pd.read_json('PIQA/valid.jsonl', lines=True)\n",
    "\n",
    "#with open('ReClor.json', 'r') as file:\n",
    " #   data = json.load(file)\n",
    "#df = pd.DataFrame(data)\n",
    "\n",
    "#df = pd.read_csv(\"ART.csv\")\n",
    "#df['combined_observations'] = df['observation_1'] + ' ' + df['observation_2']\n",
    "\n",
    "#df = pd.read_json(\"GSM8K.jsonl\", lines=True)\n",
    "\n",
    "#df1 = pd.read_csv(\"AQuA_RAT/test.csv\")\n",
    "#df2 = pd.read_csv(\"AQuA_RAT/val.csv\")\n",
    "#df = pd.concat([df1, df2])\n",
    "\n",
    "#df1 = pd.read_csv(\"TRAM/typical_time_mcq.csv\")\n",
    "#df2 = pd.read_csv(\"TRAM/typical_time_shots_mcq.csv\")\n",
    "#df = pd.concat([df1, df2])\n",
    "#df = df.loc[df['Category'] == 'Commonsense']\n",
    "\n",
    "#df = pd.read_csv(\"MMLU/elementary_mathematics_test.csv\", header=None, names=['Question', 'Option A', 'Option B', 'Option C', 'Option D', 'Answer'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0dd21a26-b2c7-4f1c-9b08-2f67dc0668cf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "'''\n",
    "# LogiQA/NLI\n",
    "df = df[['label', 'major_premise', 'conclusion', 'minor_premise']]\n",
    "def remove_empty_list_rows(df, columns):\n",
    "    for column in columns:\n",
    "        df = df[df[column].apply(lambda x: len(x) != 0)]\n",
    "    return df\n",
    "\n",
    "# Columns to check for empty lists\n",
    "columns_to_check = ['major_premise', 'minor_premise']\n",
    "\n",
    "# Remove rows with empty lists in specified columns\n",
    "cleaned_df = remove_empty_list_rows(df, columns_to_check)\n",
    "cleaned_df['major_premise'] = cleaned_df['major_premise'].apply(lambda x: x[0] if isinstance(x, list) else x)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5137c396-4661-447c-9326-b8c9b87bb035",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "445733cb-75ec-4e9a-a1cd-dfdfa52e0c3d",
   "metadata": {},
   "source": [
    "# Red Herrings with GPT-4o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c70acbed-c6e6-4620-a680-ba5a337981e7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# general datasets (except MMLU) for Red Herrings\n",
    "system_prompt = \"You are an expert in creating perturbations. For the provided text, generate Red Herrings—contextually plausible but irrelevant information—that do not alter the final answer.\"\n",
    "generated_red_hearrings = []\n",
    "for i, row in df.iterrows():\n",
    "    user_prompt = f\"\"\"Given the statement: '{row['context']}', generate a single Red Herring (contextually plausible but irrelevant information) either before, after, or within the original text to challenge the LLMs while keeping the original text and final answer intact. Ensure that the Red Herring does not provide or hint at the masked information or the final answer. Output the statement with Red Herrings only.\"\"\"\n",
    "    #user_prompt = f\"\"\"Given the goal: '{row['goal']}' and solutions: '{row['sol1']}' and '{row['sol2']}', generate a single Red Herring (contextually plausible but irrelevant information) either before, after, or within the goal statement to challenge the LLMs while keeping the original goal and solutions intact. Output the goal with Red Herrings only.\"\"\"\n",
    "    #user_prompt = f\"\"\"Given the math problem: '{row['Question']}', generate a single Red Herring (contextually plausible but irrelevant information) either before, after, or within the original text. The red herring should be a detail that is not relevant to the calculations or logic required to solve the problem, but fits naturally within the context of the problem. Ensure the original text and final answer remain intact. Output the math problem with Red Herrings only.\"\"\"\n",
    "    try:\n",
    "        response = client.chat.completions.create(\n",
    "        model=\"gpt-4o\",\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": system_prompt},\n",
    "            {\"role\": \"user\", \"content\": user_prompt}\n",
    "        ])\n",
    "        content = response.choices[0].message.content\n",
    "        time.sleep(0.1)\n",
    "        if i % 50 == 0:\n",
    "            print('Progress:', i, 'finished')\n",
    "    except Exception as e:\n",
    "        print('Error:', i, str(e)[:8])\n",
    "        content = None\n",
    "        \n",
    "    generated_red_hearrings.append(content)\n",
    "\n",
    "df['context_red_herrings'] = generated_red_hearrings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3497843",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# MMLU for Red Herrings\n",
    "directory = 'MMLU'\n",
    "\n",
    "# Function to process a single CSV file\n",
    "def process_csv(file_path):\n",
    "    df = pd.read_csv(file_path, header=None, names=['Question', 'Option A', 'Option B', 'Option C', 'Option D', 'Answer'])\n",
    "    system_prompt = \"You are an expert in creating perturbations. For the provided text, generate Red Herrings—contextually plausible but irrelevant information—that do not alter the final answer.\"\n",
    "    generated_red_herrings = []\n",
    "    \n",
    "    for i, row in df.iterrows():\n",
    "        user_prompt = f\"\"\"Given the statement: '{row['Question']}', generate a single Red Herring (contextually plausible but irrelevant information) either before, after, or within the original text to challenge the LLMs while keeping the original text and final answer intact. Ensure that the Red Herring does not provide or hint at the masked information or the final answer. Output the statement with Red Herrings only.\"\"\"\n",
    "        \n",
    "        try:\n",
    "            response = client.chat.completions.create(\n",
    "            model=\"gpt-4\",\n",
    "            messages=[\n",
    "                {\"role\": \"system\", \"content\": system_prompt},\n",
    "                {\"role\": \"user\", \"content\": user_prompt}\n",
    "            ])\n",
    "            content = response.choices[0].message.content\n",
    "            time.sleep(0.1)\n",
    "            if i % 50 == 0:\n",
    "                print('Progress:', i, 'finished')\n",
    "        except Exception as e:\n",
    "            print('Error:', i, str(e)[:8])\n",
    "            content = None\n",
    "        \n",
    "        generated_red_herrings.append(content)\n",
    "    \n",
    "    df['Question_red_herrings'] = generated_red_herrings\n",
    "    \n",
    "    # Generate the output file path\n",
    "    base_name = os.path.basename(file_path)\n",
    "    output_file_name = base_name.replace('_test.csv', '_RH.csv')\n",
    "    output_file_path = os.path.join('MMLU_RH', output_file_name)\n",
    "    os.makedirs(os.path.dirname(output_file_path), exist_ok=True)\n",
    "    df.to_csv(output_file_path, index=False)\n",
    "    print(f\"Processed {file_path} and saved results to {output_file_path}\")\n",
    "\n",
    "# List all CSV files in the directory\n",
    "#all_files = os.listdir(directory)\n",
    "#csv_files = [file for file in all_files if file.endswith('_test.csv')]\n",
    "csv_files = ['moral_scenarios_test.csv',\n",
    " 'us_foreign_policy_test.csv',\n",
    " 'public_relations_test.csv',\n",
    " 'global_facts_test.csv',\n",
    " 'electrical_engineering_test.csv',\n",
    " 'astronomy_test.csv',\n",
    " 'business_ethics_test.csv',\n",
    " 'jurisprudence_test.csv',\n",
    " 'high_school_chemistry_test.csv',\n",
    " 'college_physics_test.csv',\n",
    " 'professional_psychology_test.csv',\n",
    " 'marketing_test.csv',\n",
    " 'management_test.csv',\n",
    " 'virology_test.csv',\n",
    " 'international_law_test.csv',\n",
    " 'high_school_macroeconomics_test.csv',\n",
    " 'prehistory_test.csv',\n",
    " 'abstract_algebra_test.csv',\n",
    " 'high_school_physics_test.csv',\n",
    " 'formal_logic_test.csv',\n",
    " 'college_medicine_test.csv',\n",
    " 'high_school_us_history_test.csv',\n",
    " 'moral_disputes_test.csv',\n",
    " 'high_school_european_history_test.csv',\n",
    " 'clinical_knowledge_test.csv',\n",
    " 'world_religions_test.csv',\n",
    " 'high_school_microeconomics_test.csv',\n",
    " 'professional_law_test.csv',\n",
    " 'human_aging_test.csv',\n",
    " 'medical_genetics_test.csv',\n",
    " 'high_school_geography_test.csv',\n",
    " 'high_school_government_and_politics_test.csv',\n",
    " 'anatomy_test.csv',\n",
    " 'sociology_test.csv',\n",
    " 'logical_fallacies_test.csv',\n",
    " 'high_school_computer_science_test.csv',\n",
    " 'miscellaneous_test.csv',\n",
    " 'high_school_world_history_test.csv',\n",
    " 'professional_medicine_test.csv',\n",
    " 'high_school_biology_test.csv',\n",
    " 'high_school_statistics_test.csv',\n",
    " 'college_chemistry_test.csv',\n",
    " 'nutrition_test.csv',\n",
    " 'econometrics_test.csv',\n",
    " 'human_sexuality_test.csv',\n",
    " 'security_studies_test.csv',\n",
    " 'philosophy_test.csv',\n",
    " 'elementary_mathematics_test.csv',\n",
    " 'college_biology_test.csv',\n",
    " 'college_computer_science_test.csv',\n",
    " 'machine_learning_test.csv',\n",
    " 'professional_accounting_test.csv',\n",
    " 'college_mathematics_test.csv',\n",
    " 'high_school_mathematics_test.csv',\n",
    " 'high_school_psychology_test.csv',\n",
    " 'conceptual_physics_test.csv',\n",
    " 'computer_security_test.csv']\n",
    "\n",
    "# Iterate over each CSV file and process it\n",
    "for file_name in csv_files:\n",
    "    file_path = os.path.join(directory, file_name)\n",
    "    process_csv(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9e2d9f6-d1ce-4068-8a3f-cf497e2d2d1a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "feddb296-7022-4da3-86e8-a718c1db04fb",
   "metadata": {},
   "source": [
    "# Typos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3d10935-0875-4a61-8344-3845556be935",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# general datasets (except MMLU) for typos\n",
    "def swap_characters(word):\n",
    "    if len(word) < 2:\n",
    "        return word\n",
    "    idx = random.randint(0, len(word) - 2)\n",
    "    return word[:idx] + word[idx+1] + word[idx] + word[idx+2:]\n",
    "\n",
    "def insert_random_character(word):\n",
    "    idx = random.randint(0, len(word))\n",
    "    random_char = random.choice(string.ascii_letters + string.digits + string.punctuation)\n",
    "    return word[:idx] + random_char + word[idx:]\n",
    "\n",
    "def delete_random_character(word):\n",
    "    if len(word) < 2:\n",
    "        return word\n",
    "    idx = random.randint(0, len(word) - 1)\n",
    "    return word[:idx] + word[idx+1:]\n",
    "\n",
    "def replace_random_character(word):\n",
    "    if len(word) < 1:\n",
    "        return word\n",
    "    idx = random.randint(0, len(word) - 1)\n",
    "    random_char = random.choice(string.ascii_letters + string.digits + string.punctuation)\n",
    "    return word[:idx] + random_char + word[idx+1:]\n",
    "\n",
    "def apply_typo(word):\n",
    "    typo_type = random.choice(['swap', 'insert', 'delete', 'replace'])\n",
    "    \n",
    "    if typo_type == 'swap':\n",
    "        return swap_characters(word)\n",
    "    elif typo_type == 'insert':\n",
    "        return insert_random_character(word)\n",
    "    elif typo_type == 'delete':\n",
    "        return delete_random_character(word)\n",
    "    elif typo_type == 'replace':\n",
    "        return replace_random_character(word)\n",
    "\n",
    "def create_typos(text, min_typos=3, max_typos=6):\n",
    "    if not isinstance(text, str):\n",
    "        return text  # Return the original value if it's not a string\n",
    "    words = text.split()\n",
    "    num_typos = random.randint(min_typos, max_typos)\n",
    "    num_typos = min(num_typos, len(words))  # Ensure we don't attempt more typos than there are words\n",
    "    \n",
    "    typo_indices = random.sample(range(len(words)), num_typos)\n",
    "    for idx in typo_indices:\n",
    "        words[idx] = apply_typo(words[idx])\n",
    "    \n",
    "    return ' '.join(words)\n",
    "\n",
    "# Load the CSV file\n",
    "csv_file_path = ' '  # Replace with your actual file path\n",
    "df = pd.read_csv(csv_file_path)\n",
    "\n",
    "# Apply typos to the text column\n",
    "text_column = 'context'  # Replace with the name of your text column\n",
    "df['context_typos'] = df[text_column].apply(lambda x: create_typos(x, min_typos=3, max_typos=6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c8f5c57-b135-4838-a732-f704a0581871",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# MMLU for typos\n",
    "directory = 'MMLU'\n",
    "\n",
    "# Function to process a single CSV file\n",
    "def process_csv(file_path):\n",
    "    df = pd.read_csv(file_path, header=None, names=['Question', 'Option A', 'Option B', 'Option C', 'Option D', 'Answer'])\n",
    "    df['Question_typos'] = df['Question'].apply(lambda x: create_typos(x, min_typos=3, max_typos=6))\n",
    "    \n",
    "    # Generate the output file path\n",
    "    base_name = os.path.basename(file_path)\n",
    "    output_file_name = base_name.replace('_test.csv', '_Typos.csv')\n",
    "    output_file_path = os.path.join('MMLU_Typos', output_file_name)\n",
    "    os.makedirs(os.path.dirname(output_file_path), exist_ok=True)\n",
    "    df.to_csv(output_file_path, index=False)\n",
    "    print(f\"Processed {file_path} and saved results to {output_file_path}\")\n",
    "\n",
    "# List all CSV files in the directory\n",
    "#all_files = os.listdir(directory)\n",
    "#csv_files = [file for file in all_files if file.endswith('_test.csv')]\n",
    "csv_files = ['moral_scenarios_test.csv',\n",
    " 'us_foreign_policy_test.csv',\n",
    " 'public_relations_test.csv',\n",
    " 'global_facts_test.csv',\n",
    " 'electrical_engineering_test.csv',\n",
    " 'astronomy_test.csv',\n",
    " 'business_ethics_test.csv',\n",
    " 'jurisprudence_test.csv',\n",
    " 'high_school_chemistry_test.csv',\n",
    " 'college_physics_test.csv',\n",
    " 'professional_psychology_test.csv',\n",
    " 'marketing_test.csv',\n",
    " 'management_test.csv',\n",
    " 'virology_test.csv',\n",
    " 'international_law_test.csv',\n",
    " 'high_school_macroeconomics_test.csv',\n",
    " 'prehistory_test.csv',\n",
    " 'abstract_algebra_test.csv',\n",
    " 'high_school_physics_test.csv',\n",
    " 'formal_logic_test.csv',\n",
    " 'college_medicine_test.csv',\n",
    " 'high_school_us_history_test.csv',\n",
    " 'moral_disputes_test.csv',\n",
    " 'high_school_european_history_test.csv',\n",
    " 'clinical_knowledge_test.csv',\n",
    " 'world_religions_test.csv',\n",
    " 'high_school_microeconomics_test.csv',\n",
    " 'professional_law_test.csv',\n",
    " 'human_aging_test.csv',\n",
    " 'medical_genetics_test.csv',\n",
    " 'high_school_geography_test.csv',\n",
    " 'high_school_government_and_politics_test.csv',\n",
    " 'anatomy_test.csv',\n",
    " 'sociology_test.csv',\n",
    " 'logical_fallacies_test.csv',\n",
    " 'high_school_computer_science_test.csv',\n",
    " 'miscellaneous_test.csv',\n",
    " 'high_school_world_history_test.csv',\n",
    " 'professional_medicine_test.csv',\n",
    " 'high_school_biology_test.csv',\n",
    " 'high_school_statistics_test.csv',\n",
    " 'college_chemistry_test.csv',\n",
    " 'nutrition_test.csv',\n",
    " 'econometrics_test.csv',\n",
    " 'human_sexuality_test.csv',\n",
    " 'security_studies_test.csv',\n",
    " 'philosophy_test.csv',\n",
    " 'elementary_mathematics_test.csv',\n",
    " 'college_biology_test.csv',\n",
    " 'college_computer_science_test.csv',\n",
    " 'machine_learning_test.csv',\n",
    " 'professional_accounting_test.csv',\n",
    " 'college_mathematics_test.csv',\n",
    " 'high_school_mathematics_test.csv',\n",
    " 'high_school_psychology_test.csv',\n",
    " 'conceptual_physics_test.csv',\n",
    " 'computer_security_test.csv']\n",
    "# Iterate over each CSV file and process it\n",
    "for file_name in csv_files:\n",
    "    file_path = os.path.join(directory, file_name)\n",
    "    process_csv(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0eade3a-7da3-49cf-9e68-f911968d2f2d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "500ab95b-3e2a-478c-a3f2-9bd1d14e1c43",
   "metadata": {},
   "source": [
    "# Homophones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f1fda9c-ec39-4d37-9d72-af0bf8e2bd38",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# general datasets (except MMLU) for homophones\n",
    "import nltk\n",
    "from nltk.corpus import cmudict\n",
    "\n",
    "# Download the CMU Pronouncing Dictionary\n",
    "nltk.download('cmudict')\n",
    "\n",
    "# Load the CMU Pronouncing Dictionary\n",
    "prondict = cmudict.dict()\n",
    "\n",
    "def find_homophones(word):\n",
    "    word = word.lower()\n",
    "    homophones = []\n",
    "    if word in prondict:\n",
    "        word_pron = prondict[word]\n",
    "        for w, pron in prondict.items():\n",
    "            if w != word and pron == word_pron:\n",
    "                homophones.append(w)\n",
    "    return homophones\n",
    "\n",
    "def replace_with_homophones(sentence):\n",
    "    if not isinstance(sentence, str):\n",
    "        return sentence \n",
    "    words = sentence.split()\n",
    "    perturbed_sentence = []\n",
    "\n",
    "    for word in words:\n",
    "        homophones = find_homophones(word)\n",
    "        if homophones:\n",
    "            # Replace the word with a randomly chosen homophone\n",
    "            new_word = random.choice(homophones)\n",
    "            perturbed_sentence.append(new_word)\n",
    "        else:\n",
    "            perturbed_sentence.append(word)\n",
    "\n",
    "    return ' '.join(perturbed_sentence)\n",
    "\n",
    "# Load the CSV file\n",
    "csv_file_path = ' '  # Replace with your actual file path\n",
    "df = pd.read_csv(csv_file_path)\n",
    "\n",
    "# Apply typos to the text column\n",
    "text_column = 'context'  # Replace with the name of your text column\n",
    "df['context_HP'] = df[text_column].apply(lambda x: replace_with_homophones(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caf6f066-cdb5-4b28-964d-a770b988d252",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# MMLU for homophones\n",
    "directory = 'MMLU'\n",
    "\n",
    "# Function to process a single CSV file\n",
    "def process_csv(file_path):\n",
    "    df = pd.read_csv(file_path, header=None, names=['Question', 'Option A', 'Option B', 'Option C', 'Option D', 'Answer'])\n",
    "    df['Question_HP'] = df['Question'].apply(lambda x: replace_with_homophones(x))\n",
    "    \n",
    "    # Generate the output file path\n",
    "    base_name = os.path.basename(file_path)\n",
    "    output_file_name = base_name.replace('_test.csv', '_HP.csv')\n",
    "    output_file_path = os.path.join('MMLU_HP', output_file_name)\n",
    "    os.makedirs(os.path.dirname(output_file_path), exist_ok=True)\n",
    "    df.to_csv(output_file_path, index=False)\n",
    "    print(f\"Processed {file_path} and saved results to {output_file_path}\")\n",
    "\n",
    "# List all CSV files in the directory\n",
    "#all_files = os.listdir(directory)\n",
    "#csv_files = [file for file in all_files if file.endswith('_test.csv')]\n",
    "csv_files = ['moral_scenarios_test.csv',\n",
    " 'us_foreign_policy_test.csv',\n",
    " 'public_relations_test.csv',\n",
    " 'global_facts_test.csv',\n",
    " 'electrical_engineering_test.csv',\n",
    " 'astronomy_test.csv',\n",
    " 'business_ethics_test.csv',\n",
    " 'jurisprudence_test.csv',\n",
    " 'high_school_chemistry_test.csv',\n",
    " 'college_physics_test.csv',\n",
    " 'professional_psychology_test.csv',\n",
    " 'marketing_test.csv',\n",
    " 'management_test.csv',\n",
    " 'virology_test.csv',\n",
    " 'international_law_test.csv',\n",
    " 'high_school_macroeconomics_test.csv',\n",
    " 'prehistory_test.csv',\n",
    " 'abstract_algebra_test.csv',\n",
    " 'high_school_physics_test.csv',\n",
    " 'formal_logic_test.csv',\n",
    " 'college_medicine_test.csv',\n",
    " 'high_school_us_history_test.csv',\n",
    " 'moral_disputes_test.csv',\n",
    " 'high_school_european_history_test.csv',\n",
    " 'clinical_knowledge_test.csv',\n",
    " 'world_religions_test.csv',\n",
    " 'high_school_microeconomics_test.csv',\n",
    " 'professional_law_test.csv',\n",
    " 'human_aging_test.csv',\n",
    " 'medical_genetics_test.csv',\n",
    " 'high_school_geography_test.csv',\n",
    " 'high_school_government_and_politics_test.csv',\n",
    " 'anatomy_test.csv',\n",
    " 'sociology_test.csv',\n",
    " 'logical_fallacies_test.csv',\n",
    " 'high_school_computer_science_test.csv',\n",
    " 'miscellaneous_test.csv',\n",
    " 'high_school_world_history_test.csv',\n",
    " 'professional_medicine_test.csv',\n",
    " 'high_school_biology_test.csv',\n",
    " 'high_school_statistics_test.csv',\n",
    " 'college_chemistry_test.csv',\n",
    " 'nutrition_test.csv',\n",
    " 'econometrics_test.csv',\n",
    " 'human_sexuality_test.csv',\n",
    " 'security_studies_test.csv',\n",
    " 'philosophy_test.csv',\n",
    " 'elementary_mathematics_test.csv',\n",
    " 'college_biology_test.csv',\n",
    " 'college_computer_science_test.csv',\n",
    " 'machine_learning_test.csv',\n",
    " 'professional_accounting_test.csv',\n",
    " 'college_mathematics_test.csv',\n",
    " 'high_school_mathematics_test.csv',\n",
    " 'high_school_psychology_test.csv',\n",
    " 'conceptual_physics_test.csv',\n",
    " 'computer_security_test.csv']\n",
    "# Iterate over each CSV file and process it\n",
    "for file_name in csv_files:\n",
    "    file_path = os.path.join(directory, file_name)\n",
    "    process_csv(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0c33921-ee7f-4d4b-8015-1eac150904ac",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "5cff4f14-f0f1-4fa8-81e3-18c366594c5b",
   "metadata": {},
   "source": [
    "# LeetSpeak"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ef8391f-cb21-4412-980c-8a439abfc85a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# general datasets (except MMLU) for leetSpeak\n",
    "def leet_speak(word):\n",
    "    leet_dict = {\n",
    "        'a': ['4', '@', '/\\\\', 'ä', 'ª'],\n",
    "        'b': ['8', '|3', 'ß', '13'],\n",
    "        'c': ['(', '<', '[', '¢'],\n",
    "        'd': ['[)', '|)', '|>', 'cl'],\n",
    "        'e': ['3', '€', '&', 'ë'],\n",
    "        'f': ['|=', 'ph', 'ƒ'],\n",
    "        'g': ['6', '9', '&'],\n",
    "        'h': ['#', '[-]', '|-|', '}{'],\n",
    "        'i': ['1', '!', '|', '][', 'î'],\n",
    "        'j': ['_|', '_/', ';', '(/'],\n",
    "        'k': ['|<', '|{', '|(', ']<'],\n",
    "        'l': ['1', '|_', '|', '£'],\n",
    "        'm': ['|\\\\/|', '^^', '/\\\\/\\\\', '/V\\\\'],\n",
    "        'n': ['|\\\\|', '/\\\\/', '^/', '|V'],\n",
    "        'o': ['0', '()', '*', 'ö'],\n",
    "        'p': ['|D', '|*', '|o', '|º'],\n",
    "        'q': ['(,)', '0_', 'O_', '9'],\n",
    "        'r': ['|2', '|Z', '12', '®'],\n",
    "        's': ['5', '$', '§'],\n",
    "        't': ['7', '+', '†'],\n",
    "        'u': ['|_|', '(_)', 'µ'],\n",
    "        'v': ['\\\\/', '|/', '\\\\|'],\n",
    "        'w': ['\\\\/\\\\/', 'vv', '\\\\^/', 'uu'],\n",
    "        'x': ['><', '}{', ')('],\n",
    "        'y': ['`/', '¥', 'j'],\n",
    "        'z': ['2', '7_', '%', '>_'],\n",
    "        'A': ['4', '@', '/\\\\', 'Ä', 'ª'],\n",
    "        'B': ['8', '|3', 'ß', '13'],\n",
    "        'C': ['(', '<', '[', '¢'],\n",
    "        'D': ['[)', '|)', '|>', 'cl'],\n",
    "        'E': ['3', '€', '&', 'Ë'],\n",
    "        'F': ['|=', 'ph', 'ƒ'],\n",
    "        'G': ['6', '9', '&'],\n",
    "        'H': ['#', '[-]', '|-|', '}{'],\n",
    "        'I': ['1', '!', '|', '][', 'Î'],\n",
    "        'J': ['_|', '_/', ';', '(/'],\n",
    "        'K': ['|<', '|{', '|(', ']<'],\n",
    "        'L': ['1', '|_', '|', '£'],\n",
    "        'M': ['|\\\\/|', '^^', '/\\\\/\\\\', '/V\\\\'],\n",
    "        'N': ['|\\\\|', '/\\\\/', '^/', '|V'],\n",
    "        'O': ['0', '()', '*', 'Ö'],\n",
    "        'P': ['|D', '|*', '|o', '|º'],\n",
    "        'Q': ['(,)', '0_', 'O_', '9'],\n",
    "        'R': ['|2', '|Z', '12', '®'],\n",
    "        'S': ['5', '$', '§'],\n",
    "        'T': ['7', '+', '†'],\n",
    "        'U': ['|_|', '(_)', 'µ'],\n",
    "        'V': ['\\\\/', '|/', '\\\\|'],\n",
    "        'W': ['\\\\/\\\\/', 'vv', '\\\\^/', 'uu'],\n",
    "        'X': ['><', '}{', ')('],\n",
    "        'Y': ['`/', '¥', 'j'],\n",
    "        'Z': ['2', '7_', '%', '>_']\n",
    "    }\n",
    "\n",
    "    new_text = []\n",
    "    for char in word:\n",
    "        if char in leet_dict:\n",
    "            new_text.append(random.choice(leet_dict[char]))\n",
    "        else:\n",
    "            new_text.append(char)\n",
    "    return \"\".join(new_text)\n",
    "\n",
    "def determine_number_of_words_to_convert(text):\n",
    "    word_count = len(text.split())\n",
    "    \n",
    "    # Example logic to determine number of words based on text length\n",
    "    if word_count <= 10:\n",
    "        return 1\n",
    "    elif word_count <= 20:\n",
    "        return 2\n",
    "    elif word_count <= 30:\n",
    "        return 3\n",
    "    else:\n",
    "        return min(5, word_count // 5)  # Convert about 20% of words for longer texts\n",
    "\n",
    "def random_leet_speak(text):\n",
    "    if not isinstance(text, str):\n",
    "        return text  # Return the original value if it's not a string\n",
    "    words = text.split()\n",
    "    num_words_to_convert = determine_number_of_words_to_convert(text)\n",
    "    words_to_convert = random.sample(words, min(num_words_to_convert, len(words)))\n",
    "    #words_to_convert = random.sample([word for word in words if word != \"<mask>\"], min(num_words_to_convert, len(words))) # NumerSense only\n",
    "    \n",
    "    leeted_text = []\n",
    "    for word in words:\n",
    "        if word in words_to_convert:\n",
    "            leeted_text.append(leet_speak(word))\n",
    "        else:\n",
    "            leeted_text.append(word)\n",
    "    \n",
    "    return \" \".join(leeted_text)\n",
    "\n",
    "csv_file_path = ' '  # Replace with your actual file path\n",
    "df = pd.read_csv(csv_file_path)\n",
    "\n",
    "# Apply typos to the text column\n",
    "text_column = 'context'  # Replace with the name of your text column\n",
    "df['context_Leet'] = df[text_column].apply(lambda x: random_leet_speak(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52dde18b-c63d-40d7-9368-0ef00c7ae7bd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# MMLU for leetSpeak\n",
    "directory = 'MMLU'\n",
    "\n",
    "# Function to process a single CSV file\n",
    "def process_csv(file_path):\n",
    "    df = pd.read_csv(file_path, header=None, names=['Question', 'Option A', 'Option B', 'Option C', 'Option D', 'Answer'])\n",
    "    df['Question_Leet'] = df['Question'].apply(lambda x: random_leet_speak(x))\n",
    "    \n",
    "    # Generate the output file path\n",
    "    base_name = os.path.basename(file_path)\n",
    "    output_file_name = base_name.replace('_test.csv', '_Leet.csv')\n",
    "    output_file_path = os.path.join('MMLU_Leet', output_file_name)\n",
    "    os.makedirs(os.path.dirname(output_file_path), exist_ok=True)\n",
    "    df.to_csv(output_file_path, index=False)\n",
    "    print(f\"Processed {file_path} and saved results to {output_file_path}\")\n",
    "\n",
    "# List all CSV files in the directory\n",
    "#all_files = os.listdir(directory)\n",
    "#csv_files = [file for file in all_files if file.endswith('_test.csv')]\n",
    "csv_files = ['moral_scenarios_test.csv',\n",
    " 'us_foreign_policy_test.csv',\n",
    " 'public_relations_test.csv',\n",
    " 'global_facts_test.csv',\n",
    " 'electrical_engineering_test.csv',\n",
    " 'astronomy_test.csv',\n",
    " 'business_ethics_test.csv',\n",
    " 'jurisprudence_test.csv',\n",
    " 'high_school_chemistry_test.csv',\n",
    " 'college_physics_test.csv',\n",
    " 'professional_psychology_test.csv',\n",
    " 'marketing_test.csv',\n",
    " 'management_test.csv',\n",
    " 'virology_test.csv',\n",
    " 'international_law_test.csv',\n",
    " 'high_school_macroeconomics_test.csv',\n",
    " 'prehistory_test.csv',\n",
    " 'abstract_algebra_test.csv',\n",
    " 'high_school_physics_test.csv',\n",
    " 'formal_logic_test.csv',\n",
    " 'college_medicine_test.csv',\n",
    " 'high_school_us_history_test.csv',\n",
    " 'moral_disputes_test.csv',\n",
    " 'high_school_european_history_test.csv',\n",
    " 'clinical_knowledge_test.csv',\n",
    " 'world_religions_test.csv',\n",
    " 'high_school_microeconomics_test.csv',\n",
    " 'professional_law_test.csv',\n",
    " 'human_aging_test.csv',\n",
    " 'medical_genetics_test.csv',\n",
    " 'high_school_geography_test.csv',\n",
    " 'high_school_government_and_politics_test.csv',\n",
    " 'anatomy_test.csv',\n",
    " 'sociology_test.csv',\n",
    " 'logical_fallacies_test.csv',\n",
    " 'high_school_computer_science_test.csv',\n",
    " 'miscellaneous_test.csv',\n",
    " 'high_school_world_history_test.csv',\n",
    " 'professional_medicine_test.csv',\n",
    " 'high_school_biology_test.csv',\n",
    " 'high_school_statistics_test.csv',\n",
    " 'college_chemistry_test.csv',\n",
    " 'nutrition_test.csv',\n",
    " 'econometrics_test.csv',\n",
    " 'human_sexuality_test.csv',\n",
    " 'security_studies_test.csv',\n",
    " 'philosophy_test.csv',\n",
    " 'elementary_mathematics_test.csv',\n",
    " 'college_biology_test.csv',\n",
    " 'college_computer_science_test.csv',\n",
    " 'machine_learning_test.csv',\n",
    " 'professional_accounting_test.csv',\n",
    " 'college_mathematics_test.csv',\n",
    " 'high_school_mathematics_test.csv',\n",
    " 'high_school_psychology_test.csv',\n",
    " 'conceptual_physics_test.csv',\n",
    " 'computer_security_test.csv']\n",
    "# Iterate over each CSV file and process it\n",
    "for file_name in csv_files:\n",
    "    file_path = os.path.join(directory, file_name)\n",
    "    process_csv(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d36ba19-a742-49f9-978e-eb446e0fab78",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "ba3b6cdf-d2f8-4ceb-9744-0e72f92c3a9d",
   "metadata": {},
   "source": [
    "# It-cleft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0aeb3d8e-ea9a-4962-a76a-0ccf02c0505a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# general datasets (except MMLU) for It-cleft\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "def it_cleft(sentence):\n",
    "    doc = nlp(sentence)\n",
    "    subject = None\n",
    "    verb = None\n",
    "    aux = None\n",
    "    obj = None\n",
    "    other_tokens = []\n",
    "\n",
    "    for token in doc:\n",
    "        if token.dep_ == \"nsubj\":\n",
    "            subject = token\n",
    "        elif token.dep_ == \"ROOT\":\n",
    "            verb = token\n",
    "        elif token.dep_ in (\"dobj\", \"attr\"):\n",
    "            obj = token\n",
    "        elif token.dep_ == \"aux\":\n",
    "            aux = token\n",
    "        else:\n",
    "            other_tokens.append(token.text)\n",
    "    \n",
    "    if subject and verb:\n",
    "        other_text = \" \".join(other_tokens).strip()\n",
    "        if obj:\n",
    "            if aux:\n",
    "                return f\"It was {subject.text} that {aux.text} {verb.text} {obj.text} {other_text}\".strip()\n",
    "            else:\n",
    "                return f\"It was {subject.text} that {verb.text} {obj.text} {other_text}\".strip()\n",
    "        else:\n",
    "            if aux:\n",
    "                return f\"It was {subject.text} that {aux.text} {verb.text} {other_text}\".strip()\n",
    "            else:\n",
    "                return f\"It was {subject.text} that {verb.text} {other_text}\".strip()\n",
    "    return sentence\n",
    "\n",
    "def process_sentences(sentence):\n",
    "    if not isinstance(sentence, str):\n",
    "        return sentence\n",
    "    # Split by sentence boundaries\n",
    "    doc = nlp(sentence)\n",
    "    sentences = [sent.text for sent in doc.sents]\n",
    "    cleft_sentences = [it_cleft(sent) for sent in sentences]\n",
    "    return ' '.join(cleft_sentences)\n",
    "\n",
    "csv_file_path = ' '  # Replace with your actual file path\n",
    "df = pd.read_csv(csv_file_path)\n",
    "\n",
    "# Apply typos to the text column\n",
    "text_column = 'context'\n",
    "df['context_It_cleft'] = df[text_column].apply(lambda x: process_sentences(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f4063bc-127d-4313-9e90-36efd7676189",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# MMLU for It-cleft\n",
    "directory = 'MMLU'\n",
    "\n",
    "# Function to process a single CSV file\n",
    "def process_csv(file_path):\n",
    "    df = pd.read_csv(file_path, header=None, names=['Question', 'Option A', 'Option B', 'Option C', 'Option D', 'Answer'])\n",
    "    df['Question_It_cleft'] = df['Question'].apply(lambda x: process_sentences(x))\n",
    "    \n",
    "    # Generate the output file path\n",
    "    base_name = os.path.basename(file_path)\n",
    "    output_file_name = base_name.replace('_test.csv', '_It_cleft.csv')\n",
    "    output_file_path = os.path.join('MMLU_It_cleft', output_file_name)\n",
    "    os.makedirs(os.path.dirname(output_file_path), exist_ok=True)\n",
    "    df.to_csv(output_file_path, index=False)\n",
    "    print(f\"Processed {file_path} and saved results to {output_file_path}\")\n",
    "\n",
    "# List all CSV files in the directory\n",
    "#all_files = os.listdir(directory)\n",
    "#csv_files = [file for file in all_files if file.endswith('_test.csv')]\n",
    "csv_files = ['moral_scenarios_test.csv',\n",
    " 'us_foreign_policy_test.csv',\n",
    " 'public_relations_test.csv',\n",
    " 'global_facts_test.csv',\n",
    " 'electrical_engineering_test.csv',\n",
    " 'astronomy_test.csv',\n",
    " 'business_ethics_test.csv',\n",
    " 'jurisprudence_test.csv',\n",
    " 'high_school_chemistry_test.csv',\n",
    " 'college_physics_test.csv',\n",
    " 'professional_psychology_test.csv',\n",
    " 'marketing_test.csv',\n",
    " 'management_test.csv',\n",
    " 'virology_test.csv',\n",
    " 'international_law_test.csv',\n",
    " 'high_school_macroeconomics_test.csv',\n",
    " 'prehistory_test.csv',\n",
    " 'abstract_algebra_test.csv',\n",
    " 'high_school_physics_test.csv',\n",
    " 'formal_logic_test.csv',\n",
    " 'college_medicine_test.csv',\n",
    " 'high_school_us_history_test.csv',\n",
    " 'moral_disputes_test.csv',\n",
    " 'high_school_european_history_test.csv',\n",
    " 'clinical_knowledge_test.csv',\n",
    " 'world_religions_test.csv',\n",
    " 'high_school_microeconomics_test.csv',\n",
    " 'professional_law_test.csv',\n",
    " 'human_aging_test.csv',\n",
    " 'medical_genetics_test.csv',\n",
    " 'high_school_geography_test.csv',\n",
    " 'high_school_government_and_politics_test.csv',\n",
    " 'anatomy_test.csv',\n",
    " 'sociology_test.csv',\n",
    " 'logical_fallacies_test.csv',\n",
    " 'high_school_computer_science_test.csv',\n",
    " 'miscellaneous_test.csv',\n",
    " 'high_school_world_history_test.csv',\n",
    " 'professional_medicine_test.csv',\n",
    " 'high_school_biology_test.csv',\n",
    " 'high_school_statistics_test.csv',\n",
    " 'college_chemistry_test.csv',\n",
    " 'nutrition_test.csv',\n",
    " 'econometrics_test.csv',\n",
    " 'human_sexuality_test.csv',\n",
    " 'security_studies_test.csv',\n",
    " 'philosophy_test.csv',\n",
    " 'elementary_mathematics_test.csv',\n",
    " 'college_biology_test.csv',\n",
    " 'college_computer_science_test.csv',\n",
    " 'machine_learning_test.csv',\n",
    " 'professional_accounting_test.csv',\n",
    " 'college_mathematics_test.csv',\n",
    " 'high_school_mathematics_test.csv',\n",
    " 'high_school_psychology_test.csv',\n",
    " 'conceptual_physics_test.csv',\n",
    " 'computer_security_test.csv']\n",
    "# Iterate over each CSV file and process it\n",
    "for file_name in csv_files:\n",
    "    file_path = os.path.join(directory, file_name)\n",
    "    process_csv(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10977bc1-09a1-44d0-8a77-60072b5f4dec",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "81853980-e955-4037-8091-1aea91c90ad6",
   "metadata": {},
   "source": [
    "# Wh-cleft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26dabaa7-0c62-4aec-a990-f8e079689e5d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# general datasets (except MMLU) for Wh-cleft\n",
    "\n",
    "# Load the spaCy model\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "def wh_cleft(sentence):\n",
    "    doc = nlp(sentence)\n",
    "    subject = None\n",
    "    verb = None\n",
    "    aux = None\n",
    "    obj = None\n",
    "    other_tokens = []\n",
    "    wh_word = \"what\"  # Default to \"what\"\n",
    "\n",
    "    for token in doc:\n",
    "        if token.dep_ == \"nsubj\":\n",
    "            subject = token\n",
    "        elif token.dep_ == \"ROOT\":\n",
    "            verb = token\n",
    "        elif token.dep_ in (\"dobj\", \"attr\"):\n",
    "            obj = token\n",
    "        elif token.dep_ == \"aux\":\n",
    "            aux = token\n",
    "        else:\n",
    "            other_tokens.append(token.text)\n",
    "    \n",
    "    if subject and verb:\n",
    "        other_text = \" \".join(other_tokens).strip()\n",
    "        \n",
    "        # Determine the appropriate wh- word\n",
    "        if subject.ent_type_ in [\"PERSON\", \"ORG\"]:\n",
    "            wh_word = \"who\"\n",
    "        elif \"time\" in other_text.lower() or \"when\" in other_text.lower():\n",
    "            wh_word = \"when\"\n",
    "        elif \"place\" in other_text.lower() or \"where\" in other_text.lower():\n",
    "            wh_word = \"where\"\n",
    "        elif \"reason\" in other_text.lower() or \"why\" in other_text.lower():\n",
    "            wh_word = \"why\"\n",
    "        elif \"manner\" in other_text.lower() or \"how\" in other_text.lower():\n",
    "            wh_word = \"how\"\n",
    "        elif subject.ent_type_ in [\"GPE\", \"LOC\"]:\n",
    "            wh_word = \"where\"\n",
    "        else:\n",
    "            wh_word = \"what\"\n",
    "        \n",
    "        if obj:\n",
    "            if aux:\n",
    "                return f\"{wh_word.capitalize()} {aux.text} {verb.text} {obj.text} {other_text} was {subject.text}\".strip()\n",
    "            else:\n",
    "                return f\"{wh_word.capitalize()} {verb.text} {obj.text} {other_text} was {subject.text}\".strip()\n",
    "        else:\n",
    "            if aux:\n",
    "                return f\"{wh_word.capitalize()} {aux.text} {verb.text} {other_text} was {subject.text}\".strip()\n",
    "            else:\n",
    "                return f\"{wh_word.capitalize()} {verb.text} {other_text} was {subject.text}\".strip()\n",
    "    return sentence\n",
    "\n",
    "def process_sentences(sentence):\n",
    "    if not isinstance(sentence, str):\n",
    "        return sentence\n",
    "    doc = nlp(sentence)\n",
    "    sentences = [sent.text for sent in doc.sents]\n",
    "    cleft_sentences = [wh_cleft(sent) for sent in sentences]\n",
    "    return ' '.join(cleft_sentences)\n",
    "\n",
    "csv_file_path = ' '  # Replace with your actual file path\n",
    "df = pd.read_csv(csv_file_path)\n",
    "\n",
    "# Apply typos to the text column\n",
    "text_column = 'context'\n",
    "df['context_Wh_cleft'] = df[text_column].apply(lambda x: process_sentences(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43cb40db-9e8f-415f-b877-78e2319435bf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# MMLU for Wh-cleft\n",
    "directory = 'MMLU'\n",
    "\n",
    "# Function to process a single CSV file\n",
    "def process_csv(file_path):\n",
    "    df = pd.read_csv(file_path, header=None, names=['Question', 'Option A', 'Option B', 'Option C', 'Option D', 'Answer'])\n",
    "    df['Question_Wh_cleft'] = df['Question'].apply(lambda x: process_sentences(x))\n",
    "    \n",
    "    # Generate the output file path\n",
    "    base_name = os.path.basename(file_path)\n",
    "    output_file_name = base_name.replace('_test.csv', '_Wh_cleft.csv')\n",
    "    output_file_path = os.path.join('MMLU_Wh_cleft', output_file_name)\n",
    "    os.makedirs(os.path.dirname(output_file_path), exist_ok=True)\n",
    "    df.to_csv(output_file_path, index=False)\n",
    "    print(f\"Processed {file_path} and saved results to {output_file_path}\")\n",
    "\n",
    "# List all CSV files in the directory\n",
    "#all_files = os.listdir(directory)\n",
    "#csv_files = [file for file in all_files if file.endswith('_test.csv')]\n",
    "csv_files = ['moral_scenarios_test.csv',\n",
    " 'us_foreign_policy_test.csv',\n",
    " 'public_relations_test.csv',\n",
    " 'global_facts_test.csv',\n",
    " 'electrical_engineering_test.csv',\n",
    " 'astronomy_test.csv',\n",
    " 'business_ethics_test.csv',\n",
    " 'jurisprudence_test.csv',\n",
    " 'high_school_chemistry_test.csv',\n",
    " 'college_physics_test.csv',\n",
    " 'professional_psychology_test.csv',\n",
    " 'marketing_test.csv',\n",
    " 'management_test.csv',\n",
    " 'virology_test.csv',\n",
    " 'international_law_test.csv',\n",
    " 'high_school_macroeconomics_test.csv',\n",
    " 'prehistory_test.csv',\n",
    " 'abstract_algebra_test.csv',\n",
    " 'high_school_physics_test.csv',\n",
    " 'formal_logic_test.csv',\n",
    " 'college_medicine_test.csv',\n",
    " 'high_school_us_history_test.csv',\n",
    " 'moral_disputes_test.csv',\n",
    " 'high_school_european_history_test.csv',\n",
    " 'clinical_knowledge_test.csv',\n",
    " 'world_religions_test.csv',\n",
    " 'high_school_microeconomics_test.csv',\n",
    " 'professional_law_test.csv',\n",
    " 'human_aging_test.csv',\n",
    " 'medical_genetics_test.csv',\n",
    " 'high_school_geography_test.csv',\n",
    " 'high_school_government_and_politics_test.csv',\n",
    " 'anatomy_test.csv',\n",
    " 'sociology_test.csv',\n",
    " 'logical_fallacies_test.csv',\n",
    " 'high_school_computer_science_test.csv',\n",
    " 'miscellaneous_test.csv',\n",
    " 'high_school_world_history_test.csv',\n",
    " 'professional_medicine_test.csv',\n",
    " 'high_school_biology_test.csv',\n",
    " 'high_school_statistics_test.csv',\n",
    " 'college_chemistry_test.csv',\n",
    " 'nutrition_test.csv',\n",
    " 'econometrics_test.csv',\n",
    " 'human_sexuality_test.csv',\n",
    " 'security_studies_test.csv',\n",
    " 'philosophy_test.csv',\n",
    " 'elementary_mathematics_test.csv',\n",
    " 'college_biology_test.csv',\n",
    " 'college_computer_science_test.csv',\n",
    " 'machine_learning_test.csv',\n",
    " 'professional_accounting_test.csv',\n",
    " 'college_mathematics_test.csv',\n",
    " 'high_school_mathematics_test.csv',\n",
    " 'high_school_psychology_test.csv',\n",
    " 'conceptual_physics_test.csv',\n",
    " 'computer_security_test.csv']\n",
    "# Iterate over each CSV file and process it\n",
    "for file_name in csv_files:\n",
    "    file_path = os.path.join(directory, file_name)\n",
    "    process_csv(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "155dc02c-a252-4ca0-a40f-d96a3dbfd1db",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "ff9b5ebe-4e81-49d8-b594-a235d6a9d881",
   "metadata": {},
   "source": [
    "# StressTest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5bff7454-2101-4ed5-b0c9-3f38b1332b64",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# general datasets (except MMLU) for StressTest\n",
    "stress_statements = [\n",
    "    \"and true is true\",\n",
    "    \"and false is not true\",\n",
    "    \"and true is true\",\n",
    "    \"if one is equal to one\",\n",
    "    \"and the sky is blue\",\n",
    "    \"if water is wet\",\n",
    "    \"and two plus two is four\",\n",
    "    \"if the earth orbits the sun\",\n",
    "    \"and fire is hot\",\n",
    "    \"if gravity pulls objects down\"\n",
    "]\n",
    "\n",
    "def create_stress_test(statements, repetitions=5):\n",
    "    return \" \".join(random.choices(statements, k=repetitions))\n",
    "\n",
    "def insert_stress_test_randomly(sentence, statements, repetitions=5):\n",
    "    if not isinstance(sentence, str):\n",
    "        return sentence\n",
    "    stress_test = create_stress_test(statements, repetitions)\n",
    "    words = sentence.split()\n",
    "    stress_test_words = stress_test.split()\n",
    "    \n",
    "    if len(stress_test_words) >= len(words):\n",
    "        # Append stress test at the end if not enough words to insert randomly\n",
    "        return f\"{sentence} {stress_test}\"\n",
    "    \n",
    "    # Determine random insertion points\n",
    "    insert_points = sorted(random.sample(range(len(words) + 1), len(stress_test_words)))\n",
    "\n",
    "    # Insert stress test words at the determined points\n",
    "    perturbed_sentence = []\n",
    "    stress_test_index = 0\n",
    "    for i in range(len(words) + len(stress_test_words)):\n",
    "        if i in insert_points and stress_test_index < len(stress_test_words):\n",
    "            perturbed_sentence.append(stress_test_words[stress_test_index])\n",
    "            stress_test_index += 1\n",
    "        else:\n",
    "            if words:\n",
    "                perturbed_sentence.append(words.pop(0))\n",
    "    \n",
    "    return ' '.join(perturbed_sentence)\n",
    "\n",
    "csv_file_path = ' '  # Replace with your actual file path\n",
    "df = pd.read_csv(csv_file_path)\n",
    "\n",
    "# Apply typos to the text column\n",
    "text_column = 'context'\n",
    "df['context_stress'] = df[text_column].apply(lambda x: insert_stress_test_randomly(x, stress_statements, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3f1a1db-7c4a-4575-b35e-2c9fa94306d2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# MMLU for StressTest\n",
    "directory = 'MMLU'\n",
    "\n",
    "# Function to process a single CSV file\n",
    "def process_csv(file_path):\n",
    "    df = pd.read_csv(file_path, header=None, names=['Question', 'Option A', 'Option B', 'Option C', 'Option D', 'Answer'])\n",
    "    df['Question_stress'] = df['Question'].apply(lambda x: insert_stress_test_randomly(x, stress_statements, 5))\n",
    "    \n",
    "    # Generate the output file path\n",
    "    base_name = os.path.basename(file_path)\n",
    "    output_file_name = base_name.replace('_test.csv', '_stress.csv')\n",
    "    output_file_path = os.path.join('MMLU_stress', output_file_name)\n",
    "    os.makedirs(os.path.dirname(output_file_path), exist_ok=True)\n",
    "    df.to_csv(output_file_path, index=False)\n",
    "    print(f\"Processed {file_path} and saved results to {output_file_path}\")\n",
    "\n",
    "# List all CSV files in the directory\n",
    "#all_files = os.listdir(directory)\n",
    "#csv_files = [file for file in all_files if file.endswith('_test.csv')]\n",
    "csv_files = ['moral_scenarios_test.csv',\n",
    " 'us_foreign_policy_test.csv',\n",
    " 'public_relations_test.csv',\n",
    " 'global_facts_test.csv',\n",
    " 'electrical_engineering_test.csv',\n",
    " 'astronomy_test.csv',\n",
    " 'business_ethics_test.csv',\n",
    " 'jurisprudence_test.csv',\n",
    " 'high_school_chemistry_test.csv',\n",
    " 'college_physics_test.csv',\n",
    " 'professional_psychology_test.csv',\n",
    " 'marketing_test.csv',\n",
    " 'management_test.csv',\n",
    " 'virology_test.csv',\n",
    " 'international_law_test.csv',\n",
    " 'high_school_macroeconomics_test.csv',\n",
    " 'prehistory_test.csv',\n",
    " 'abstract_algebra_test.csv',\n",
    " 'high_school_physics_test.csv',\n",
    " 'formal_logic_test.csv',\n",
    " 'college_medicine_test.csv',\n",
    " 'high_school_us_history_test.csv',\n",
    " 'moral_disputes_test.csv',\n",
    " 'high_school_european_history_test.csv',\n",
    " 'clinical_knowledge_test.csv',\n",
    " 'world_religions_test.csv',\n",
    " 'high_school_microeconomics_test.csv',\n",
    " 'professional_law_test.csv',\n",
    " 'human_aging_test.csv',\n",
    " 'medical_genetics_test.csv',\n",
    " 'high_school_geography_test.csv',\n",
    " 'high_school_government_and_politics_test.csv',\n",
    " 'anatomy_test.csv',\n",
    " 'sociology_test.csv',\n",
    " 'logical_fallacies_test.csv',\n",
    " 'high_school_computer_science_test.csv',\n",
    " 'miscellaneous_test.csv',\n",
    " 'high_school_world_history_test.csv',\n",
    " 'professional_medicine_test.csv',\n",
    " 'high_school_biology_test.csv',\n",
    " 'high_school_statistics_test.csv',\n",
    " 'college_chemistry_test.csv',\n",
    " 'nutrition_test.csv',\n",
    " 'econometrics_test.csv',\n",
    " 'human_sexuality_test.csv',\n",
    " 'security_studies_test.csv',\n",
    " 'philosophy_test.csv',\n",
    " 'elementary_mathematics_test.csv',\n",
    " 'college_biology_test.csv',\n",
    " 'college_computer_science_test.csv',\n",
    " 'machine_learning_test.csv',\n",
    " 'professional_accounting_test.csv',\n",
    " 'college_mathematics_test.csv',\n",
    " 'high_school_mathematics_test.csv',\n",
    " 'high_school_psychology_test.csv',\n",
    " 'conceptual_physics_test.csv',\n",
    " 'computer_security_test.csv']\n",
    "# Iterate over each CSV file and process it\n",
    "for file_name in csv_files:\n",
    "    file_path = os.path.join(directory, file_name)\n",
    "    process_csv(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58f8e74e-a2e8-473d-961e-2962e0919054",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "68dc176c-2c1b-40ee-99f7-881e24757698",
   "metadata": {},
   "source": [
    "# CheckList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de62faf9-2191-4135-b659-1056208e52c7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# general datasets (except MMLU) for CheckList\n",
    "random_urls_handles = [\n",
    "    \"http://google.com\",\n",
    "    \"https://github.com\",\n",
    "    \"@realuser1\",\n",
    "    \"@tech_guru\",\n",
    "    \"https://bit.ly/3uA1abc\",\n",
    "    \"http://tinyurl.com/real-example\",\n",
    "    \"@famous_person\",\n",
    "    \"https://short.url/tech\",\n",
    "    \"http://example.org/news\",\n",
    "    \"@cool_handle\",\n",
    "    \"http://stackoverflow.com\",\n",
    "    \"https://twitter.com/realuser\",\n",
    "    \"@developer123\",\n",
    "    \"@random_tech\",\n",
    "    \"https://news.ycombinator.com\",\n",
    "    \"http://nytimes.com\",\n",
    "    \"https://linkedin.com/in/someone\",\n",
    "    \"@official_account\",\n",
    "    \"@news_update\",\n",
    "    \"https://medium.com/@writer\",\n",
    "    \"http://bbc.com\",\n",
    "    \"https://cnn.com\",\n",
    "    \"@follower1\",\n",
    "    \"@social_media\",\n",
    "    \"https://t.co/xyz123\",\n",
    "    \"http://espn.com\",\n",
    "    \"https://facebook.com/profile\",\n",
    "    \"@techie_life\",\n",
    "    \"@daily_news\",\n",
    "    \"http://instagram.com\",\n",
    "    \"https://youtube.com\",\n",
    "    \"@content_creator\",\n",
    "    \"https://pinterest.com\",\n",
    "    \"http://reddit.com\",\n",
    "    \"@trending_now\",\n",
    "    \"https://snapchat.com\",\n",
    "    \"http://whatsapp.com\",\n",
    "    \"@messenger_app\",\n",
    "    \"https://tiktok.com\",\n",
    "    \"http://quora.com\",\n",
    "    \"@ask_me_anything\",\n",
    "    \"https://twitch.tv\",\n",
    "    \"http://vimeo.com\",\n",
    "    \"@video_stream\",\n",
    "    \"https://flickr.com\",\n",
    "    \"http://tumblr.com\",\n",
    "    \"@blogger_handle\",\n",
    "    \"https://wordpress.com\",\n",
    "    \"http://medium.com\",\n",
    "    \"@writing_hub\",\n",
    "    \"https://producthunt.com\",\n",
    "    \"http://angel.co\",\n",
    "    \"@startup_founder\",\n",
    "    \"https://crunchbase.com\",\n",
    "    \"http://gizmodo.com\",\n",
    "    \"@tech_news\",\n",
    "    \"https://mashable.com\",\n",
    "    \"http://engadget.com\",\n",
    "    \"@gadget_guru\",\n",
    "    \"https://techcrunch.com\",\n",
    "    \"http://theverge.com\",\n",
    "    \"@tech_updates\",\n",
    "    \"https://wired.com\",\n",
    "    \"http://cnet.com\",\n",
    "    \"@tech_reviews\",\n",
    "    \"https://recode.net\",\n",
    "    \"http://vox.com\",\n",
    "    \"@media_outlet\",\n",
    "    \"https://buzzfeed.com\",\n",
    "    \"http://huffpost.com\",\n",
    "    \"@news_aggregator\",\n",
    "    \"https://forbes.com\",\n",
    "    \"http://bloomberg.com\",\n",
    "    \"@finance_news\",\n",
    "    \"https://economist.com\",\n",
    "    \"http://wsj.com\",\n",
    "    \"@market_updates\",\n",
    "    \"https://fortune.com\",\n",
    "    \"http://marketwatch.com\",\n",
    "    \"@stock_tips\",\n",
    "    \"https://seekingalpha.com\",\n",
    "    \"http://investopedia.com\",\n",
    "    \"@financial_guru\",\n",
    "    \"https://cnbc.com\",\n",
    "    \"http://reuters.com\",\n",
    "    \"@newswire\",\n",
    "    \"https://apnews.com\",\n",
    "    \"http://npr.org\",\n",
    "    \"@public_radio\",\n",
    "    \"https://bbc.co.uk\",\n",
    "    \"http://aljazeera.com\",\n",
    "    \"@global_news\",\n",
    "    \"https://dw.com\",\n",
    "    \"http://france24.com\",\n",
    "    \"@world_news\",\n",
    "    \"https://rt.com\",\n",
    "    \"http://sputniknews.com\",\n",
    "    \"@news_russia\",\n",
    "    \"https://japantimes.co.jp\",\n",
    "    \"http://chinadaily.com.cn\",\n",
    "    \"@news_asia\"\n",
    "]\n",
    "\n",
    "def create_random_elements(elements, repetitions=5):\n",
    "    return \" \".join(random.choices(elements, k=repetitions))\n",
    "\n",
    "def insert_random_elements_randomly(sentence, elements, repetitions=5):\n",
    "    if not isinstance(sentence, str):\n",
    "        return sentence\n",
    "    random_elements = create_random_elements(elements, repetitions)\n",
    "    words = sentence.split()\n",
    "    random_elements_words = random_elements.split()\n",
    "    \n",
    "    if len(random_elements_words) >= len(words):\n",
    "        # Append random elements at the end if not enough words to insert randomly\n",
    "        return f\"{sentence} {random_elements}\"\n",
    "    \n",
    "    # Determine random insertion points\n",
    "    insert_points = sorted(random.sample(range(len(words) + 1), len(random_elements_words)))\n",
    "\n",
    "    # Insert random elements at the determined points\n",
    "    perturbed_sentence = []\n",
    "    random_elements_index = 0\n",
    "    for i in range(len(words) + len(random_elements_words)):\n",
    "        if i in insert_points and random_elements_index < len(random_elements_words):\n",
    "            perturbed_sentence.append(random_elements_words[random_elements_index])\n",
    "            random_elements_index += 1\n",
    "        else:\n",
    "            if words:\n",
    "                perturbed_sentence.append(words.pop(0))\n",
    "    \n",
    "    return ' '.join(perturbed_sentence)\n",
    "\n",
    "csv_file_path = ' '  # Replace with your actual file path\n",
    "df = pd.read_csv(csv_file_path)\n",
    "\n",
    "# Apply typos to the text column\n",
    "text_column = 'context'\n",
    "df['context_checklist'] = df[text_column].apply(lambda x: insert_random_elements_randomly(x, random_urls_handles, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdca5631-dc08-4a08-a2a3-7ae7a1e7f921",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# MMLU for CheckList\n",
    "\n",
    "directory = 'MMLU'\n",
    "\n",
    "# Function to process a single CSV file\n",
    "def process_csv(file_path):\n",
    "    df = pd.read_csv(file_path, header=None, names=['Question', 'Option A', 'Option B', 'Option C', 'Option D', 'Answer'])\n",
    "    df['Question_checklist'] = df['Question'].apply(lambda x: insert_random_elements_randomly(x, random_urls_handles, 5))\n",
    "    \n",
    "    # Generate the output file path\n",
    "    base_name = os.path.basename(file_path)\n",
    "    output_file_name = base_name.replace('_test.csv', '_checklist.csv')\n",
    "    output_file_path = os.path.join('MMLU_checklist', output_file_name)\n",
    "    os.makedirs(os.path.dirname(output_file_path), exist_ok=True)\n",
    "    df.to_csv(output_file_path, index=False)\n",
    "    print(f\"Processed {file_path} and saved results to {output_file_path}\")\n",
    "\n",
    "# List all CSV files in the directory\n",
    "#all_files = os.listdir(directory)\n",
    "#csv_files = [file for file in all_files if file.endswith('_test.csv')]\n",
    "csv_files = ['moral_scenarios_test.csv',\n",
    " 'us_foreign_policy_test.csv',\n",
    " 'public_relations_test.csv',\n",
    " 'global_facts_test.csv',\n",
    " 'electrical_engineering_test.csv',\n",
    " 'astronomy_test.csv',\n",
    " 'business_ethics_test.csv',\n",
    " 'jurisprudence_test.csv',\n",
    " 'high_school_chemistry_test.csv',\n",
    " 'college_physics_test.csv',\n",
    " 'professional_psychology_test.csv',\n",
    " 'marketing_test.csv',\n",
    " 'management_test.csv',\n",
    " 'virology_test.csv',\n",
    " 'international_law_test.csv',\n",
    " 'high_school_macroeconomics_test.csv',\n",
    " 'prehistory_test.csv',\n",
    " 'abstract_algebra_test.csv',\n",
    " 'high_school_physics_test.csv',\n",
    " 'formal_logic_test.csv',\n",
    " 'college_medicine_test.csv',\n",
    " 'high_school_us_history_test.csv',\n",
    " 'moral_disputes_test.csv',\n",
    " 'high_school_european_history_test.csv',\n",
    " 'clinical_knowledge_test.csv',\n",
    " 'world_religions_test.csv',\n",
    " 'high_school_microeconomics_test.csv',\n",
    " 'professional_law_test.csv',\n",
    " 'human_aging_test.csv',\n",
    " 'medical_genetics_test.csv',\n",
    " 'high_school_geography_test.csv',\n",
    " 'high_school_government_and_politics_test.csv',\n",
    " 'anatomy_test.csv',\n",
    " 'sociology_test.csv',\n",
    " 'logical_fallacies_test.csv',\n",
    " 'high_school_computer_science_test.csv',\n",
    " 'miscellaneous_test.csv',\n",
    " 'high_school_world_history_test.csv',\n",
    " 'professional_medicine_test.csv',\n",
    " 'high_school_biology_test.csv',\n",
    " 'high_school_statistics_test.csv',\n",
    " 'college_chemistry_test.csv',\n",
    " 'nutrition_test.csv',\n",
    " 'econometrics_test.csv',\n",
    " 'human_sexuality_test.csv',\n",
    " 'security_studies_test.csv',\n",
    " 'philosophy_test.csv',\n",
    " 'elementary_mathematics_test.csv',\n",
    " 'college_biology_test.csv',\n",
    " 'college_computer_science_test.csv',\n",
    " 'machine_learning_test.csv',\n",
    " 'professional_accounting_test.csv',\n",
    " 'college_mathematics_test.csv',\n",
    " 'high_school_mathematics_test.csv',\n",
    " 'high_school_psychology_test.csv',\n",
    " 'conceptual_physics_test.csv',\n",
    " 'computer_security_test.csv']\n",
    "# Iterate over each CSV file and process it\n",
    "for file_name in csv_files:\n",
    "    file_path = os.path.join(directory, file_name)\n",
    "    process_csv(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98b0ded6-907e-4fc3-9667-fb58472fc2c9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "b30e86a0-7a47-41cd-8ca1-4a8d22e2800f",
   "metadata": {},
   "source": [
    "# Compound Variations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "893721e3-556e-47d2-b636-3b82d07baae9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# general datasets for compound variations (except NumerSense and MMLU)\n",
    "import nltk\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('punkt')\n",
    "\n",
    "def complex_sentence_variation(sentence):\n",
    "    if not isinstance(sentence, str):\n",
    "        return sentence  \n",
    "    words = nltk.word_tokenize(sentence)\n",
    "    pos_tags = nltk.pos_tag(words)\n",
    "\n",
    "    new_sentence = sentence\n",
    "\n",
    "    # Expanded lists of subordinating conjunctions and quantifiers\n",
    "    conjunctions = [\n",
    "        \"although\", \"because\", \"since\", \"when\", \"while\", \"after\", \"before\", \n",
    "        \"if\", \"even though\", \"unless\", \"whereas\", \"though\", \"provided that\", \n",
    "        \"as long as\", \"in case\", \"until\", \"as soon as\", \"now that\", \"once\"\n",
    "    ]\n",
    "    quantifiers = [\n",
    "        \"some\", \"many\", \"several\", \"few\", \"a lot of\", \"a couple of\", \n",
    "        \"numerous\", \"plenty of\", \"various\", \"a handful of\", \"an abundance of\", \n",
    "        \"a number of\", \"much\", \"more\", \"less\", \"a majority of\", \"a minority of\", \n",
    "        \"sufficient\", \"all\", \"no\"\n",
    "    ]\n",
    "\n",
    "    # Add subordinating conjunctions to create complex sentences\n",
    "    chosen_conj = random.choice(conjunctions)\n",
    "    if chosen_conj not in new_sentence:\n",
    "        new_sentence = chosen_conj.capitalize() + \" \" + new_sentence\n",
    "\n",
    "    # Add quantifiers without touching numbers\n",
    "    for i, (word, pos) in enumerate(pos_tags):\n",
    "        if word.isdigit():\n",
    "            continue\n",
    "        if pos in ['NN', 'NNS'] and not word.isdigit():  # Singular and plural nouns\n",
    "            new_sentence = new_sentence.replace(word, random.choice(quantifiers) + \" \" + word, 1)\n",
    "            break\n",
    "\n",
    "    # Modify Punctuation\n",
    "    words = nltk.word_tokenize(new_sentence)\n",
    "    if len(words) > 2:\n",
    "        insert_pos = random.randint(1, len(words) - 2)\n",
    "        words.insert(insert_pos, ',')\n",
    "        new_sentence = \" \".join(words)\n",
    "\n",
    "    return new_sentence\n",
    "\n",
    "csv_file_path = ' '  # Replace with your actual file path\n",
    "df = pd.read_csv(csv_file_path)\n",
    "\n",
    "# Apply typos to the text column\n",
    "text_column = 'context'\n",
    "df['context_comp'] = df[text_column].apply(lambda x: complex_sentence_variation(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1f24d24-1043-4eb9-a1a9-898a3f37f1da",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# NumerSense for compound variations (do not move the <mask> in the text)\n",
    "def complex_sentence_variation(sentence):\n",
    "    if not isinstance(sentence, str):\n",
    "        return sentence\n",
    "\n",
    "    # Split the sentence around <mask> tokens and process each segment separately\n",
    "    segments = sentence.split('<mask>')\n",
    "    new_segments = []\n",
    "\n",
    "    for i, segment in enumerate(segments):\n",
    "        if i > 0:  # Re-add the <mask> token at the beginning of each segment except the first\n",
    "            new_segments.append('<mask>')\n",
    "\n",
    "        words = nltk.word_tokenize(segment)\n",
    "        pos_tags = nltk.pos_tag(words)\n",
    "\n",
    "        # Expanded lists of subordinating conjunctions and quantifiers\n",
    "        conjunctions = [\n",
    "            \"although\", \"because\", \"since\", \"when\", \"while\", \"after\", \"before\", \n",
    "            \"if\", \"even though\", \"unless\", \"whereas\", \"though\", \"provided that\", \n",
    "            \"as long as\", \"in case\", \"until\", \"as soon as\", \"now that\", \"once\"\n",
    "        ]\n",
    "        quantifiers = [\n",
    "            \"some\", \"many\", \"several\", \"few\", \"a lot of\", \"a couple of\", \n",
    "            \"numerous\", \"plenty of\", \"various\", \"a handful of\", \"an abundance of\", \n",
    "            \"a number of\", \"much\", \"more\", \"less\", \"a majority of\", \"a minority of\", \n",
    "            \"sufficient\", \"all\", \"no\"\n",
    "        ]\n",
    "\n",
    "        # Add subordinating conjunctions to create complex sentences\n",
    "        if segment.strip():\n",
    "            chosen_conj = random.choice(conjunctions)\n",
    "            segment = chosen_conj.capitalize() + \" \" + segment\n",
    "\n",
    "            # Add quantifiers without touching numbers\n",
    "            for word, pos in pos_tags:\n",
    "                if word.isdigit():\n",
    "                    continue\n",
    "                if pos in ['NN', 'NNS'] and not word.isdigit():  # Singular and plural nouns\n",
    "                    segment = segment.replace(word, random.choice(quantifiers) + \" \" + word, 1)\n",
    "                    break\n",
    "\n",
    "            # Modify Punctuation\n",
    "            words = nltk.word_tokenize(segment)\n",
    "            if len(words) > 2:\n",
    "                insert_pos = random.randint(1, len(words) - 2)\n",
    "                words.insert(insert_pos, ',')\n",
    "                segment = \" \".join(words)\n",
    "\n",
    "        new_segments.append(segment)\n",
    "\n",
    "    new_sentence = ''.join(new_segments)\n",
    "    return new_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e123efee-a94a-4540-bdd5-c0181eaee967",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# MMLU for compound variations\n",
    "directory = 'MMLU'\n",
    "\n",
    "# Function to process a single CSV file\n",
    "def process_csv(file_path):\n",
    "    df = pd.read_csv(file_path, header=None, names=['Question', 'Option A', 'Option B', 'Option C', 'Option D', 'Answer'])\n",
    "    df['Question_comp'] = df['Question'].apply(lambda x: complex_sentence_variation(x))\n",
    "    \n",
    "    # Generate the output file path\n",
    "    base_name = os.path.basename(file_path)\n",
    "    output_file_name = base_name.replace('_test.csv', '_comp.csv')\n",
    "    output_file_path = os.path.join('MMLU_comp', output_file_name)\n",
    "    os.makedirs(os.path.dirname(output_file_path), exist_ok=True)\n",
    "    df.to_csv(output_file_path, index=False)\n",
    "    print(f\"Processed {file_path} and saved results to {output_file_path}\")\n",
    "\n",
    "# List all CSV files in the directory\n",
    "#all_files = os.listdir(directory)\n",
    "#csv_files = [file for file in all_files if file.endswith('_test.csv')]\n",
    "csv_files = ['moral_scenarios_test.csv',\n",
    " 'us_foreign_policy_test.csv',\n",
    " 'public_relations_test.csv',\n",
    " 'global_facts_test.csv',\n",
    " 'electrical_engineering_test.csv',\n",
    " 'astronomy_test.csv',\n",
    " 'business_ethics_test.csv',\n",
    " 'jurisprudence_test.csv',\n",
    " 'high_school_chemistry_test.csv',\n",
    " 'college_physics_test.csv',\n",
    " 'professional_psychology_test.csv',\n",
    " 'marketing_test.csv',\n",
    " 'management_test.csv',\n",
    " 'virology_test.csv',\n",
    " 'international_law_test.csv',\n",
    " 'high_school_macroeconomics_test.csv',\n",
    " 'prehistory_test.csv',\n",
    " 'abstract_algebra_test.csv',\n",
    " 'high_school_physics_test.csv',\n",
    " 'formal_logic_test.csv',\n",
    " 'college_medicine_test.csv',\n",
    " 'high_school_us_history_test.csv',\n",
    " 'moral_disputes_test.csv',\n",
    " 'high_school_european_history_test.csv',\n",
    " 'clinical_knowledge_test.csv',\n",
    " 'world_religions_test.csv',\n",
    " 'high_school_microeconomics_test.csv',\n",
    " 'professional_law_test.csv',\n",
    " 'human_aging_test.csv',\n",
    " 'medical_genetics_test.csv',\n",
    " 'high_school_geography_test.csv',\n",
    " 'high_school_government_and_politics_test.csv',\n",
    " 'anatomy_test.csv',\n",
    " 'sociology_test.csv',\n",
    " 'logical_fallacies_test.csv',\n",
    " 'high_school_computer_science_test.csv',\n",
    " 'miscellaneous_test.csv',\n",
    " 'high_school_world_history_test.csv',\n",
    " 'professional_medicine_test.csv',\n",
    " 'high_school_biology_test.csv',\n",
    " 'high_school_statistics_test.csv',\n",
    " 'college_chemistry_test.csv',\n",
    " 'nutrition_test.csv',\n",
    " 'econometrics_test.csv',\n",
    " 'human_sexuality_test.csv',\n",
    " 'security_studies_test.csv',\n",
    " 'philosophy_test.csv',\n",
    " 'elementary_mathematics_test.csv',\n",
    " 'college_biology_test.csv',\n",
    " 'college_computer_science_test.csv',\n",
    " 'machine_learning_test.csv',\n",
    " 'professional_accounting_test.csv',\n",
    " 'college_mathematics_test.csv',\n",
    " 'high_school_mathematics_test.csv',\n",
    " 'high_school_psychology_test.csv',\n",
    " 'conceptual_physics_test.csv',\n",
    " 'computer_security_test.csv']\n",
    "# Iterate over each CSV file and process it\n",
    "for file_name in csv_files:\n",
    "    file_path = os.path.join(directory, file_name)\n",
    "    process_csv(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38db5322-1293-49a1-9a9f-222e5b8c7998",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}