examples/03_classification_with_proba_detailed_workflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification (with probabilities) + Detailed workflow\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "from glob import glob\n",
    "import re \n",
    "import numpy as np\n",
    "np.random.seed(0) # ensure reproducibility\n",
    "np.set_printoptions(suppress = True)\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.metrics import log_loss\n",
    "# Models\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "# NN\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "# Stacking\n",
    "from vecstack import stacking"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train shape: (400, 5)\n",
      "Test shape:  (100, 5)\n"
     ]
    }
   ],
   "source": [
    "n_classes = 3\n",
    "\n",
    "# Create data: 500 example, 5 feature, 3 classes\n",
    "X, y = make_classification(n_samples=500, n_features=5, \n",
    "                           n_informative=3, n_redundant=1, \n",
    "                           n_classes=n_classes, flip_y=0, \n",
    "                           random_state=0)\n",
    "\n",
    "# Make train/test split\n",
    "# As usual in machine learning task we have X_train, y_train, and X_test\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)\n",
    "\n",
    "print('Train shape:', X_train.shape)\n",
    "print('Test shape: ', X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialize 1st level models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_keras_model_1():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(64, \n",
    "                    input_dim=X_train.shape[1], \n",
    "                    kernel_initializer='normal', \n",
    "                    activation='relu'))\n",
    "    model.add(Dense(n_classes, \n",
    "                    kernel_initializer='normal', \n",
    "                    activation='softmax'))\n",
    "    model.compile(optimizer='rmsprop', \n",
    "                  loss='categorical_crossentropy', \n",
    "                  metrics=['categorical_accuracy'])\n",
    "    return model\n",
    "\n",
    "# Caution! All models and parameter values are just \n",
    "# demonstrational and shouldn't be considered as recommended.\n",
    "models_1 = [ \n",
    "    GaussianNB(),\n",
    "    \n",
    "    LogisticRegression(random_state=0),\n",
    "    \n",
    "    ExtraTreesClassifier(random_state=0, n_jobs=-1, \n",
    "                         n_estimators=100, max_depth=3),\n",
    "                         \n",
    "    RandomForestClassifier(random_state=0, n_jobs=-1, \n",
    "                           n_estimators=100, max_depth=3),\n",
    "        \n",
    "    XGBClassifier(random_state=0, n_jobs=-1, learning_rate=0.1, \n",
    "                  n_estimators=100, max_depth=3),\n",
    "                  \n",
    "    LGBMClassifier(random_state=0, n_jobs=-1, learning_rate=0.1, \n",
    "                   n_estimators=100, max_depth=3),\n",
    "                  \n",
    "    KerasClassifier(build_fn=build_keras_model_1, epochs=2, \n",
    "                    batch_size=32, verbose=0)\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "task:       [classification]\n",
      "n_classes:  [3]\n",
      "metric:     [log_loss]\n",
      "mode:       [oof_pred]\n",
      "n_models:   [7]\n",
      "\n",
      "model 0:    [GaussianNB]\n",
      "    fold 0: [0.65551778]\n",
      "    fold 1: [0.42335961]\n",
      "    fold 2: [0.38132309]\n",
      "    fold 3: [0.57180128]\n",
      "    fold 4: [0.30426116]\n",
      "    ----\n",
      "    MEAN:   [0.46725258] + [0.12825825]\n",
      "    FULL:   [0.46842847]\n",
      "\n",
      "    Fitting on full train set...\n",
      "\n",
      "model 1:    [LogisticRegression]\n",
      "    fold 0: [0.59821304]\n",
      "    fold 1: [0.54202039]\n",
      "    fold 2: [0.55194968]\n",
      "    fold 3: [0.46887313]\n",
      "    fold 4: [0.44995007]\n",
      "    ----\n",
      "    MEAN:   [0.52220126] + [0.05499033]\n",
      "    FULL:   [0.52280210]\n",
      "\n",
      "    Fitting on full train set...\n",
      "\n",
      "model 2:    [ExtraTreesClassifier]\n",
      "    fold 0: [0.79961086]\n",
      "    fold 1: [0.75093790]\n",
      "    fold 2: [0.77930597]\n",
      "    fold 3: [0.76984042]\n",
      "    fold 4: [0.75288684]\n",
      "    ----\n",
      "    MEAN:   [0.77051640] + [0.01799067]\n",
      "    FULL:   [0.77062834]\n",
      "\n",
      "    Fitting on full train set...\n",
      "\n",
      "model 3:    [RandomForestClassifier]\n",
      "    fold 0: [0.61575788]\n",
      "    fold 1: [0.40598536]\n",
      "    fold 2: [0.37631065]\n",
      "    fold 3: [0.48927717]\n",
      "    fold 4: [0.35383760]\n",
      "    ----\n",
      "    MEAN:   [0.44823373] + [0.09551690]\n",
      "    FULL:   [0.44901890]\n",
      "\n",
      "    Fitting on full train set...\n",
      "\n",
      "model 4:    [XGBClassifier]\n",
      "    fold 0: [0.79844773]\n",
      "    fold 1: [0.29219564]\n",
      "    fold 2: [0.32080725]\n",
      "    fold 3: [0.47567222]\n",
      "    fold 4: [0.28695253]\n",
      "    ----\n",
      "    MEAN:   [0.43481507] + [0.19447336]\n",
      "    FULL:   [0.43610692]\n",
      "\n",
      "    Fitting on full train set...\n",
      "\n",
      "model 5:    [LGBMClassifier]\n",
      "    fold 0: [0.72121715]\n",
      "    fold 1: [0.28661436]\n",
      "    fold 2: [0.32019533]\n",
      "    fold 3: [0.44805274]\n",
      "    fold 4: [0.29008710]\n",
      "    ----\n",
      "    MEAN:   [0.41323334] + [0.16487562]\n",
      "    FULL:   [0.41430248]\n",
      "\n",
      "    Fitting on full train set...\n",
      "\n",
      "model 6:    [KerasClassifier]\n",
      "    fold 0: [0.95679030]\n",
      "    fold 1: [0.97577886]\n",
      "    fold 2: [0.99147471]\n",
      "    fold 3: [0.97480903]\n",
      "    fold 4: [0.96731000]\n",
      "    ----\n",
      "    MEAN:   [0.97323258] + [0.01137690]\n",
      "    FULL:   [0.97322745]\n",
      "\n",
      "    Fitting on full train set...\n",
      "\n",
      "Result was saved to [./[2018.02.01].[15.41.41].305268.0eadc0.npy]\n"
     ]
    }
   ],
   "source": [
    "S_train_1, S_test_1 = stacking(models_1,                   # list of models\n",
    "                               X_train, y_train, X_test,   # data\n",
    "                               regression=False,           # classification task (if you need \n",
    "                                                           #     regression - set to True)\n",
    "                               mode='oof_pred',            # mode: oof for train set, fit on full \n",
    "                                                           #     train and predict test set once\n",
    "                               needs_proba=True,           # predict probabilities (if you need \n",
    "                                                           #     class labels - set to False) \n",
    "                               save_dir='.',               # save result and log in current dir \n",
    "                                                           #     (to disable saving - set to None)\n",
    "                               metric=log_loss,            # metric: callable\n",
    "                               n_folds=5,                  # number of folds\n",
    "                               stratified=True,            # stratified split for folds\n",
    "                               shuffle=True,               # shuffle the data\n",
    "                               random_state=0,             # ensure reproducibility\n",
    "                               verbose=2)                  # print all info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Look at the result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We have 3 classes and 7 models so in resulting arrays we expect to see 21 columns.\n",
      "S_train_1 shape: (400, 21)\n",
      "S_test_1 shape:  (100, 21)\n"
     ]
    }
   ],
   "source": [
    "print('We have %d classes and %d models so in resulting arrays \\\n",
    "we expect to see %d columns.' % (n_classes, len(models_1), n_classes * len(models_1)))\n",
    "print('S_train_1 shape:', S_train_1.shape)\n",
    "print('S_test_1 shape: ', S_test_1.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.00118767,  0.02222581,  0.97658652,  0.06963771,  0.05618856,\n",
       "         0.87417373,  0.21796766,  0.21491663,  0.56711571,  0.03741131,\n",
       "         0.08982228,  0.87276641,  0.00182469,  0.00537052,  0.99280483,\n",
       "         0.00040829,  0.00281319,  0.99677852,  0.3023589 ,  0.26565766,\n",
       "         0.43198347],\n",
       "       [ 0.96030684,  0.03969316,  0.        ,  0.75245808,  0.24720408,\n",
       "         0.00033784,  0.5615216 ,  0.26871071,  0.16976769,  0.85696824,\n",
       "         0.12811857,  0.01491319,  0.9877857 ,  0.01111581,  0.00109853,\n",
       "         0.99732125,  0.00258249,  0.00009626,  0.38591456,  0.31510866,\n",
       "         0.29897675]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S_train_1[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.38824186,  0.37434678,  0.23741136,  0.35886342,  0.38701687,\n",
       "         0.2541197 ,  0.31662764,  0.29004533,  0.39332704,  0.27655712,\n",
       "         0.55408115,  0.16936173,  0.58901626,  0.3923738 ,  0.01860998,\n",
       "         0.60639131,  0.3588515 ,  0.03475718,  0.33691144,  0.31800038,\n",
       "         0.34508815],\n",
       "       [ 0.32313599,  0.67239959,  0.00446442,  0.32348396,  0.54466285,\n",
       "         0.13185319,  0.31999925,  0.36345201,  0.31654874,  0.10054021,\n",
       "         0.81354061,  0.08591918,  0.02955116,  0.95850134,  0.01194747,\n",
       "         0.03609523,  0.90174785,  0.06215692,  0.3260029 ,  0.37157273,\n",
       "         0.3024244 ]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S_test_1[:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Our arrays and log were saved in current dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Arrays:\n",
      "[2018.02.01].[15.41.41].305268.0eadc0.npy\n",
      "\n",
      "Logs:\n",
      "[2018.02.01].[15.41.41].305268.0eadc0.log.txt\n"
     ]
    }
   ],
   "source": [
    "names = sorted(glob('*.npy'))\n",
    "npy_1_name = names[0] # for later use\n",
    "\n",
    "print('Arrays:')\n",
    "for name in names:\n",
    "    print(name)\n",
    "\n",
    "names = sorted(glob('*.log.txt'))\n",
    "log_1_name = names[0] # for later use\n",
    "\n",
    "print('\\nLogs:')\n",
    "for name in names:\n",
    "    print(name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialize some other 1st level model(s)\n",
    "\n",
    "As we continue to work on the problem we create many other models.  \n",
    "Let's say we want to try more powerful neural network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_keras_model_2():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(256, \n",
    "                    input_dim=X_train.shape[1], \n",
    "                    kernel_initializer='normal', \n",
    "                    activation='relu'))\n",
    "    model.add(Dense(64, \n",
    "                    kernel_initializer='normal', \n",
    "                    activation='relu'))\n",
    "    model.add(Dense(n_classes, \n",
    "                    kernel_initializer='normal', \n",
    "                    activation='softmax'))\n",
    "    model.compile(optimizer='rmsprop', \n",
    "                  loss='categorical_crossentropy', \n",
    "                  metrics=['categorical_accuracy'])\n",
    "    return model\n",
    "\n",
    "# Caution! All models and parameter values are just \n",
    "# demonstrational and shouldn't be considered as recommended.\n",
    "models_2 = [        \n",
    "    KerasClassifier(build_fn=build_keras_model_2, epochs=5, \n",
    "                    batch_size=32, verbose=0)\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform stacking again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "task:       [classification]\n",
      "n_classes:  [3]\n",
      "metric:     [log_loss]\n",
      "mode:       [oof_pred]\n",
      "n_models:   [1]\n",
      "\n",
      "model 0:    [KerasClassifier]\n",
      "    fold 0: [0.54741578]\n",
      "    fold 1: [0.42787166]\n",
      "    fold 2: [0.40649939]\n",
      "    fold 3: [0.45298407]\n",
      "    fold 4: [0.39133918]\n",
      "    ----\n",
      "    MEAN:   [0.44522202] + [0.05515007]\n",
      "    FULL:   [0.44570354]\n",
      "\n",
      "    Fitting on full train set...\n",
      "\n",
      "Result was saved to [./[2018.02.01].[15.42.30].250441.2c4c1b.npy]\n"
     ]
    }
   ],
   "source": [
    "S_train_2, S_test_2 = stacking(models_2,                   # list of models\n",
    "                               X_train, y_train, X_test,   # data\n",
    "                               regression=False,           # classification task (if you need \n",
    "                                                           #     regression - set to True)\n",
    "                               mode='oof_pred',            # mode: oof for train set, fit on full \n",
    "                                                           #     train and predict test set once\n",
    "                               needs_proba=True,           # predict probabilities (if you need \n",
    "                                                           #     class labels - set to False) \n",
    "                               save_dir='.',               # save result and log in current dir \n",
    "                                                           #     (to disable saving - set to None)\n",
    "                               metric=log_loss,            # metric: callable\n",
    "                               n_folds=5,                  # number of folds\n",
    "                               stratified=True,            # stratified split for folds\n",
    "                               shuffle=True,               # shuffle the data\n",
    "                               random_state=0,             # ensure reproducibility\n",
    "                               verbose=2)                  # print all info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# New arrays and log were saved too"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Arrays:\n",
      "[2018.02.01].[15.41.41].305268.0eadc0.npy\n",
      "[2018.02.01].[15.42.30].250441.2c4c1b.npy\n",
      "\n",
      "Logs:\n",
      "[2018.02.01].[15.41.41].305268.0eadc0.log.txt\n",
      "[2018.02.01].[15.42.30].250441.2c4c1b.log.txt\n"
     ]
    }
   ],
   "source": [
    "names = sorted(glob('*.npy'))\n",
    "\n",
    "print('Arrays:')\n",
    "for name in names:\n",
    "    print(name)\n",
    "    \n",
    "names = sorted(glob('*.log.txt'))\n",
    "\n",
    "print('\\nLogs:')\n",
    "for name in names:\n",
    "    print(name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Time to collect results\n",
    "\n",
    "After several (many) days of building, optimizing, and testing models we have a lot of files with saved OOF.  \n",
    "At this point we can load and use OOF of specific model or all OOF we have."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Find specific model\n",
    "\n",
    "We can open logs and find the model of interest.  \n",
    "We can do it programmatically or just open logs in editor.  \n",
    "Name of the `.log.txt` file is the same as the name of corresponding `.npy` file (except extension).  \n",
    "To find columns containing OOF of specific model we use model index from log:\n",
    "* if we predicted class labels - corresponding column index is just model index\n",
    "* if we predicted probabilities - corresponding column index is model index multiplied by number of classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Let's open this log: [2018.02.01].[15.41.41].305268.0eadc0.log.txt\n",
      "Let's look what models did we build in those session.\n",
      "\n",
      "model 0:    [GaussianNB]\n",
      "\n",
      "model 1:    [LogisticRegression]\n",
      "\n",
      "model 2:    [ExtraTreesClassifier]\n",
      "\n",
      "model 3:    [RandomForestClassifier]\n",
      "\n",
      "model 4:    [XGBClassifier]\n",
      "\n",
      "model 5:    [LGBMClassifier]\n",
      "\n",
      "model 6:    [KerasClassifier]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Let's open this log: %s\" % log_1_name)\n",
    "with open(log_1_name) as f:\n",
    "    lines = f.readlines()\n",
    "\n",
    "print(\"Let's look what models did we build in those session.\\n\")\n",
    "for line in lines:\n",
    "    if re.search(r'^model [0-9]+', line):\n",
    "        print(line)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load specific model OOF\n",
    "\n",
    "Let's say we are interested in `LGBMClassifier`.  \n",
    "We found out that it has index 5.  \n",
    "Then we load target `.npy` file and because of probabilities we need 3 columns from 15 (5 \\* 3) to 18 (5 \\* 3 + 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Let's load this .npy file: [2018.02.01].[15.41.41].305268.0eadc0.npy\n"
     ]
    }
   ],
   "source": [
    "print(\"Let's load this .npy file: %s\" % npy_1_name)\n",
    "S = np.load(npy_1_name)\n",
    "S_train_lgbm = S[0][:, 15:18]\n",
    "S_test_lgbm = S[1][:, 15:18]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.00040829,  0.00281319,  0.99677852],\n",
       "       [ 0.99732125,  0.00258249,  0.00009626],\n",
       "       [ 0.98322854,  0.01610955,  0.00066191],\n",
       "       [ 0.00107737,  0.99633895,  0.00258368],\n",
       "       [ 0.97101719,  0.02843959,  0.00054321]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S_train_lgbm[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.60639131,  0.3588515 ,  0.03475718],\n",
       "       [ 0.03609523,  0.90174785,  0.06215692],\n",
       "       [ 0.08650007,  0.89717473,  0.0163252 ],\n",
       "       [ 0.00068572,  0.98858075,  0.01073353],\n",
       "       [ 0.00122693,  0.99814513,  0.00062793]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S_test_lgbm[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute score of specific model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LGBMCLassifier log loss: 0.41430248\n"
     ]
    }
   ],
   "source": [
    "print('LGBMCLassifier log loss: %.8f' % log_loss(y_train, S_train_lgbm))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load ALL OOF\n",
    "\n",
    "***Note:*** If you load OOF from scratch, don't forget to load `y_train` from initial dataset too."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We have 3 classes and 8 models TOTAL so in resulting arrays we expect to see 24 columns.\n"
     ]
    }
   ],
   "source": [
    "print('We have %d classes and %d models TOTAL so in resulting arrays \\\n",
    "we expect to see %d columns.' % (n_classes, len(models_1) + len(models_2), \n",
    "                                 n_classes * (len(models_1) + len(models_2))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading: [2018.02.01].[15.41.41].305268.0eadc0.npy\n",
      "Loading: [2018.02.01].[15.42.30].250441.2c4c1b.npy\n",
      "\n",
      "S_train_all shape: (400, 24)\n",
      "S_test_all shape:  (100, 24)\n"
     ]
    }
   ],
   "source": [
    "# Create empty arrays\n",
    "S_train_all = np.zeros((X_train.shape[0], 0))\n",
    "S_test_all = np.zeros((X_test.shape[0], 0))\n",
    "\n",
    "# Load results\n",
    "for name in sorted(glob('*.npy')):\n",
    "    print('Loading: %s' % name)\n",
    "    S = np.load(name)\n",
    "    S_train_all = np.c_[S_train_all, S[0]]\n",
    "    S_test_all = np.c_[S_test_all, S[1]]\n",
    "    \n",
    "print('\\nS_train_all shape:', S_train_all.shape)\n",
    "print('S_test_all shape: ', S_test_all.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Apply 2nd level model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final prediction score: 0.38636334\n"
     ]
    }
   ],
   "source": [
    "# Initialize 2nd level model\n",
    "model = XGBClassifier(random_state=0, n_jobs=-1, learning_rate=0.1, \n",
    "                      n_estimators=100, max_depth=3)\n",
    "    \n",
    "# Fit 2nd level model\n",
    "model = model.fit(S_train_all, y_train)\n",
    "\n",
    "# Predict\n",
    "y_pred = model.predict_proba(S_test_all)\n",
    "\n",
    "# Final prediction score\n",
    "print('Final prediction score: %.8f' % log_loss(y_test, y_pred))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}