det.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "from os import chdir\n",
    "chdir(\"..\")\n",
    "# np.random.seed(91)\n",
    "np.set_printoptions(precision = 2, suppress = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 1, 2],\n",
       "       [1, 0, 0],\n",
       "       [2, 0, 1],\n",
       "       [2, 0, 2],\n",
       "       [2, 2, 2]])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = np.random.randint(0, 3, (5, 3))\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import pairwise_kernels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  6.,   1.,   4.,   6.,   8.],\n",
       "       [  1.,   1.,   2.,   2.,   2.],\n",
       "       [  4.,   2.,   5.,   6.,   6.],\n",
       "       [  6.,   2.,   6.,   8.,   8.],\n",
       "       [  8.,   2.,   6.,   8.,  12.]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K = pairwise_kernels(X, metric = 'linear')\n",
    "K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "S = [1, 3] ## selected\n",
    "C = [0, 2, 4] ## remaining candidates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 1.,  2.],\n",
       "        [ 2.,  8.]]), array([[  6.,   4.,   8.],\n",
       "        [  4.,   5.,   6.],\n",
       "        [  8.,   6.,  12.]]), array([[ 1.,  2.,  2.],\n",
       "        [ 6.,  6.,  8.]]))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Kss = K[S, :][:, S]\n",
    "Kcc = K[C, :][:, C]\n",
    "Ksc = K[S, :][:, C]\n",
    "Kss, Kcc, Ksc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2.  , -0.5 ],\n",
       "       [-0.5 ,  0.25]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K1 = np.linalg.inv(Kss)\n",
    "K1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## like Been Kim's impl, i.e. $\\mathbf{c} = K_{ss}^{-1} K_{sc} \\odot K_{sc}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5.,  5.,  8.])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c0 = (K1.dot(Ksc) * Ksc).sum(axis = 0) ## like been kim's impl\n",
    "c0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## $\\textbf{c} = diag(K_{sc}^T K_{ss}^{-1} K_{sc} )$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5.,  5.,  8.])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c1 = Ksc.T.dot(K1).dot(Ksc).diagonal()\n",
    "c1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## det = $diag(K_{cc}) - \\textbf c$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.,  0.,  4.])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logdet1 = Kcc.diagonal() - c1\n",
    "logdet1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## det = $ det( K_{S+c,S+c}) $"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4.0, 0.0, 15.999999999999998]\n"
     ]
    }
   ],
   "source": [
    "logdet2 = []\n",
    "for c in C:\n",
    "    newS = np.append(S, c)\n",
    "    Kss_new = K[newS, :][:, newS]\n",
    "    logdet2.append(np.linalg.det(Kss_new))\n",
    "print(logdet2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.linalg.det(Kss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "seems like:\n",
    "$$ det(K_{n+1}) = det(K_n) * (k_{n,n} - \\textbf k_{n+1, :n} K_n^{-1} \\textbf k_{:n, n+1}) $$\n",
    "\n",
    "but where does this equation come from ??"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## repeating multiple times over multiple kernel functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "testing for kernel linear\n",
      "X...X..XXXX...X...X.X..XX\n",
      "testing for kernel rbf\n",
      ".........................\n",
      "testing for kernel cosine\n",
      ".........................\n"
     ]
    }
   ],
   "source": [
    "N = 25 ## number of times to run experiment\n",
    "n, d = 500, 50 ## data dimensions\n",
    "\n",
    "all_items = np.array(range(n))\n",
    "\n",
    "for kernel in (\"linear\", \"rbf\", \"cosine\"):\n",
    "    print(\"testing for kernel {}\".format(kernel))\n",
    "    i = 0\n",
    "    while i < N:\n",
    "        X = np.random.randint(1, 10, (n, d)).astype(np.float64)\n",
    "        K = pairwise_kernels(X, metric = kernel)\n",
    "        S = np.random.choice(all_items, np.random.randint(1, n-1)) ## randomly choose selected indices\n",
    "        C = np.setdiff1d(all_items, S)\n",
    "        Kss = K[S, :][:, S]\n",
    "        Kcc = K[C, :][:, C]\n",
    "        Ksc = K[S, :][:, C]\n",
    "        det_Kss = np.linalg.det(Kss)\n",
    "        if np.isclose(det_Kss, 0):\n",
    "            continue ## singluar matrix\n",
    "        \n",
    "        K1 = np.linalg.inv(Kss)\n",
    "        c1 = (K1.dot(Ksc) * Ksc).sum(axis = 0) ## like been kim's impl\n",
    "#         c2 = Ksc.T.dot(K1).dot(Ksc).diagonal() ## as derieved by Dr. @akmenon\n",
    "#         assert np.allclose(c1, c2), \"{} =!= {}\".format(c1, c2)\n",
    "        \n",
    "        det1 = det_Kss * (Kcc.diagonal() - c1)\n",
    "        det2 = []\n",
    "        for c in C:\n",
    "            newS = np.append(S, c)\n",
    "            Kss_new = K[newS, :][:, newS]\n",
    "            det2.append(np.linalg.det(Kss_new))\n",
    "        if np.allclose(det1, det2): print(\".\", end = \"\")\n",
    "        else: print(\"X\", end = \"\")\n",
    "        i += 1\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}