liberia.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(\"testdata/liberia.csv\")\n",
    "df.drop([\"quality\", \"problems\"], axis=1, inplace=True)\n",
    "df[:1000].to_csv(\"testdata/liberia_noq.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11724"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import holoclean\n",
    "from detect import *\n",
    "from repair.featurize import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Setup\n",
    "Setup holoclean session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "06:44:25 - [DEBUG] - initiating session with parameters: {'db_user': 'holocleanuser', 'db_pwd': 'abcd1234', 'db_host': 'localhost', 'db_name': 'holo', 'threads': 1, 'timeout': 180000, 'seed': 45, 'layer_sizes': [1], 'learning_rate': 0.001, 'optimizer': 'adam', 'epochs': 20, 'weight_decay': 0.01, 'momentum': 0.0, 'batch_size': 1, 'weak_label_thresh': 0.99, 'domain_thresh_1': 0.0, 'domain_thresh_2': 0.0, 'max_domain': 10000, 'cor_strength': 0.6, 'nb_cor_strength': 0.8, 'feature_norm': False, 'weight_norm': False, 'estimator_type': 'NaiveBayes', 'estimator_epochs': 10, 'estimator_batch_size': 32, 'estimator_embedding_size': 10, 'train_attrs': None, 'infer_mode': 'dk', 'verbose': True, 'bias': False, 'print_fw': True, 'debug_mode': False}\n"
     ]
    }
   ],
   "source": [
    "hc = holoclean.HoloClean(\n",
    "    db_name='holo',\n",
    "    domain_thresh_1=0.0,\n",
    "    domain_thresh_2=0.0,\n",
    "    weak_label_thresh=0.99,\n",
    "    max_domain=10000,\n",
    "    cor_strength=0.6,\n",
    "    nb_cor_strength=0.8,\n",
    "    weight_decay=0.01,\n",
    "    learning_rate=0.001,\n",
    "    threads=1,\n",
    "    batch_size=1,\n",
    "    verbose=True,\n",
    "    timeout=3 * 60000,\n",
    "    print_fw=True,\n",
    ").session"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Load training data and denial constraints\n",
    "\n",
    "Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "06:44:25 - [WARNING] - Dropping the following null column from the dataset: 'Region'\n",
      "06:44:25 - [WARNING] - Dropping the following null column from the dataset: 'Stroke test'\n",
      "06:44:25 - [ INFO] - Loaded 1000 rows with 19000 cells\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [ INFO] - DONE Loading liberia_noq.csv\n",
      "06:44:25 - [DEBUG] - Time to load dataset: 0.22 secs\n"
     ]
    }
   ],
   "source": [
    "hc.load_data(\"liberia\", \"testdata/liberia_noq.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Constrains"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "constrain_path = \"testdata/liberia_constraints.txt\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "!cat $constrain_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "hc.load_dcs(constrain_path)\n",
    "hc.ds.set_constraints(hc.get_dcs())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Detectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "06:44:25 - [DEBUG] - DONE with Error Detector: NullDetector in 0.04 secs\n",
      "06:44:25 - [ INFO] - detected 483 potentially erroneous cells\n",
      "06:44:25 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:25 - [ INFO] - DONE with error detection.\n",
      "06:44:25 - [DEBUG] - Time to detect errors: 0.08 secs\n"
     ]
    }
   ],
   "source": [
    "detectors = [NullDetector()]\n",
    "hc.detect_errors(detectors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Repair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "06:44:25 - [DEBUG] - Computing correlations...\n",
      "06:44:32 - [DEBUG] - correlations:\n",
      "attr             Unnamed: 0  Instance  Submission Date  Submitter  settlement  \\\n",
      "cond_attr                                                                       \n",
      "CMMoney            0.178756  0.178756         0.182726   0.168557    0.192597   \n",
      "Functionality      0.109320  0.109320         0.111965   0.087271    0.113627   \n",
      "Geo Code           0.988325  0.988325         0.995203   0.998560    0.997882   \n",
      "Instance           1.000000  1.000000         1.000000   1.000000    1.000000   \n",
      "Latitude           0.993098  0.993098         1.000000   1.000000    1.000000   \n",
      "Longitude          0.993098  0.993098         1.000000   1.000000    1.000000   \n",
      "Submission Date    0.992775  0.992775         1.000000   0.999674    0.999793   \n",
      "Submitter          0.585167  0.585167         0.590139   1.000000    0.613940   \n",
      "Unnamed: 0         1.000000  1.000000         1.000000   1.000000    1.000000   \n",
      "Wptype             0.026905  0.026905         0.029122   0.088595    0.040718   \n",
      "YearCons           0.354940  0.354940         0.358721   0.223316    0.356973   \n",
      "committee          0.109022  0.109022         0.111665   0.101341    0.120695   \n",
      "drinking?          0.083260  0.083260         0.085769   0.076938    0.089179   \n",
      "handpump           0.099625  0.099625         0.102220   0.126282    0.109051   \n",
      "partner            0.729834  0.729834         0.735533   0.723274    0.742956   \n",
      "photo              0.992976  0.992976         1.000000   1.000000    1.000000   \n",
      "quantity           0.173257  0.173257         0.176234   0.122849    0.177834   \n",
      "settlement         0.957531  0.957531         0.964371   0.995318    1.000000   \n",
      "\n",
      "attr               Wptype  handpump  Functionality  YearCons   partner  \\\n",
      "cond_attr                                                                \n",
      "CMMoney          0.951049  0.744687       0.521480  0.314553  0.246513   \n",
      "Functionality    0.931390  0.733954       1.000000  0.298901  0.198882   \n",
      "Geo Code         0.999436  0.996619       0.994949  0.996656  0.996934   \n",
      "Instance         1.000000  1.000000       1.000000  1.000000  1.000000   \n",
      "Latitude         1.000000  1.000000       1.000000  1.000000  1.000000   \n",
      "Longitude        1.000000  1.000000       1.000000  1.000000  1.000000   \n",
      "Submission Date  1.000000  1.000000       1.000000  0.999586  0.999762   \n",
      "Submitter        0.969971  0.831470       0.535768  0.482146  0.627726   \n",
      "Unnamed: 0       1.000000  1.000000       1.000000  1.000000  1.000000   \n",
      "Wptype           1.000000  0.761506       0.466353  0.302570  0.166037   \n",
      "YearCons         0.959124  0.784619       0.498981  1.000000  0.380487   \n",
      "committee        0.934076  0.732856       0.466171  0.291265  0.199898   \n",
      "drinking?        0.930546  0.729746       0.651284  0.290177  0.180482   \n",
      "handpump         0.963661  1.000000       0.474810  0.322976  0.221755   \n",
      "partner          0.983062  0.916437       0.698272  0.691148  1.000000   \n",
      "photo            1.000000  1.000000       1.000000  1.000000  1.000000   \n",
      "quantity         0.933570  0.732716       0.535806  0.297367  0.230158   \n",
      "settlement       0.997421  0.979733       0.937511  0.955632  0.974529   \n",
      "\n",
      "attr             drinking?  quantity  committee   CMMoney     photo  Latitude  \\\n",
      "cond_attr                                                                       \n",
      "CMMoney           0.328969  0.219604   0.000000  1.000000  0.182726  0.182596   \n",
      "Functionality     0.562279  0.217212   0.328256  0.430005  0.111897  0.112065   \n",
      "Geo Code          0.985859  0.992548   0.996176  0.998757  0.995204  0.995204   \n",
      "Instance          1.000000  1.000000   1.000000  1.000000  1.000000  1.000000   \n",
      "Latitude          1.000000  1.000000   1.000000  1.000000  1.000000  1.000000   \n",
      "Longitude         1.000000  1.000000   1.000000  1.000000  1.000000  1.000000   \n",
      "Submission Date   1.000000  1.000000   1.000000  1.000000  0.999798  0.999796   \n",
      "Submitter         0.267137  0.324643   0.466151  0.674407  0.590202  0.591021   \n",
      "Unnamed: 0        1.000000  1.000000   1.000000  1.000000  1.000000  1.000000   \n",
      "Wptype            0.189388  0.151665   0.334559  0.423152  0.029067  0.029304   \n",
      "YearCons          0.218376  0.176586   0.345997  0.475619  0.358819  0.359399   \n",
      "committee         0.202467  0.149695   1.000000  0.415637  0.111597  0.111808   \n",
      "drinking?         1.000000  0.214351   0.334826  0.422561  0.085705  0.085732   \n",
      "handpump          0.194244  0.153998   0.336673  0.429497  0.102153  0.102168   \n",
      "partner           0.501919  0.513555   0.626205  0.738250  0.735574  0.735927   \n",
      "photo             1.000000  1.000000   1.000000  1.000000  1.000000  1.000000   \n",
      "quantity          0.325599  1.000000   0.330927  0.425682  0.176156  0.176641   \n",
      "settlement        0.898126  0.929603   0.966176  0.987533  0.964377  0.964249   \n",
      "\n",
      "attr             Longitude  Geo Code  \n",
      "cond_attr                             \n",
      "CMMoney           0.182596  0.183674  \n",
      "Functionality     0.112065  0.112835  \n",
      "Geo Code          0.995204  1.000000  \n",
      "Instance          1.000000  1.000000  \n",
      "Latitude          1.000000  1.000000  \n",
      "Longitude         1.000000  1.000000  \n",
      "Submission Date   0.999796  0.999795  \n",
      "Submitter         0.591021  0.593579  \n",
      "Unnamed: 0        1.000000  1.000000  \n",
      "Wptype            0.029304  0.030612  \n",
      "YearCons          0.359399  0.360306  \n",
      "committee         0.111808  0.112987  \n",
      "drinking?         0.085732  0.086003  \n",
      "handpump          0.102168  0.102700  \n",
      "partner           0.735927  0.737075  \n",
      "photo             1.000000  1.000000  \n",
      "quantity          0.176641  0.177145  \n",
      "settlement        0.964249  0.966866  \n",
      "06:44:32 - [DEBUG] - summary of correlations:\n",
      "attr   Unnamed: 0   Instance  Submission Date  Submitter  settlement  \\\n",
      "count   18.000000  18.000000        18.000000  18.000000   18.000000   \n",
      "mean     0.575994   0.575994         0.580204   0.595110    0.586402   \n",
      "std      0.418003   0.418003         0.419176   0.439011    0.418665   \n",
      "min      0.026905   0.026905         0.029122   0.076938    0.040718   \n",
      "25%      0.125304   0.125304         0.128032   0.123707    0.134980   \n",
      "50%      0.657501   0.657501         0.662836   0.859296    0.678448   \n",
      "75%      0.992926   0.992926         1.000000   1.000000    1.000000   \n",
      "max      1.000000   1.000000         1.000000   1.000000    1.000000   \n",
      "\n",
      "attr      Wptype   handpump  Functionality   YearCons    partner  drinking?  \\\n",
      "count  18.000000  18.000000      18.000000  18.000000  18.000000  18.000000   \n",
      "mean    0.975184   0.885797       0.765632   0.680165   0.634620   0.648576   \n",
      "std     0.028479   0.122744       0.240640   0.336962   0.386167   0.361597   \n",
      "min     0.930546   0.729746       0.466171   0.290177   0.166037   0.189388   \n",
      "25%     0.953068   0.748892       0.525052   0.305566   0.223856   0.281752   \n",
      "50%     0.990242   0.948085       0.817891   0.823390   0.801128   0.730202   \n",
      "75%     1.000000   1.000000       1.000000   1.000000   1.000000   1.000000   \n",
      "max     1.000000   1.000000       1.000000   1.000000   1.000000   1.000000   \n",
      "\n",
      "attr    quantity  committee    CMMoney      photo   Latitude  Longitude  \\\n",
      "count  18.000000  18.000000  18.000000  18.000000  18.000000  18.000000   \n",
      "mean    0.613526   0.670330   0.745617   0.580182   0.580328   0.580328   \n",
      "std     0.397393   0.353261   0.273618   0.419189   0.419106   0.419106   \n",
      "min     0.149695   0.000000   0.415637   0.029067   0.029304   0.029304   \n",
      "25%     0.215066   0.335288   0.429624   0.127962   0.128209   0.128209   \n",
      "50%     0.721579   0.796191   0.862891   0.662888   0.663474   0.663474   \n",
      "75%     1.000000   1.000000   1.000000   0.999950   0.999949   0.999949   \n",
      "max     1.000000   1.000000   1.000000   1.000000   1.000000   1.000000   \n",
      "\n",
      "attr    Geo Code  \n",
      "count  18.000000  \n",
      "mean    0.581310  \n",
      "std     0.419157  \n",
      "min     0.030612  \n",
      "25%     0.129027  \n",
      "50%     0.665327  \n",
      "75%     1.000000  \n",
      "max     1.000000  \n",
      "06:44:32 - [DEBUG] - computing frequency and co-occurrence statistics from raw data...\n",
      "06:44:32 - [DEBUG] - Collecting single/pair-wise statistics...\n",
      "06:44:34 - [DEBUG] - DONE computing statistics in 2.41s\n",
      "06:44:34 - [DEBUG] - preparing pruned co-occurring statistics...\n",
      "100%|██████████| 18/18 [00:00<00:00, 49.43it/s]\n",
      "06:44:34 - [DEBUG] - DONE with pruned co-occurring statistics in 0.37 secs\n",
      "06:44:34 - [DEBUG] - generating initial set of un-pruned domain values...\n",
      "100%|██████████| 1000/1000 [00:00<00:00, 1472.86it/s]\n",
      "06:44:35 - [DEBUG] - domain size stats: count    6970.000000\n",
      "mean       11.580631\n",
      "std        15.005491\n",
      "min         2.000000\n",
      "25%         4.000000\n",
      "50%         6.000000\n",
      "75%        12.000000\n",
      "max        80.000000\n",
      "Name: domain_size, dtype: float64\n",
      "06:44:35 - [DEBUG] - domain count by attr: partner      999\n",
      "Latitude     998\n",
      "Geo Code     998\n",
      "Longitude    998\n",
      "YearCons     993\n",
      "CMMoney      992\n",
      "Submitter    992\n",
      "Name: attribute, dtype: int64\n",
      "06:44:35 - [DEBUG] - DONE generating initial set of domain values in 0.72s\n",
      "06:44:36 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:36 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:36 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:36 - [DEBUG] - Time to create table: 0.00 secs\n",
      "06:44:37 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:37 - [ INFO] - DONE with domain preparation.\n",
      "06:44:37 - [DEBUG] - Time to generate the domain: 11.21 secs\n"
     ]
    }
   ],
   "source": [
    "hc.generate_domain()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "06:44:37 - [DEBUG] - training estimator for estimating domain value probabilities...\n",
      "06:44:37 - [DEBUG] - using estimator: NaiveBayes\n",
      "06:44:37 - [DEBUG] - DONE training estimator in 0.01s\n",
      "06:44:37 - [DEBUG] - predicting domain value probabilities from estimator...\n",
      "06:44:37 - [DEBUG] - DONE predictions in 0.00 secs, re-constructing cell domain...\n",
      "06:44:37 - [DEBUG] - re-assembling final cell domain table...\n",
      "100%|██████████| 6970/6970 [00:02<00:00, 3115.74it/s]\n",
      "100%|██████████| 6970/6970 [00:00<00:00, 142325.85it/s]\n",
      "06:44:39 - [DEBUG] - DONE assembling cell domain table in 2.52s\n",
      "06:44:39 - [ INFO] - number of (additional) weak labels assigned from estimator: 144\n",
      "06:44:39 - [DEBUG] - DONE generating domain and weak labels\n",
      "06:44:40 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:40 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:40 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:44:40 - [DEBUG] - Time to create table: 0.00 secs\n",
      "06:44:40 - [DEBUG] - Time to create index: 0.00 secs\n"
     ]
    }
   ],
   "source": [
    "hc.run_estimator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "06:44:40 - [DEBUG] - Time to execute query: 0.00 secs\n",
      "06:44:40 - [DEBUG] - Time to execute query: 0.00 secs\n",
      "06:44:40 - [DEBUG] - Time to execute query: 0.00 secs\n",
      "06:44:40 - [DEBUG] - featurizing training data...\n",
      "06:44:40 - [DEBUG] - Time to execute query: 0.01 secs\n",
      "100%|██████████| 6970/6970 [00:12<00:00, 559.48it/s]\n",
      "06:44:54 - [DEBUG] - DONE featurization. Feature tensor size: torch.Size([6970, 80, 144])\n",
      "06:44:54 - [DEBUG] - generating weak labels...\n",
      "06:44:54 - [DEBUG] - Time to execute query: 0.00 secs\n",
      "100%|██████████| 6661/6661 [00:00<00:00, 71341.86it/s]\n",
      "06:44:54 - [DEBUG] - DONE generating weak labels.\n",
      "06:44:54 - [DEBUG] - generating mask...\n",
      "06:44:54 - [DEBUG] - Time to execute query: 0.00 secs\n",
      "100%|██████████| 6970/6970 [00:00<00:00, 103917.20it/s]\n",
      "06:44:54 - [DEBUG] - DONE generating mask.\n",
      "06:44:54 - [ INFO] - DONE setting up featurized dataset.\n",
      "06:44:54 - [DEBUG] - Time to featurize data: 13.99 secs\n",
      "06:44:54 - [DEBUG] - training model with first layer size: ['torch.Size([18, 1])', 'torch.Size([126, 1])']\n",
      "06:44:54 - [ INFO] - DONE setting up repair model.\n",
      "06:44:54 - [DEBUG] - Time to setup repair model: 13.99 secs\n",
      "06:44:54 - [ INFO] - training with 6661 training examples (cells)\n",
      "  0%|          | 0/20 [00:00<?, ?it/s]06:44:56 - [DEBUG] - Epoch 1, cost = 0.252406, acc = 97.97%\n",
      "  5%|▌         | 1/20 [00:02<00:41,  2.20s/it]06:44:58 - [DEBUG] - Epoch 2, cost = 0.063995, acc = 98.05%\n",
      " 10%|█         | 2/20 [00:04<00:39,  2.19s/it]06:45:00 - [DEBUG] - Epoch 3, cost = 0.063603, acc = 98.05%\n",
      " 15%|█▌        | 3/20 [00:06<00:35,  2.10s/it]06:45:02 - [DEBUG] - Epoch 4, cost = 0.063573, acc = 98.05%\n",
      " 20%|██        | 4/20 [00:08<00:32,  2.02s/it]06:45:04 - [DEBUG] - Epoch 5, cost = 0.063571, acc = 98.05%\n",
      " 25%|██▌       | 5/20 [00:09<00:29,  1.97s/it]06:45:06 - [DEBUG] - Epoch 6, cost = 0.063570, acc = 98.05%\n",
      " 30%|███       | 6/20 [00:11<00:27,  1.94s/it]06:45:08 - [DEBUG] - Epoch 7, cost = 0.063570, acc = 98.05%\n",
      " 35%|███▌      | 7/20 [00:13<00:24,  1.91s/it]06:45:10 - [DEBUG] - Epoch 8, cost = 0.063570, acc = 98.05%\n",
      " 40%|████      | 8/20 [00:15<00:23,  1.93s/it]06:45:12 - [DEBUG] - Epoch 9, cost = 0.063570, acc = 98.05%\n",
      " 45%|████▌     | 9/20 [00:17<00:21,  1.92s/it]06:45:14 - [DEBUG] - Epoch 10, cost = 0.063570, acc = 98.05%\n",
      " 50%|█████     | 10/20 [00:19<00:19,  1.91s/it]06:45:16 - [DEBUG] - Epoch 11, cost = 0.063570, acc = 98.05%\n",
      " 55%|█████▌    | 11/20 [00:21<00:17,  2.00s/it]06:45:18 - [DEBUG] - Epoch 12, cost = 0.063570, acc = 98.05%\n",
      " 60%|██████    | 12/20 [00:23<00:16,  2.06s/it]06:45:20 - [DEBUG] - Epoch 13, cost = 0.063570, acc = 98.05%\n",
      " 65%|██████▌   | 13/20 [00:26<00:14,  2.10s/it]06:45:22 - [DEBUG] - Epoch 14, cost = 0.063570, acc = 98.05%\n",
      " 70%|███████   | 14/20 [00:28<00:12,  2.13s/it]06:45:25 - [DEBUG] - Epoch 15, cost = 0.063570, acc = 98.05%\n",
      " 75%|███████▌  | 15/20 [00:30<00:10,  2.15s/it]06:45:27 - [DEBUG] - Epoch 16, cost = 0.063570, acc = 98.05%\n",
      " 80%|████████  | 16/20 [00:32<00:08,  2.17s/it]06:45:29 - [DEBUG] - Epoch 17, cost = 0.063570, acc = 98.05%\n",
      " 85%|████████▌ | 17/20 [00:34<00:06,  2.07s/it]06:45:30 - [DEBUG] - Epoch 18, cost = 0.063570, acc = 98.05%\n",
      " 90%|█████████ | 18/20 [00:36<00:04,  2.02s/it]06:45:32 - [DEBUG] - Epoch 19, cost = 0.063570, acc = 98.05%\n",
      " 95%|█████████▌| 19/20 [00:38<00:01,  1.98s/it]06:45:34 - [DEBUG] - Epoch 20, cost = 0.063570, acc = 98.05%\n",
      "100%|██████████| 20/20 [00:40<00:00,  1.97s/it]\n",
      "06:45:34 - [ INFO] - DONE training repair model.\n",
      "06:45:34 - [DEBUG] - Time to fit repair model: 41.25 secs\n",
      "06:45:34 - [ INFO] - inferring on 453 examples (cells)\n",
      "06:45:34 - [DEBUG] - Time to execute query: 0.04 secs\n",
      "06:45:35 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:45:35 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:45:35 - [ INFO] - DONE inferring repairs.\n",
      "06:45:35 - [DEBUG] - Time to infer correct cell values: 0.17 secs\n",
      "06:45:35 - [DEBUG] - Time to create table: 0.00 secs\n",
      "06:45:35 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:45:35 - [DEBUG] - Time to create index: 0.00 secs\n",
      "06:45:35 - [ INFO] - DONE collecting the inferred values.\n",
      "06:45:35 - [DEBUG] - Time to collect inferred values: 0.02 secs\n",
      "06:45:35 - [ INFO] - DONE generating repaired dataset\n",
      "06:45:35 - [DEBUG] - Time to store repaired dataset: 0.08 secs\n",
      "06:45:35 - [ INFO] - featurizer FreqFeaturizer,size 18,max 0.0073,min -0.2639,avg -0.0143,abs_avg 0.0152,weights:\n",
      "Unnamed: 0 -0.0\n",
      "Instance -0.0\n",
      "Submission Date 0.0\n",
      "Submitter -0.0\n",
      "settlement 0.0\n",
      "Wptype 0.0\n",
      "handpump 0.0\n",
      "Functionality -0.0\n",
      "YearCons 0.007\n",
      "partner -0.001\n",
      "drinking? 0.0\n",
      "quantity -0.0\n",
      "committee 0.0\n",
      "CMMoney -0.264\n",
      "photo 0.0\n",
      "Latitude 0.0\n",
      "Longitude 0.0\n",
      "Geo Code 0.0\n",
      "featurizer OccurAttrFeaturizer,size 126,max 1.2106,min -0.2599,avg 0.1930,abs_avg 0.2070,weights:\n",
      "CMMoney | Unnamed: 0 0.262\n",
      "CMMoney | Instance 0.262\n",
      "CMMoney | Submission Date 0.262\n",
      "CMMoney | Submitter -0.166\n",
      "CMMoney | settlement 0.257\n",
      "CMMoney | Wptype -0.26\n",
      "CMMoney | handpump -0.26\n",
      "CMMoney | Functionality 0.505\n",
      "CMMoney | YearCons 0.514\n",
      "CMMoney | partner 0.642\n",
      "CMMoney | drinking? -0.201\n",
      "CMMoney | quantity 0.344\n",
      "CMMoney | committee 0.046\n",
      "CMMoney | CMMoney 0.004\n",
      "CMMoney | photo 0.921\n",
      "CMMoney | Latitude 0.916\n",
      "CMMoney | Longitude 0.928\n",
      "CMMoney | Geo Code 0.274\n",
      "Geo Code | Unnamed: 0 0.485\n",
      "Geo Code | Instance 0.014\n",
      "Geo Code | Submission Date 0.004\n",
      "Geo Code | Submitter 1.196\n",
      "Geo Code | settlement 1.205\n",
      "Geo Code | Wptype 1.211\n",
      "Geo Code | handpump 0.025\n",
      "Geo Code | Functionality 0.329\n",
      "Geo Code | YearCons 0.008\n",
      "Geo Code | partner 0.004\n",
      "Geo Code | drinking? 0.973\n",
      "Geo Code | quantity 0.33\n",
      "Geo Code | committee 0.984\n",
      "Geo Code | CMMoney 0.618\n",
      "Geo Code | photo 0.312\n",
      "Geo Code | Latitude 0.003\n",
      "Geo Code | Longitude 0.007\n",
      "Geo Code | Geo Code 0.936\n",
      "Latitude | Unnamed: 0 0.943\n",
      "Latitude | Instance 0.551\n",
      "Latitude | Submission Date 0.461\n",
      "Latitude | Submitter 0.415\n",
      "Latitude | settlement 0.013\n",
      "Latitude | Wptype 0.021\n",
      "Latitude | handpump 0.941\n",
      "Latitude | Functionality 0.913\n",
      "Latitude | YearCons 0.98\n",
      "Latitude | partner 0.313\n",
      "Latitude | drinking? 0.537\n",
      "Latitude | quantity 0.016\n",
      "Latitude | committee 0.031\n",
      "Latitude | CMMoney 0.503\n",
      "Latitude | photo 0.503\n",
      "Latitude | Latitude 0.476\n",
      "Latitude | Longitude 0.467\n",
      "Latitude | Geo Code 0.001\n",
      "Longitude | Unnamed: 0 0.0\n",
      "Longitude | Instance 0.003\n",
      "Longitude | Submission Date 0.618\n",
      "Longitude | Submitter 0.575\n",
      "Longitude | settlement 0.575\n",
      "Longitude | Wptype 0.563\n",
      "Longitude | handpump -0.0\n",
      "Longitude | Functionality -0.0\n",
      "Longitude | YearCons 0.0\n",
      "Longitude | partner 0.0\n",
      "Longitude | drinking? 0.0\n",
      "Longitude | quantity -0.0\n",
      "Longitude | committee -0.0\n",
      "Longitude | CMMoney 0.0\n",
      "Longitude | photo -0.0\n",
      "Longitude | Latitude 0.0\n",
      "Longitude | Longitude 0.0\n",
      "Longitude | Geo Code 0.0\n",
      "Submitter | Unnamed: 0 -0.0\n",
      "Submitter | Instance 0.0\n",
      "Submitter | Submission Date 0.0\n",
      "Submitter | Submitter -0.0\n",
      "Submitter | settlement -0.0\n",
      "Submitter | Wptype 0.0\n",
      "Submitter | handpump 0.0\n",
      "Submitter | Functionality -0.0\n",
      "Submitter | YearCons 0.0\n",
      "Submitter | partner -0.0\n",
      "Submitter | drinking? 0.0\n",
      "Submitter | quantity 0.0\n",
      "Submitter | committee -0.0\n",
      "Submitter | CMMoney -0.0\n",
      "Submitter | photo 0.0\n",
      "Submitter | Latitude 0.0\n",
      "Submitter | Longitude -0.0\n",
      "Submitter | Geo Code -0.0\n",
      "YearCons | Unnamed: 0 0.0\n",
      "YearCons | Instance 0.0\n",
      "YearCons | Submission Date -0.0\n",
      "YearCons | Submitter 0.0\n",
      "YearCons | settlement 0.0\n",
      "YearCons | Wptype 0.0\n",
      "YearCons | handpump -0.0\n",
      "YearCons | Functionality 0.0\n",
      "YearCons | YearCons 0.0\n",
      "YearCons | partner 0.0\n",
      "YearCons | drinking? 0.0\n",
      "YearCons | quantity -0.0\n",
      "YearCons | committee -0.0\n",
      "YearCons | CMMoney -0.0\n",
      "YearCons | photo 0.0\n",
      "YearCons | Latitude 0.0\n",
      "YearCons | Longitude -0.0\n",
      "YearCons | Geo Code -0.0\n",
      "partner | Unnamed: 0 -0.0\n",
      "partner | Instance -0.0\n",
      "partner | Submission Date 0.0\n",
      "partner | Submitter 0.0\n",
      "partner | settlement -0.0\n",
      "partner | Wptype 0.0\n",
      "partner | handpump 0.0\n",
      "partner | Functionality 0.0\n",
      "partner | YearCons 0.0\n",
      "partner | partner 0.0\n",
      "partner | drinking? 0.0\n",
      "partner | quantity -0.0\n",
      "partner | committee 0.0\n",
      "partner | CMMoney -0.0\n",
      "partner | photo -0.0\n",
      "partner | Latitude -0.0\n",
      "partner | Longitude 0.0\n",
      "partner | Geo Code -0.0\n",
      "\n",
      "06:45:35 - [DEBUG] - Time to store featurizer weights: 0.00 secs\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'featurizer FreqFeaturizer,size 18,max 0.0073,min -0.2639,avg -0.0143,abs_avg 0.0152,weights:\\nUnnamed: 0 -0.0\\nInstance -0.0\\nSubmission Date 0.0\\nSubmitter -0.0\\nsettlement 0.0\\nWptype 0.0\\nhandpump 0.0\\nFunctionality -0.0\\nYearCons 0.007\\npartner -0.001\\ndrinking? 0.0\\nquantity -0.0\\ncommittee 0.0\\nCMMoney -0.264\\nphoto 0.0\\nLatitude 0.0\\nLongitude 0.0\\nGeo Code 0.0\\nfeaturizer OccurAttrFeaturizer,size 126,max 1.2106,min -0.2599,avg 0.1930,abs_avg 0.2070,weights:\\nCMMoney | Unnamed: 0 0.262\\nCMMoney | Instance 0.262\\nCMMoney | Submission Date 0.262\\nCMMoney | Submitter -0.166\\nCMMoney | settlement 0.257\\nCMMoney | Wptype -0.26\\nCMMoney | handpump -0.26\\nCMMoney | Functionality 0.505\\nCMMoney | YearCons 0.514\\nCMMoney | partner 0.642\\nCMMoney | drinking? -0.201\\nCMMoney | quantity 0.344\\nCMMoney | committee 0.046\\nCMMoney | CMMoney 0.004\\nCMMoney | photo 0.921\\nCMMoney | Latitude 0.916\\nCMMoney | Longitude 0.928\\nCMMoney | Geo Code 0.274\\nGeo Code | Unnamed: 0 0.485\\nGeo Code | Instance 0.014\\nGeo Code | Submission Date 0.004\\nGeo Code | Submitter 1.196\\nGeo Code | settlement 1.205\\nGeo Code | Wptype 1.211\\nGeo Code | handpump 0.025\\nGeo Code | Functionality 0.329\\nGeo Code | YearCons 0.008\\nGeo Code | partner 0.004\\nGeo Code | drinking? 0.973\\nGeo Code | quantity 0.33\\nGeo Code | committee 0.984\\nGeo Code | CMMoney 0.618\\nGeo Code | photo 0.312\\nGeo Code | Latitude 0.003\\nGeo Code | Longitude 0.007\\nGeo Code | Geo Code 0.936\\nLatitude | Unnamed: 0 0.943\\nLatitude | Instance 0.551\\nLatitude | Submission Date 0.461\\nLatitude | Submitter 0.415\\nLatitude | settlement 0.013\\nLatitude | Wptype 0.021\\nLatitude | handpump 0.941\\nLatitude | Functionality 0.913\\nLatitude | YearCons 0.98\\nLatitude | partner 0.313\\nLatitude | drinking? 0.537\\nLatitude | quantity 0.016\\nLatitude | committee 0.031\\nLatitude | CMMoney 0.503\\nLatitude | photo 0.503\\nLatitude | Latitude 0.476\\nLatitude | Longitude 0.467\\nLatitude | Geo Code 0.001\\nLongitude | Unnamed: 0 0.0\\nLongitude | Instance 0.003\\nLongitude | Submission Date 0.618\\nLongitude | Submitter 0.575\\nLongitude | settlement 0.575\\nLongitude | Wptype 0.563\\nLongitude | handpump -0.0\\nLongitude | Functionality -0.0\\nLongitude | YearCons 0.0\\nLongitude | partner 0.0\\nLongitude | drinking? 0.0\\nLongitude | quantity -0.0\\nLongitude | committee -0.0\\nLongitude | CMMoney 0.0\\nLongitude | photo -0.0\\nLongitude | Latitude 0.0\\nLongitude | Longitude 0.0\\nLongitude | Geo Code 0.0\\nSubmitter | Unnamed: 0 -0.0\\nSubmitter | Instance 0.0\\nSubmitter | Submission Date 0.0\\nSubmitter | Submitter -0.0\\nSubmitter | settlement -0.0\\nSubmitter | Wptype 0.0\\nSubmitter | handpump 0.0\\nSubmitter | Functionality -0.0\\nSubmitter | YearCons 0.0\\nSubmitter | partner -0.0\\nSubmitter | drinking? 0.0\\nSubmitter | quantity 0.0\\nSubmitter | committee -0.0\\nSubmitter | CMMoney -0.0\\nSubmitter | photo 0.0\\nSubmitter | Latitude 0.0\\nSubmitter | Longitude -0.0\\nSubmitter | Geo Code -0.0\\nYearCons | Unnamed: 0 0.0\\nYearCons | Instance 0.0\\nYearCons | Submission Date -0.0\\nYearCons | Submitter 0.0\\nYearCons | settlement 0.0\\nYearCons | Wptype 0.0\\nYearCons | handpump -0.0\\nYearCons | Functionality 0.0\\nYearCons | YearCons 0.0\\nYearCons | partner 0.0\\nYearCons | drinking? 0.0\\nYearCons | quantity -0.0\\nYearCons | committee -0.0\\nYearCons | CMMoney -0.0\\nYearCons | photo 0.0\\nYearCons | Latitude 0.0\\nYearCons | Longitude -0.0\\nYearCons | Geo Code -0.0\\npartner | Unnamed: 0 -0.0\\npartner | Instance -0.0\\npartner | Submission Date 0.0\\npartner | Submitter 0.0\\npartner | settlement -0.0\\npartner | Wptype 0.0\\npartner | handpump 0.0\\npartner | Functionality 0.0\\npartner | YearCons 0.0\\npartner | partner 0.0\\npartner | drinking? 0.0\\npartner | quantity -0.0\\npartner | committee 0.0\\npartner | CMMoney -0.0\\npartner | photo -0.0\\npartner | Latitude -0.0\\npartner | Longitude 0.0\\npartner | Geo Code -0.0\\n'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "featurizers = [FreqFeaturizer(), OccurAttrFeaturizer()]\n",
    "hc.repair_errors(featurizers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = hc.ds.get_raw_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.equals(df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}