"""Reproduce out-of-sample variance adjustment from the Hillstrom public CSV.GZ."""
from collections import defaultdict
import csv
import gzip
from hashlib import sha256
import json
from pathlib import Path
from statistics import mean, variance
import sys


ARMS = ('No E-Mail', 'Mens E-Mail', 'Womens E-Mail')


def contrast(groups, outcome, treatment):
    control, treated = ([outcome(row) for row in groups[arm]] for arm in (ARMS[0], treatment))
    return {'difference': mean(treated)-mean(control),
            'variance': variance(treated)/len(treated)+variance(control)/len(control)}


def analyze(path):
    train = []
    held = defaultdict(list)
    with gzip.open(path, 'rt', newline='') as stream:
        for i, row in enumerate(csv.DictReader(stream)):
            pair = (float(row['history']), float(row['spend']))
            if i % 2 == 0:
                train.append(pair)
            else:
                held[row['segment']].append(pair)
    assert set(held) == set(ARMS) and len(train) == sum(map(len, held.values())) == 32000
    histories, outcomes = zip(*train)
    center = mean(histories)
    outcome_mean = mean(outcomes)
    covariance = sum((x-center)*(y-outcome_mean) for x, y in train)/(len(train)-1)
    theta = covariance/variance(histories)
    adjust = lambda row: row[1]-theta*(row[0]-center)
    unadjusted = lambda row: row[1]
    contrasts = {arm: {'raw': contrast(held, unadjusted, arm),
                       'adjusted': contrast(held, adjust, arm)} for arm in ARMS[1:]}
    return {'sha256': sha256(Path(path).read_bytes()).hexdigest(), 'training': len(train),
            'held': {arm: len(held[arm]) for arm in ARMS}, 'theta': theta,
            'center': center, 'training_correlation': covariance/(variance(histories)*variance(outcomes))**0.5,
            'contrasts': contrasts}


if __name__ == '__main__':
    print(json.dumps(analyze(sys.argv[1]), indent=2))
