Overfitting and Cross-Validation: Find What Really Makes a Better Cup

"""Which roast settings make a better cup? Brewline Roasters' 200 roasts, 66 measurements each."""
import warnings

import numpy as np
import pandas as pd
from sklearn.feature_selection import SelectKBest, f_regression
from sklearn.linear_model import Lasso, LinearRegression, Ridge
from sklearn.model_selection import GridSearchCV, KFold, cross_val_score, train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.tree import DecisionTreeRegressor

warnings.filterwarnings("ignore", category=UserWarning)
TARGET = "cup_score"
SETTINGS = ["charge_temp", "first_crack_min", "development_ratio", "bean_moisture", "drop_temp", "airflow_pct"]
FOLDS = KFold(n_splits=5, shuffle=True, random_state=0)


def load():
    """(X_train, X_test, y_train, y_test): 75/25 split, random_state=0."""
    df = pd.read_csv("roasts.csv")
    X, y = df.drop(columns=["roast_id", TARGET]), df[TARGET]
    return train_test_split(X, y, test_size=0.25, random_state=0)


# ---------- Step 1: watch it overfit ----------
def depth_curve(X_train, y_train, X_test, y_test, depths):
    """For each max_depth, a DecisionTreeRegressor(max_depth=d, random_state=0) fitted on the training set:
    [{"depth": d, "train": R² on train, "test": R² on test}, ...], scores rounded to 3."""
    # TODO (Step 1): for each d, fit DecisionTreeRegressor(max_depth=d, random_state=0) on the training set and
    # record .score() (R²) on train and on test, rounded to 3.
    raise NotImplementedError("Step 1: write depth_curve()")


# ---------- Step 2: one split lies, cross-validation averages ----------
def split_spread(model, X, y, n=20):
    """Hold out 25% of (X, y) with random_state 0..n-1, fit a fresh clone of model on the rest, score R² on the
    held-out part. Returns {"min", "max"} of the n scores, rounded to 3."""
    raise NotImplementedError("split_spread() arrives in Step 2")


def cv_r2(model, X, y):
    """Mean R² over FOLDS, rounded to 3."""
    raise NotImplementedError("cv_r2() arrives in Step 2")


# ---------- Step 3: regularisation ----------
def linear(alpha=None, kind="ridge"):
    """Scaled features into LinearRegression (alpha None), Ridge(alpha) or Lasso(alpha, max_iter=50000)."""
    if alpha is None:
        return make_pipeline(StandardScaler(), LinearRegression())
    reg = Ridge(alpha=alpha) if kind == "ridge" else Lasso(alpha=alpha, max_iter=50000)
    return make_pipeline(StandardScaler(), reg)


def lasso_path(X, y, alphas):
    """For each alpha: {"alpha", "cv": cv_r2 of the Lasso, "features": names with a non-zero coefficient after
    fitting on all of (X, y), in column order}."""
    raise NotImplementedError("lasso_path() arrives in Step 3")


# ---------- Step 4: leakage inside cross-validation ----------
def noise_data(n=150, p=1000, seed=0):
    """A dataset with no signal at all: n rows of p random columns and a random target."""
    rng = np.random.default_rng(seed)
    return pd.DataFrame(rng.normal(size=(n, p))).add_prefix("x"), pd.Series(rng.normal(size=n))


def leaky_cv(X, y, k=20):
    """Pick the k columns most correlated with y using ALL rows, then cross-validate a LinearRegression on them."""
    raise NotImplementedError("leaky_cv() arrives in Step 4")


def honest_cv(X, y, k=20):
    """The same selection and model, as one pipeline, so each fold picks its columns from its own training rows."""
    raise NotImplementedError("honest_cv() arrives in Step 4")


# ---------- Step 5: choose by CV, test once ----------
ALPHAS = [0.01, 0.03, 0.1, 0.2, 0.3, 0.5, 1.0]


def final_model(X_train, y_train):
    """A GridSearchCV over Lasso alphas (ALPHAS) with cv=FOLDS, fitted on the training set only."""
    raise NotImplementedError("final_model() arrives in Step 5")


def setting_effects(search):
    """The roast settings' coefficients in the best model (cup-score points per standard deviation), largest
    effect first: [(name, coefficient rounded to 2), ...], settings only."""
    raise NotImplementedError("setting_effects() arrives in Step 5")