Step 1: Trees against a linear model
Swiftline's parcels arrive late about one time in five, and the customer service team wants a warning before it happens. What makes a parcel late is rarely one thing: rain matters most at rush hour; one depot struggles only at weekends; new drivers are slow, and after a year they aren't. A linear model adds effects up. Gradient-boosted trees learn these combinations.
XGBoost can read pandas categories and missing values directly, so you don't need one-hot encoding, scaling or imputation.
Do this
1. Write to_frame(df): the features, with depot and address_type as the pandas category dtype.
2. Write booster(**params): an XGBClassifier set up for this lab, on one CPU thread.
3. Run it and compare the two cross-validated AUCs.
boost.py, the file you edit107 lines
"""Which Swiftline parcels will arrive late? Gradient-boosted trees on 6,000 past deliveries."""
import warnings
import numpy as np
import pandas as pd
import xgboost as xgb
from sklearn.compose import make_column_transformer
from sklearn.impute import SimpleImputer
from sklearn.inspection import permutation_importance
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import RandomizedSearchCV, StratifiedKFold, cross_val_score, train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler
warnings.filterwarnings("ignore", category=UserWarning)
TARGET = "late"
CATEGORICAL = ["depot", "address_type"]
FOLDS = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)
# ---------- Step 1: trees against a linear model ----------
def to_frame(df):
"""Features for the models: every column except the target, CATEGORICAL columns as pandas 'category' dtype,
missing numbers left as NaN (the trees handle them)."""
# TODO (Step 1): drop TARGET, convert each CATEGORICAL column with .astype("category"), leave NaN alone.
raise NotImplementedError("Step 1: write to_frame()")
def load():
"""(X_train, X_test, y_train, y_test): 80/20, stratified, random_state=0."""
df = pd.read_csv("parcels.csv")
return train_test_split(to_frame(df), df[TARGET], test_size=0.2, stratify=df[TARGET], random_state=0)
def logistic():
"""The linear baseline: one-hot categories, median-imputed and scaled numbers, logistic regression."""
numeric = make_pipeline(SimpleImputer(strategy="median"), StandardScaler())
prep = make_column_transformer((OneHotEncoder(handle_unknown="ignore"), CATEGORICAL),
(numeric, lambda X: [c for c in X.columns if c not in CATEGORICAL]))
return make_pipeline(prep, LogisticRegression(max_iter=2000))
def booster(**params):
"""An XGBClassifier that reads pandas categories (enable_categorical=True, tree_method="hist"), on one CPU
thread (n_jobs=1), random_state=0, eval_metric="logloss", with any extra params passed through."""
# TODO (Step 1): return xgb.XGBClassifier(enable_categorical=True, tree_method="hist", n_jobs=1, random_state=0,
# eval_metric="logloss", **params)
raise NotImplementedError("Step 1: write booster()")
def cv_auc(model, X, y):
"""Mean ROC AUC over FOLDS, a float rounded to 3."""
return round(float(cross_val_score(model, X, y, cv=FOLDS, scoring="roc_auc").mean()), 3)
# ---------- Step 2: learning rate, rounds and early stopping ----------
def round_curve(X_train, y_train, learning_rate, rounds=1000):
"""Hold out 25% of the training set (stratified, random_state=0), fit booster(n_estimators=rounds,
learning_rate=learning_rate, max_depth=6) with that holdout as eval_set, and read the log loss after every
round. Returns {"best_round": the round (from 1) with the lowest holdout log loss, "best_val": that loss,
"last_val": holdout loss after the final round, "last_train": training loss after the final round},
losses rounded to 4."""
raise NotImplementedError("round_curve() arrives in Step 2")
def early_stopped(X_train, y_train, **params):
"""booster(n_estimators=2000, early_stopping_rounds=50, **params) fitted on 75% of the training set with the
same 25% holdout as eval_set. Returns the fitted model (its best_iteration says when to stop)."""
raise NotImplementedError("early_stopped() arrives in Step 2")
# ---------- Step 3: tune the trees ----------
SPACE = {"max_depth": [2, 3, 4, 5, 6], "min_child_weight": [1, 3, 5, 10], "subsample": [0.6, 0.8, 1.0],
"colsample_bytree": [0.6, 0.8, 1.0]}
def tune(X_train, y_train, n_iter=12):
"""RandomizedSearchCV over SPACE for booster(n_estimators=300, learning_rate=0.05): n_iter candidates,
3-fold StratifiedKFold (shuffle, random_state=0), scoring roc_auc, random_state=0, n_jobs=1. Fitted."""
raise NotImplementedError("tune() arrives in Step 3")
BEST = {"max_depth": 3, "min_child_weight": 3, "subsample": 0.8, "colsample_bytree": 1.0, "learning_rate": 0.05}
# ---------- Step 4: which features matter? ----------
def importances(model, X_val, y_val):
"""Three rankings of the features, most important first, each a list of names:
"splits": how often the trees split on the feature (booster importance_type="weight"),
"gain": the average loss reduction of those splits (importance_type="gain"),
"permutation": how much ROC AUC on (X_val, y_val) drops when the feature's values are shuffled
(permutation_importance, n_repeats=5, random_state=0). Features the trees never used rank last."""
raise NotImplementedError("importances() arrives in Step 4")
# ---------- Step 5: the final model, and why this parcel? ----------
def final_model(X_train, y_train):
"""Early-stop BEST to find the number of rounds, then fit booster(**BEST, n_estimators=that number) on the
whole training set."""
raise NotImplementedError("final_model() arrives in Step 5")
def explain(model, row):
"""Why the model scored one parcel (a one-row DataFrame) as it did: xgboost's per-feature contributions to the
log-odds (predict with pred_contribs=True). Returns {"base": the bias term, "contributions": [(feature,
contribution rounded to 3), ...] largest absolute first}."""
raise NotImplementedError("explain() arrives in Step 5")Provided for you:
parcels.csvtry_it.py