Data Cleaning and Feature Engineering: Predict Rents From a Messy Export

"""Predict monthly rents for Hearthside Lettings from its listings export."""
import numpy as np
import pandas as pd
from sklearn.compose import ColumnTransformer, TransformedTargetRegressor
from sklearn.impute import SimpleImputer
from sklearn.linear_model import Ridge
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler

TARGET = "rent"
SQFT_PER_SQM = 10.764
EPC = "ABCDEFG"


# ---------- Step 1: clean the export ----------
def clean(raw):
    """One consistent table: borough in Title Case without spaces around it; property_type one of flat, house,
    studio; floor_area in square metres; rent per calendar month (pw * 52 / 12, rounded to whole pounds);
    furnished 1/0 (NaN when blank); epc_rating a letter or NaN; listed_on a date. Drops area_unit and
    rent_period, which the conversions make redundant."""
    # TODO (Step 1): work on raw.copy().
    # - borough: strip spaces, Title Case.  property_type: "studio" if it mentions studio, "house" if it mentions
    #   house, otherwise "flat" (np.select on the lowercased text).
    # - floor_area: rows with area_unit "sqft" divided by SQFT_PER_SQM (round to 1).  rent: rows with rent_period
    #   "pw" times 52 / 12 (round) - only when the rent column exists.
    # - furnished: y/yes -> 1, n/no -> 0, anything else NaN.  epc_rating: keep A-G, anything else NaN.
    # - listed_on: pd.to_datetime.  Drop area_unit and rent_period.
    raise NotImplementedError("Step 1: write clean()")


# ---------- Step 2: a fair test and a baseline to beat ----------
def split(df):
    """80/20 train/test split with random_state=42."""
    return train_test_split(df.drop(columns=[TARGET]), df[TARGET], test_size=0.2, random_state=42)


def baseline_mae(X_train, y_train, X_test, y_test):
    """MAE on the test set of two no-model guesses: the median rent of the training set ("median"), and the
    median training rent of the listing's borough ("borough_median"), rounded to whole pounds."""
    raise NotImplementedError("baseline_mae() arrives in Step 2")


NUMERIC = ["bedrooms", "bathrooms", "floor_area", "tube_km", "garden", "furnished"]
CATEGORICAL = ["borough", "property_type", "epc_rating"]


def make_model(numeric=NUMERIC, categorical=CATEGORICAL, indicator=True, log_target=False):
    """Ridge regression behind a ColumnTransformer. Numeric columns: median imputation (with missing-value
    indicator columns when indicator=True), then scaling. Categorical: missing values become the category
    "missing", then one-hot encoding that ignores unseen categories. log_target=True fits on log(rent)."""
    raise NotImplementedError("make_model() arrives in Step 2")


def cv_mae(model, X, y):
    """Mean absolute error over 5-fold cross-validation (KFold, shuffle=True, random_state=42), in whole pounds."""
    folds = KFold(n_splits=5, shuffle=True, random_state=42)
    return round(-cross_val_score(model, X, y, cv=folds, scoring="neg_mean_absolute_error").mean())


# ---------- Step 3: missing values carry information ----------
def epc_score(df):
    """A copy of df with epc_score: 1 for EPC A up to 7 for G, NaN where the rating is missing."""
    raise NotImplementedError("epc_score() arrives in Step 3")


def missing_values_report(X, y):
    """Cross-validated MAE of three ways to use the EPC rating (X must already have epc_score):
    "one_hot": NUMERIC + CATEGORICAL (the rating as a category, missing as its own category);
    "score": NUMERIC + epc_score with borough and property_type, median-imputed without indicators;
    "score_indicator": the same with missing-value indicators."""
    raise NotImplementedError("missing_values_report() arrives in Step 3")


# ---------- Step 4: features that match how rent works ----------
def add_features(df):
    """A copy of df with: log_area = log(floor_area), log_tube = log1p(tube_km), area_per_room =
    floor_area / (bedrooms + 1), listed_month = the month number of listed_on."""
    out = df.copy()
    out["log_area"] = np.log(out["floor_area"])
    out["log_tube"] = np.log1p(out["tube_km"])
    out["area_per_room"] = out["floor_area"] / (out["bedrooms"] + 1)
    out["listed_month"] = out["listed_on"].dt.month
    return out


BASE_NUMERIC = ["bedrooms", "bathrooms", "floor_area", "tube_km", "garden", "furnished", "epc_score"]
BASE_CATEGORICAL = ["borough", "property_type"]
# candidate feature -> the column it replaces (None when it is simply added)
CANDIDATES = {"log_area": "floor_area", "log_tube": "tube_km", "area_per_room": None, "listed_month": None}


def ablation(X, y):
    """For each candidate: the cross-validated MAE of the log-target model on BASE_NUMERIC with every candidate in
    place ("all"), and without that one candidate (its replaced column comes back). Returns
    {"all": mae, candidate: gain, ...} where gain = MAE without it - MAE with all (positive = it helps)."""
    raise NotImplementedError("ablation() arrives in Step 4")


# ---------- Step 5: one look at the test set ----------
FINAL_NUMERIC = list(BASE_NUMERIC)


def prepare(raw):
    """clean(), then epc_score() and add_features()."""
    return add_features(epc_score(clean(raw)))


def final_model():
    return make_model(FINAL_NUMERIC, BASE_CATEGORICAL, indicator=True, log_target=True)


def predict_new(model, raw_new):
    """A DataFrame with listing_id and predicted_rent (whole pounds) for the new listings."""
    raise NotImplementedError("predict_new() arrives in Step 5")