def fit(columns):
Phi = np.column_stack([np.ones(n)] + columns)
beta_hat = np.linalg.lstsq(Phi, y, rcond=None)[0]
return beta_hat, Phi
def mean_squared_error(beta_hat, Phi):
return np.mean((Phi @ beta_hat - y) ** 2)
def rmse(beta_hat, Phi):
return np.sqrt(mean_squared_error(beta_hat, Phi))
def r_squared(beta_hat, Phi):
sst = np.sum((y - y.mean()) ** 2)
return 1 - mean_squared_error(beta_hat, Phi) * n / sst
def ols_summary(beta_hat, Phi, y_obs=y):
"""Residual degrees of freedom, the estimated noise variance, and (Phi'Phi)^{-1}."""
df = len(y_obs) - Phi.shape[1]
sigma2 = np.sum((y_obs - Phi @ beta_hat) ** 2) / df
# The pseudoinverse gives (Phi'Phi)^{-1} without forming Phi'Phi,
# which is badly conditioned for high-degree polynomials
pinv = np.linalg.pinv(Phi)
return df, sigma2, pinv @ pinv.T
def coef_ci(beta_hat, Phi):
"""95% interval for each coefficient: the estimate plus or minus a t quantile times its standard error."""
df, sigma2, PtP_inv = ols_summary(beta_hat, Phi)
half = stats.t.ppf(0.975, df) * np.sqrt(sigma2 * np.diag(PtP_inv))
return np.column_stack([beta_hat - half, beta_hat + half])
def contrast_ci(beta_hat, Phi, c):
"""95% interval for the linear combination c'beta of the coefficients."""
df, sigma2, PtP_inv = ols_summary(beta_hat, Phi)
half = stats.t.ppf(0.975, df) * np.sqrt(sigma2 * c @ PtP_inv @ c)
return c @ beta_hat - half, c @ beta_hat + half
def mean_ci(beta_hat, Phi, Phi_new, prediction=False, y_obs=y):
"""95% interval at the rows of Phi_new: for the mean response, or with prediction=True for one new response."""
df, sigma2, PtP_inv = ols_summary(beta_hat, Phi, y_obs)
# The variance of the fitted value, plus the noise of one new response for a prediction interval
variance = sigma2 * (np.sum((Phi_new @ PtP_inv) * Phi_new, axis=1) + prediction)
half = stats.t.ppf(0.975, df) * np.sqrt(variance)
fitted_new = Phi_new @ beta_hat
return fitted_new - half, fitted_new + half
def bootstrap_r2_ci(Phi, B=2000, seed=571):
"""95% percentile interval for R^2 from B resamples of the cars, refitted each time."""
rng = np.random.default_rng(seed)
r2 = np.empty(B)
for b in range(B):
rows = rng.integers(0, n, n)
beta_b = np.linalg.lstsq(Phi[rows], y[rows], rcond=None)[0]
sse = np.sum((y[rows] - Phi[rows] @ beta_b) ** 2)
r2[b] = 1 - sse / np.sum((y[rows] - y[rows].mean()) ** 2)
return np.quantile(r2, [0.025, 0.975])