End-to-End Analysis Pipeline

pip install pandas matplotlib seaborn scikit-learn

import pandas as pd  # pandas is the library every step below builds on: it turns raw rows into a DataFrame you can filter, group, and clean

# In a real project this data would already exist as a CSV exported from a database or spreadsheet.
# It's generated inline here so this tutorial is fully self-contained and reproducible with no external download.
SAMPLE_CSV = """square_feet,bedrooms,age_years,neighborhood_score,price
1200,2,15,7.2,245000
1550,3,8,8.1,312000
980,2,32,5.4,178000
2100,4,5,9.0,415000
1400,3,20,,268000
1750,3,12,7.8,335000
2300,4,3,9.4,455000
1100,2,25,,205000
1600,3,18,6.9,298000
1900,4,10,8.5,372000
"""  # note the two blank entries in the neighborhood_score column — those become NaN values pandas has to handle in Step 2

with open("housing.csv", "w") as f:   # write the sample data to disk once, so read_csv() below behaves exactly like it would on a real file
    f.write(SAMPLE_CSV)

df = pd.read_csv("housing.csv")   # read_csv infers each column's dtype automatically (int64, float64, etc.) from the text it parses

print(df.head())      # first 5 rows — a quick sanity check that every column parsed into the right place
print(df.info())      # column dtypes plus non-null counts per column — this is where the 2 missing neighborhood_score values surface
print(df.describe())  # count/mean/std/min/max per numeric column — a fast way to spot outliers or unexpected scale before modeling

print(df.isna().sum())  # confirms exactly which column(s) are missing values and how many — here, neighborhood_score: 2

# Why median, not mean: the mean is pulled toward outliers, so one unusually high or low
# neighborhood_score would skew every row that gets filled with it. The median doesn't have that problem.
median_score = df["neighborhood_score"].median()
df["neighborhood_score"] = df["neighborhood_score"].fillna(median_score)  # fill, don't drop — dropping wastes otherwise-good rows

# Why fillna() over dropna(): with only 10 rows, dropping the 2 rows missing neighborhood_score
# would throw away 20% of the dataset over a single blemished column, discarding perfectly good
# square_feet/bedrooms/age_years/price data along with it. Filling keeps every row usable.
print(df.isna().sum())  # every column should now report 0 missing values

import matplotlib.pyplot as plt  # the plotting engine; pyplot gives a simple, stateful interface for building one figure at a time
import seaborn as sns            # wraps matplotlib with statistical-plot defaults — cleaner styling and a built-in heatmap() call

# Scatter plot: does square footage actually predict price? This is the question Step 4 needs
# answered before it's worth trusting square_feet as a model feature at all.
sns.scatterplot(data=df, x="square_feet", y="price")
plt.title("Price vs. Square Footage")
plt.xlabel("Square Feet")
plt.ylabel("Price ($)")
plt.savefig("price_vs_sqft.png")  # saved to a file instead of plt.show() so this script also runs unattended in a terminal or CI job
plt.close()                       # closes the current figure so the next plt call starts a fresh, empty figure

# Heatmap: of the four numeric features, which correlates most strongly with price? This
# guides feature selection in Step 4 instead of guessing which columns actually matter.
plt.figure(figsize=(6, 5))
sns.heatmap(df.corr(numeric_only=True), annot=True, cmap="coolwarm")  # annot=True prints each correlation value directly on the cell
plt.title("Feature Correlation Heatmap")
plt.savefig("correlation_heatmap.png")
plt.close()

from sklearn.model_selection import train_test_split  # splits data so evaluation happens on rows the model never trained on
from sklearn.linear_model import LinearRegression       # a simple, interpretable baseline — see the note below for why it fits here

FEATURES = ["square_feet", "bedrooms", "age_years", "neighborhood_score"]  # every numeric column except the target itself
X = df[FEATURES]
y = df["price"]

# test_size=0.2 holds back 20% of rows purely for evaluation in Step 5; random_state=42 makes the
# split reproducible, so re-running this script doesn't silently change which rows land in the test set.
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Why LinearRegression and not a more powerful model like RandomForestRegressor: Step 3's scatter
# plot already showed price trending roughly linearly with square footage, and with only 10 rows
# total, a linear model is both sufficient and far more interpretable — its coefficients say
# directly "how many dollars per extra square foot," which a tree ensemble's output can't.
model = LinearRegression()
model.fit(X_train, y_train)  # learns one coefficient per feature plus an intercept from X_train/y_train

from sklearn.metrics import mean_absolute_error, r2_score  # two complementary ways to judge fit: error in dollars, and variance explained

predictions = model.predict(X_test)  # predict on the held-out rows the model never saw during fit()

mae = mean_absolute_error(y_test, predictions)  # average absolute dollar error — easy to explain to a non-technical stakeholder
r2 = r2_score(y_test, predictions)              # fraction of price variance the model explains; 1.0 would be a perfect fit

print(f"Mean Absolute Error: ${mae:,.2f}")
print(f"R^2 Score: {r2:.3f}")

# Printing each coefficient alongside its feature name turns the model's internals into a plain-
# English sentence: "every extra square foot is worth about $X in this data," which is the kind
# of interpretability that was the whole justification for choosing LinearRegression in Step 4.
for feature, coef in zip(FEATURES, model.coef_):
    print(f"  {feature}: ${coef:,.2f}")
print(f"Intercept: ${model.intercept_:,.2f}")

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_absolute_error, r2_score

# --- Step 1: Set up and load the sample dataset ---
SAMPLE_CSV = """square_feet,bedrooms,age_years,neighborhood_score,price
1200,2,15,7.2,245000
1550,3,8,8.1,312000
980,2,32,5.4,178000
2100,4,5,9.0,415000
1400,3,20,,268000
1750,3,12,7.8,335000
2300,4,3,9.4,455000
1100,2,25,,205000
1600,3,18,6.9,298000
1900,4,10,8.5,372000
"""

with open("housing.csv", "w") as f:
    f.write(SAMPLE_CSV)

df = pd.read_csv("housing.csv")
print(df.head())
print(df.info())
print(df.describe())

# --- Step 2: Clean the data ---
print(df.isna().sum())
median_score = df["neighborhood_score"].median()
df["neighborhood_score"] = df["neighborhood_score"].fillna(median_score)
print(df.isna().sum())

# --- Step 3: Visualize relationships ---
sns.scatterplot(data=df, x="square_feet", y="price")
plt.title("Price vs. Square Footage")
plt.xlabel("Square Feet")
plt.ylabel("Price ($)")
plt.savefig("price_vs_sqft.png")
plt.close()

plt.figure(figsize=(6, 5))
sns.heatmap(df.corr(numeric_only=True), annot=True, cmap="coolwarm")
plt.title("Feature Correlation Heatmap")
plt.savefig("correlation_heatmap.png")
plt.close()

# --- Step 4: Train a model ---
FEATURES = ["square_feet", "bedrooms", "age_years", "neighborhood_score"]
X = df[FEATURES]
y = df["price"]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

model = LinearRegression()
model.fit(X_train, y_train)

# --- Step 5: Evaluate the model ---
predictions = model.predict(X_test)
mae = mean_absolute_error(y_test, predictions)
r2 = r2_score(y_test, predictions)

print(f"Mean Absolute Error: ${mae:,.2f}")
print(f"R^2 Score: {r2:.3f}")
for feature, coef in zip(FEATURES, model.coef_):
    print(f"  {feature}: ${coef:,.2f}")
print(f"Intercept: ${model.intercept_:,.2f}")
print("Saved price_vs_sqft.png and correlation_heatmap.png")