Exploratory Data Analysis (EDA) in Python: A Complete Project

import numpy as np
import pandas as pd

rng = np.random.default_rng(2026)
n = 500
cities = rng.choice(["Mumbai", "Delhi", "Bengaluru", "Pune", "Chennai"], n, p=[0.28, 0.25, 0.22, 0.13, 0.12])
channel = rng.choice(["search", "social", "referral"], n, p=[0.5, 0.3, 0.2])
member = rng.random(n) < 0.35
orders = rng.poisson(lam=np.where(member, 9, 5))
spend = (orders * rng.normal(900, 150, n) + np.where(channel == "referral", 1500, 0)).round(0).clip(min=0)
age = rng.normal(34, 9, n).round(0).clip(18, 70)

df = pd.DataFrame({
    "customer_id": np.arange(1001, 1001 + n),
    "city": cities, "age": age, "channel": channel,
    "orders": orders, "spend": spend, "member": np.where(member, "yes", "no"),
})
# real-world messiness
df.loc[rng.choice(n, 25, replace=False), "age"] = np.nan
df.loc[rng.choice(n, 40, replace=False), "city"] = df["city"].str.lower()
df.loc[rng.choice(n, 15, replace=False), "city"] = " " + df["city"] + " "
df.loc[7, "spend"] = 950000                      # a data entry error
df = pd.concat([df, df.iloc[[10, 20, 30]]], ignore_index=True)   # duplicated rows
df.to_csv("customers.csv", index=False)
print(df.shape)

import pandas as pd

df = pd.read_csv("customers.csv")
print(df.head())
df.info()

import pandas as pd

df = pd.read_csv("customers.csv")
print(df.describe().round(1))
print(df["city"].value_counts())
print("duplicate rows:", df.duplicated().sum())

import pandas as pd

df = pd.read_csv("customers.csv")
before = len(df)

df = df.drop_duplicates()
df["city"] = df["city"].str.strip().str.title()

q1, q3 = df["spend"].quantile([0.25, 0.75])
upper = q3 + 3 * (q3 - q1)                     # a generous outlier fence
print("outliers:", df.loc[df["spend"] > upper, ["customer_id", "orders", "spend"]].to_dict("records"))
df = df[df["spend"] <= upper]

df["age"] = df["age"].fillna(df["age"].median())

print(before, "->", len(df), "rows")
print(df["city"].value_counts())
print(df.isna().sum().sum(), "missing values left")
df.to_csv("customers_clean.csv", index=False)

import pandas as pd

df = pd.read_csv("customers_clean.csv")
print(df[["age", "orders", "spend"]].describe().round(1))
print("median spend:", df["spend"].median())
print(df["member"].value_counts(normalize=True).round(3))

import pandas as pd

df = pd.read_csv("customers_clean.csv")
by_city = df.groupby("city").agg(customers=("customer_id", "count"),
                                 avg_spend=("spend", "mean"),
                                 total_spend=("spend", "sum")).round(0)
print(by_city.sort_values("total_spend", ascending=False))

by_channel = df.groupby("channel").agg(customers=("customer_id", "count"),
                                       avg_orders=("orders", "mean"),
                                       avg_spend=("spend", "mean")).round(1)
print(by_channel.sort_values("avg_spend", ascending=False))

import pandas as pd
from scipy import stats

df = pd.read_csv("customers_clean.csv")
print(df.groupby("member")[["orders", "spend"]].agg(["mean", "median"]).round(1))

members = df.loc[df["member"] == "yes", "spend"]
others = df.loc[df["member"] == "no", "spend"]
result = stats.ttest_ind(members, others, equal_var=False)
print(f"p-value: {result.pvalue:.1e}")

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

df = pd.read_csv("customers_clean.csv")
fig, axes = plt.subplots(1, 3, figsize=(15, 4))

sns.histplot(data=df, x="spend", bins=30, ax=axes[0])
axes[0].set_title("Spend per customer (₹)")

order = df.groupby("channel")["spend"].mean().sort_values(ascending=False).index
sns.barplot(data=df, x="channel", y="spend", order=order, ax=axes[1])
axes[1].set_title("Average spend by signup channel")

sns.boxplot(data=df, x="member", y="spend", hue="member", order=["no", "yes"], ax=axes[2])
axes[2].set_title("Spend: loyalty members vs others")

fig.tight_layout()
plt.show()