Capstone: Complete EDA Pipeline

Capstone: Complete EDA Pipeline

The EDA Framework

A structured EDA follows these steps:

1. Load and inspect — understand the shape and types
2. Data quality — missing values, duplicates, impossible values
3. Univariate analysis — distribution of each variable
4. Bivariate analysis — relationships between pairs of variables
5. Multivariate analysis — patterns across multiple variables
6. Statistical testing — confirm observed patterns formally
7. Summary and findings — communicate key insights

---

Step 1: Load and Inspect

library(dplyr)
library(ggplot2)
library(readr)

# We'll use the built-in mpg dataset (fuel economy data)
data <- mpg

# Shape and types
cat("Shape:", nrow(data), "×", ncol(data), "\n")
str(data)
glimpse(data)

# First look
head(data, 5)
tail(data, 3)

---

Step 2: Data Quality Assessment

# Missing values
na_counts <- colSums(is.na(data))
cat("Missing values:\n")
print(na_counts[na_counts > 0])

# Duplicates
n_dupes <- sum(duplicated(data))
cat(sprintf("\nDuplicate rows: %d\n", n_dupes))

# Unique values per column (good for spotting unexpected categories)
sapply(data, function(x) length(unique(x)))

# Data quality summary function
data_quality <- function(df) {
  data.frame(
    column   = names(df),
    type     = sapply(df, class),
    n_unique = sapply(df, function(x) length(unique(x))),
    n_na     = colSums(is.na(df)),
    pct_na   = round(colMeans(is.na(df)) * 100, 1)
  ) %>%
    arrange(desc(pct_na))
}
print(data_quality(data))

---

Step 3: Univariate Analysis

# Numeric variables — distribution summary
numeric_cols <- data %>% select(where(is.numeric))

describe_all <- numeric_cols %>%
  summarise(across(everything(), list(
    mean   = \(x) round(mean(x, na.rm=T), 2),
    median = \(x) round(median(x, na.rm=T), 2),
    sd     = \(x) round(sd(x, na.rm=T), 2),
    min    = \(x) min(x, na.rm=T),
    max    = \(x) max(x, na.rm=T),
    iqr    = \(x) round(IQR(x, na.rm=T), 2)
  )))

# Detect outliers using IQR rule
detect_outliers <- function(x) {
  q <- quantile(x, c(0.25, 0.75), na.rm = TRUE)
  iqr <- q[2] - q[1]
  lower <- q[1] - 1.5 * iqr
  upper <- q[2] + 1.5 * iqr
  sum(x < lower | x > upper, na.rm = TRUE)
}

cat("Outlier counts:\n")
print(sapply(numeric_cols, detect_outliers))

# Distribution plots — histograms for all numeric columns
library(tidyr)
numeric_long <- numeric_cols %>%
  pivot_longer(everything(), names_to = "variable", values_to = "value")

ggplot(numeric_long, aes(x = value)) +
  geom_histogram(bins = 20, fill = "#2196F3", color = "white", alpha = 0.8) +
  facet_wrap(~variable, scales = "free") +
  labs(title = "Distribution of All Numeric Variables") +
  theme_minimal()

---

Step 4: Bivariate Analysis

# Correlation heatmap
cor_mat <- cor(numeric_cols, use = "complete.obs")

library(reshape2)
cor_melted <- melt(cor_mat)
ggplot(cor_melted, aes(Var1, Var2, fill = value)) +
  geom_tile(color = "white") +
  scale_fill_gradient2(low = "#B71C1C", mid = "white", high = "#1565C0",
                       midpoint = 0, limits = c(-1, 1)) +
  geom_text(aes(label = round(value, 2)), size = 3) +
  labs(title = "Correlation Matrix", x = NULL, y = NULL) +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1))

# Box plots: hwy mpg by drive type
ggplot(data, aes(x = drv, y = hwy, fill = drv)) +
  geom_boxplot(alpha = 0.8) +
  geom_jitter(width = 0.1, alpha = 0.4, size = 1) +
  labs(title = "Highway MPG by Drive Type", x = "Drive Type", y = "Hwy MPG") +
  theme_minimal() +
  theme(legend.position = "none")

---

Step 5: Statistical Testing

# Does highway MPG differ significantly between drive types?
anova_model <- aov(hwy ~ drv, data = data)
summary(anova_model)

# Post-hoc pairwise comparisons
tukey <- TukeyHSD(anova_model)
print(tukey)

# Correlation test: engine displacement vs MPG
cor.test(data$displ, data$hwy)
# Expected: strong negative correlation

# Effect size for key comparison
front <- data$hwy[data$drv == "f"]
four  <- data$hwy[data$drv == "4"]
cohens_d <- function(x, y) {
  pooled <- sqrt(((length(x)-1)*var(x) + (length(y)-1)*var(y)) /
                  (length(x)+length(y)-2))
  abs(mean(x) - mean(y)) / pooled
}
cat(sprintf("\nCohen's d (f vs 4WD): %.2f\n", cohens_d(front, four)))

---

Step 6: Summary Report

cat("=== EDA SUMMARY ===\n")
cat(sprintf("Dataset: %d rows × %d columns\n", nrow(data), ncol(data)))
cat(sprintf("Missing values: %d (%.1f%%)\n",
    sum(is.na(data)), mean(is.na(data)) * 100))
cat(sprintf("Duplicate rows: %d\n", sum(duplicated(data))))

cat("\nKey findings:\n")
cat(sprintf("1. Mean highway MPG: %.1f (range: %d-%d)\n",
    mean(data$hwy), min(data$hwy), max(data$hwy)))
cat(sprintf("2. Engine displacement negatively correlates with MPG: r=%.2f\n",
    cor(data$displ, data$hwy)))
cat(sprintf("3. ANOVA confirms hwy MPG differs by drive type: p < 0.001\n"))
cat(sprintf("4. Front-wheel drive cars average %.1f mpg hwy vs 4WD's %.1f mpg\n",
    mean(data$hwy[data$drv=="f"]), mean(data$hwy[data$drv=="4"])))