A reusable Python toolkit that audits any CSV for missing values, duplicates, wrong types, and outliers — then fixes them and produces a clean output file with a full report.
The first step is always to understand what is wrong before fixing anything.
import pandas as pd
import numpy as np
df = pd.read_csv('dirty_data.csv')
def audit(df):
print(f"Shape: {df.shape[0]:,} rows × {df.shape[1]} columns\n")
print("Missing values:")
missing = df.isnull().sum()
missing_pct = (missing / len(df) * 100).round(1)
for col in df.columns:
if missing[col] > 0:
print(f" {col}: {missing[col]} missing ({missing_pct[col]}%)")
print(f"\nDuplicates: {df.duplicated().sum():,} rows")
print("\nData types:")
for col, dtype in df.dtypes.items():
print(f" {col}: {dtype}")
audit(df)Shape: 1,000 rows × 6 columns Missing values: age: 47 missing (4.7%) salary: 23 missing (2.3%) email: 12 missing (1.2%) Duplicates: 18 rows Data types: name: object age: float64 salary: object ← should be numeric! email: object
Different strategies for different column types — fill numeric with median, drop rows with missing key fields.
def fix_missing(df):
report = []
for col in df.select_dtypes(include='number').columns:
n = df[col].isnull().sum()
if n > 0:
median = df[col].median()
df[col] = df[col].fillna(median)
report.append(f" Filled {n} missing '{col}' values with median ({median:.1f})")
# Drop rows where key text fields are missing
key_cols = [c for c in ['email', 'name', 'id'] if c in df.columns]
before = len(df)
df = df.dropna(subset=key_cols)
dropped = before - len(df)
if dropped:
report.append(f" Dropped {dropped} rows with missing key fields: {key_cols}")
return df, report
df, log = fix_missing(df)
for line in log:
print(line)Filled 47 missing 'age' values with median (32.0) Filled 23 missing 'salary' values with median (52000.0) Dropped 12 rows with missing email
Drop exact duplicate rows and convert columns to their correct types.
def fix_types_and_duplicates(df):
report = []
# Remove duplicates
before = len(df)
df = df.drop_duplicates()
report.append(f" Removed {before - len(df)} duplicate rows")
# Fix salary: remove currency symbols, commas, convert to float
if 'salary' in df.columns and df['salary'].dtype == object:
df['salary'] = (df['salary']
.str.replace(r'[₹$,]', '', regex=True)
.str.strip()
.astype(float))
report.append(" Converted 'salary' from string to float")
# Standardise text columns: strip whitespace, title-case names
if 'name' in df.columns:
df['name'] = df['name'].str.strip().str.title()
report.append(" Cleaned 'name' column: stripped whitespace, title-cased")
# Fix email: lowercase
if 'email' in df.columns:
df['email'] = df['email'].str.lower().str.strip()
report.append(" Normalised 'email' to lowercase")
return df, report
df, log = fix_types_and_duplicates(df)
for line in log:
print(line)Use the IQR method to find outliers and cap them rather than deleting rows.
def fix_outliers(df, columns):
report = []
for col in columns:
if col not in df.columns:
continue
Q1 = df[col].quantile(0.25)
Q3 = df[col].quantile(0.75)
IQR = Q3 - Q1
lower = Q1 - 1.5 * IQR
upper = Q3 + 1.5 * IQR
outliers = ((df[col] < lower) | (df[col] > upper)).sum()
df[col] = df[col].clip(lower=lower, upper=upper)
if outliers:
report.append(f" Capped {outliers} outliers in '{col}' to [{lower:.0f}, {upper:.0f}]")
return df, report
df, log = fix_outliers(df, ['age', 'salary'])
for line in log:
print(line)
# Save clean file
df.to_csv('clean_data.csv', index=False)
print(f"\nSaved clean_data.csv — {len(df):,} rows remaining")Capped 11 outliers in 'age' to [15, 68] Capped 8 outliers in 'salary' to [18000, 120000] Saved clean_data.csv — 970 rows remaining
Data scientists spend 60-80% of their time cleaning data. This toolkit handles the 5 most common problems: missing values, duplicates, wrong types, inconsistent formatting, and outliers. Drop it into any project as a reusable module.
Spotted a bug, broken code, or something that doesn't look right? Tell us what's off and we'll fix it.