Missing Data

df.isna()            # element-wise bool DataFrame (True = missing)
df.isnull()          # alias for isna()
df.notna()           # element-wise True = present
df.notnull()         # alias for notna()

# Per-column count of missing values
df.isna().sum()

# Per-column percentage missing
df.isna().mean() * 100

# Total missing across entire DataFrame
df.isna().sum().sum()

# Rows with ANY missing value
df[df.isna().any(axis=1)]

# Rows where ALL values are missing
df[df.isna().all(axis=1)]

# Columns with any missing
df.columns[df.isna().any()]

# Drop rows with ANY NaN (default)
df.dropna()

# Drop rows where ALL values are NaN
df.dropna(how="all")

# Drop rows if NaN appears in specific columns
df.dropna(subset=["col1", "col2"])

# Drop columns with any NaN
df.dropna(axis=1)

# Require at least N non-null values per row
df.dropna(thresh=3)   # keep rows with ≥3 non-null values

# Series
s.dropna()

# Scalar fill
df.fillna(0)
df.fillna("")
df["col"].fillna(0)

# Per-column fill via dict
df.fillna({"age": df["age"].median(), "city": "Unknown"})

# Forward-fill (propagate last valid value forward)
df.ffill()
df["col"].ffill()
df.ffill(limit=2)     # propagate at most 2 consecutive NaNs

# Backward-fill
df.bfill()
df.bfill(limit=1)

# Fill with column mean/median/mode
df["score"].fillna(df["score"].mean())
df["score"].fillna(df["score"].median())
df["category"].fillna(df["category"].mode()[0])

# Fill with interpolation (see Interpolation section)
df["val"].interpolate()

# Linear interpolation (default)
df["val"].interpolate()

# Time-aware (requires DatetimeIndex)
df["val"].interpolate(method="time")

# Polynomial / spline
df["val"].interpolate(method="polynomial", order=2)
df["val"].interpolate(method="spline", order=3)

# Nearest neighbor
df["val"].interpolate(method="nearest")

# Limit direction
df["val"].interpolate(limit=2, limit_direction="both")   # "forward" | "backward" | "both"

# Only interpolate interior (not leading/trailing NaN)
df["val"].interpolate(limit_area="inside")   # "inside" | "outside"

# Keep value where condition True; replace False with NaN
df["col"].where(df["col"] > 0)

# Replace NaN with a specific value
df["col"].where(df["col"].notna(), other="MISSING")

# Count non-null values per column
df.count()        # equivalent to df.notna().sum()

# Count non-null per row
df.count(axis=1)

# Percentage complete per column
(df.notna().mean() * 100).round(1)

# These dtypes support pd.NA without converting to float
s = pd.array([1, 2, None, 4], dtype="Int64")   # capital I
s = pd.array([True, False, None], dtype="boolean")
s = pd.array(["a", "b", None], dtype="string")

# Convert a column to nullable integer
df["n"] = df["n"].astype("Int64")

# Arithmetic with pd.NA propagates NA (unlike NaN which may behave oddly with integers)

pd.isna(None)       # True
pd.isna(np.nan)     # True
pd.isna(pd.NaT)     # True
pd.isna(pd.NA)      # True
pd.isna(0)          # False

pd.notna("hello")   # True

missing = (
    df.isna()
      .sum()
      .rename("count")
      .to_frame()
      .assign(pct=lambda d: (d["count"] / len(df) * 100).round(2))
      .sort_values("count", ascending=False)
)
print(missing[missing["count"] > 0])