Reading and Writing

import pandas as pd

df = pd.read_csv("data.csv")

# Key parameters
df = pd.read_csv(
    "data.csv",
    sep=",",             # delimiter (use sep="\t" for TSV)
    header=0,            # row number for column names (None = no header)
    names=["a", "b"],    # supply column names manually
    index_col="id",      # column to use as row index
    usecols=["a", "b"],  # only read these columns
    dtype={"age": int},  # per-column dtype
    parse_dates=["date"],# parse these columns as datetime
    nrows=1000,          # read only first N rows
    skiprows=2,          # skip N rows from top (or list of row numbers)
    skipfooter=1,        # skip N rows at end (engine="python" required)
    na_values=["NA", "?"],# extra strings to treat as NaN
    keep_default_na=True,
    encoding="utf-8",
    chunksize=10_000,    # returns TextFileReader iterator
    engine="c",          # "c" (fast) or "python" (flexible)
    thousands=",",       # thousands separator
    decimal=".",
    comment="#",         # ignore lines starting with this char
    quotechar='"',
    low_memory=False,    # disable dtype guessing per chunk
)

chunks = pd.read_csv("big.csv", chunksize=10_000)
df = pd.concat(chunks, ignore_index=True)

# Process without loading all into memory
for chunk in pd.read_csv("big.csv", chunksize=10_000):
    process(chunk)

df.to_csv("out.csv")

df.to_csv(
    "out.csv",
    index=False,          # don't write row index
    columns=["a", "b"],   # subset of columns
    sep=",",
    na_rep="",            # string for NaN
    float_format="%.2f",
    date_format="%Y-%m-%d",
    encoding="utf-8",
    header=True,
    mode="a",             # append mode
    lineterminator="\n",
)

# Reading
df = pd.read_excel("data.xlsx")
df = pd.read_excel("data.xlsx", sheet_name="Sheet1")
df = pd.read_excel("data.xlsx", sheet_name=0)   # by position
all_sheets = pd.read_excel("data.xlsx", sheet_name=None)  # dict of DataFrames

# Writing (requires openpyxl)
df.to_excel("out.xlsx", index=False)
df.to_excel("out.xlsx", sheet_name="Results", index=False)

# Multiple sheets
with pd.ExcelWriter("out.xlsx") as writer:
    df1.to_excel(writer, sheet_name="Sheet1", index=False)
    df2.to_excel(writer, sheet_name="Sheet2", index=False)

# Append to existing file
with pd.ExcelWriter("out.xlsx", mode="a", engine="openpyxl") as writer:
    df.to_excel(writer, sheet_name="New")

# Reading
df = pd.read_json("data.json")
df = pd.read_json("data.json", orient="records")  # list of row dicts
df = pd.read_json("data.json", orient="columns")  # {col: {idx: val}}
df = pd.read_json("data.json", orient="index")
df = pd.read_json("data.json", orient="split")    # {index, columns, data}
df = pd.read_json("data.json", lines=True)        # newline-delimited JSON

# Writing
df.to_json("out.json")
df.to_json("out.json", orient="records", lines=True)
df.to_json("out.json", orient="records", indent=2)
json_str = df.to_json(orient="records")

# Requires pyarrow or fastparquet
df = pd.read_parquet("data.parquet")
df = pd.read_parquet("data.parquet", columns=["a", "b"])
df = pd.read_parquet("data/", engine="pyarrow")   # directory of parts

df.to_parquet("out.parquet", index=False)
df.to_parquet("out.parquet", engine="pyarrow", compression="snappy")

# Fast binary; use for in-process hand-off
df = pd.read_feather("data.feather")
df.to_feather("out.feather")

import sqlalchemy as sa

engine = sa.create_engine("postgresql://user:pw@host/db")

# Read entire table
df = pd.read_sql_table("orders", con=engine)

# Read with query
df = pd.read_sql("SELECT * FROM orders WHERE amount > 100", con=engine)
df = pd.read_sql_query("SELECT id, name FROM users", con=engine)

# Write
df.to_sql("orders", con=engine, if_exists="append", index=False)
# if_exists: "fail" | "replace" | "append"

# Chunked read
for chunk in pd.read_sql("SELECT * FROM big_table", con=engine, chunksize=5000):
    process(chunk)

# Clipboard (great for quick ad-hoc work)
df = pd.read_clipboard()
df.to_clipboard(index=False)

# HTML (reads first <table> by default)
dfs = pd.read_html("https://example.com/table")  # returns list
df = dfs[0]

# Fixed-width format
df = pd.read_fwf("data.fwf", colspecs=[(0, 10), (10, 20)])

# HDF5 (requires tables/pytables)
df.to_hdf("data.h5", key="df", mode="w")
df = pd.read_hdf("data.h5", key="df")

# Pickle
df.to_pickle("data.pkl")
df = pd.read_pickle("data.pkl")

# Stata / SAS / SPSS
df = pd.read_stata("data.dta")
df = pd.read_sas("data.sas7bdat")
df = pd.read_spss("data.sav")

# Read directly from URL
df = pd.read_csv("https://example.com/data.csv")

# Compressed files — auto-detected by extension
df = pd.read_csv("data.csv.gz")
df = pd.read_csv("data.csv.zip")
df = pd.read_csv("data.csv.bz2")
df = pd.read_csv("data.csv.xz")

# Write compressed
df.to_csv("out.csv.gz", index=False, compression="gzip")
df.to_parquet("out.parquet", compression="zstd")

import io

# Parse CSV from a string
csv_string = "a,b\n1,2\n3,4"
df = pd.read_csv(io.StringIO(csv_string))

# Write CSV to a string buffer
buf = io.StringIO()
df.to_csv(buf, index=False)
csv_str = buf.getvalue()