Inference

model.eval()                          # disable Dropout, use BN running stats
with torch.inference_mode():          # no grad graph built
    output = model(input_tensor)

import torch
from torchvision import transforms

# Single image (PIL → tensor → batch)
transform = transforms.Compose([
    transforms.Resize((224, 224)),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406],
                         std=[0.229, 0.224, 0.225]),
])
img_tensor = transform(pil_image).unsqueeze(0)   # (1, 3, 224, 224)

# Batch of numpy arrays
x = torch.from_numpy(np_array).float()
x = x.to(device)

model.eval()

with torch.inference_mode():
    # Single sample — add batch dim
    single = input_tensor.unsqueeze(0).to(device)  # (1, C, H, W)
    out = model(single)
    out = out.squeeze(0)   # remove batch dim

    # Batch
    for x in loader:
        x = x.to(device)
        out = model(x)

# Multi-class classification
logits = model(x)                      # (N, C)
probs = torch.softmax(logits, dim=1)   # (N, C)
pred_class = logits.argmax(dim=1)      # (N,)
top5_probs, top5_idx = torch.topk(probs, k=5, dim=1)

# Binary classification
logit = model(x)                       # (N, 1) or (N,)
prob = torch.sigmoid(logit)
pred = (prob > 0.5).long()

# Regression
value = model(x)   # (N, 1) or (N, out_dim)

# Segmentation / dense output
mask_logits = model(x)                 # (N, num_classes, H, W)
mask = mask_logits.argmax(dim=1)       # (N, H, W)

# Object detection — unpack model-specific output dict
detections = model(images)             # list of dicts: boxes, labels, scores

with torch.inference_mode():
    out = model(x.to(device))

# To numpy / Python
arr = out.cpu().numpy()            # stays float32
scalar = out.item()                # single-element tensor → Python float
python_list = out.tolist()

# With class labels
class_idx = out.argmax(dim=1).item()
class_name = idx_to_class[class_idx]

from torch.utils.data import DataLoader, TensorDataset

# Build dataset and loader
test_dataset = MyDataset(test_data)
test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False,
                         num_workers=4, pin_memory=True)

model.eval()
all_preds = []
all_probs = []

with torch.inference_mode():
    for x in test_loader:
        x = x.to(device, non_blocking=True)
        logits = model(x)
        probs = torch.softmax(logits, dim=1)
        preds = logits.argmax(dim=1)
        all_preds.append(preds.cpu())
        all_probs.append(probs.cpu())

all_preds = torch.cat(all_preds)    # (N,)
all_probs = torch.cat(all_probs)    # (N, C)

from torch.amp import autocast

model.eval()
with torch.inference_mode():
    with autocast('cuda', dtype=torch.float16):
        output = model(x.to(device))

# Load a traced or scripted model (no class definition needed)
scripted = torch.jit.load('model_scripted.pt', map_location=device)
scripted.eval()

with torch.inference_mode():
    output = scripted(input_tensor.to(device))

import onnxruntime as ort
import numpy as np

session = ort.InferenceSession('model.onnx',
                               providers=['CUDAExecutionProvider',
                                          'CPUExecutionProvider'])

input_name = session.get_inputs()[0].name
output_name = session.get_outputs()[0].name

# Input must be numpy array
x_np = x.numpy()
result = session.run([output_name], {input_name: x_np})[0]
output = torch.from_numpy(result)

model.eval()
compiled = torch.compile(model, mode='reduce-overhead')   # good for small batches
compiled = torch.compile(model, mode='max-autotune')      # best throughput, slow warmup

with torch.inference_mode():
    output = compiled(x)

import time

model.eval()
results = {}

for batch_size in [1, 8, 16, 32, 64, 128]:
    x = torch.randn(batch_size, 3, 224, 224, device=device)
    # warmup
    with torch.inference_mode():
        for _ in range(5):
            model(x)
    torch.cuda.synchronize()

    start = time.perf_counter()
    with torch.inference_mode():
        for _ in range(50):
            model(x)
    torch.cuda.synchronize()
    elapsed = time.perf_counter() - start

    throughput = batch_size * 50 / elapsed
    results[batch_size] = throughput
    print(f'bs={batch_size:3d}  {throughput:.0f} samples/sec')

from torchao.quantization import (
    quantize_,
    Int8WeightOnlyConfig,
    Int8DynamicActivationInt8WeightConfig,
    Int4WeightOnlyConfig,
)

# Weight-only int8 — safe general speedup, ~2x less VRAM
quantize_(model, Int8WeightOnlyConfig())

# Dynamic int8 activations + int8 weights — Linear-heavy models
quantize_(model, Int8DynamicActivationInt8WeightConfig())

# Weight-only int4 (grouped) — LLM inference
quantize_(model, Int4WeightOnlyConfig(group_size=128))

model = torch.compile(model, mode='max-autotune')   # pair with compile
with torch.inference_mode():
    output = model(x)

import torch.ao.quantization as tq

# Post-training dynamic (weights only) — LSTM/Transformer on CPU
quantized = tq.quantize_dynamic(model, qconfig_spec={torch.nn.Linear},
                                dtype=torch.qint8)

# Post-training static (activations calibrated)
model.qconfig = tq.get_default_qconfig('x86')   # or 'qnnpack' for ARM/mobile
tq.prepare(model, inplace=True)
with torch.inference_mode():
    for x, _ in calibration_loader:             # calibration pass
        model(x)
tq.convert(model, inplace=True)

# Quantization-aware training (QAT)
model.qconfig = tq.get_default_qat_qconfig('x86')
tq.prepare_qat(model, inplace=True)
# ... train normally ...
tq.convert(model.eval(), inplace=True)

import torch.nn.utils.prune as prune

# Unstructured pruning — zero out lowest-magnitude weights
prune.l1_unstructured(model.fc1, name='weight', amount=0.3)   # prune 30%

# Structured pruning — remove entire filters/channels
prune.ln_structured(model.conv1, name='weight', amount=0.2, n=2, dim=0)

# Make pruning permanent (remove mask, keep zeroed weights)
prune.remove(model.fc1, 'weight')

# Global pruning across layers
prune.global_unstructured(
    [(model.fc1, 'weight'), (model.fc2, 'weight')],
    pruning_method=prune.L1Unstructured,
    amount=0.2,
)

# Check sparsity
total = model.fc1.weight.numel()
zero = float(torch.sum(model.fc1.weight == 0))
print(f'Sparsity: {100 * zero / total:.1f}%')

from torch.utils.benchmark import Timer

t = Timer(
    stmt='model(x)',
    setup='',
    globals={'model': model, 'x': x},
    num_threads=1,
)
result = t.timeit(100)
print(result)
# Includes: mean, median, IQR — handles CUDA sync automatically