Open-Source Models

from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"
tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.bfloat16,
    device_map="auto"
)

messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Explain gradient descent."}
]
input_ids = tokenizer.apply_chat_template(messages, return_tensors="pt").to("cuda")
output = model.generate(input_ids, max_new_tokens=500, temperature=0.7, do_sample=True)
response = tokenizer.decode(output[0][input_ids.shape[-1]:], skip_special_tokens=True)

# Mixtral with transformers
model = AutoModelForCausalLM.from_pretrained(
    "mistralai/Mixtral-8x7B-Instruct-v0.1",
    torch_dtype=torch.float16,
    device_map="auto"
)

from diffusers import FluxPipeline
import torch

pipe = FluxPipeline.from_pretrained("black-forest-labs/FLUX.1-schnell",
                                     torch_dtype=torch.bfloat16)
pipe.enable_model_cpu_offload()  # save VRAM

image = pipe(
    "A serene mountain lake at dusk, photorealistic",
    guidance_scale=0.0,   # Flux.1-schnell uses 0 guidance
    num_inference_steps=4,
    max_sequence_length=256
).images[0]
image.save("mountain.png")

# Install: https://ollama.com
ollama pull llama3.1:8b
ollama run llama3.1:8b
ollama pull nomic-embed-text
ollama pull qwen3:8b

import ollama

response = ollama.chat(
    model="llama3.1:8b",
    messages=[{"role": "user", "content": "What is backpropagation?"}]
)
print(response["message"]["content"])

# Embeddings
emb = ollama.embeddings(model="nomic-embed-text", prompt="Hello world")

from llama_cpp import Llama

# Download from HuggingFace: TheBloke/*.GGUF or bartowski/*.GGUF
llm = Llama.from_pretrained(
    repo_id="bartowski/Meta-Llama-3.1-8B-Instruct-GGUF",
    filename="Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf",
    n_gpu_layers=-1,   # use all GPU layers
    n_ctx=8192
)
output = llm.create_chat_completion(
    messages=[{"role": "user", "content": "Explain transformers"}],
    max_tokens=512, temperature=0.7
)

from transformers import pipeline

# Text generation
generator = pipeline("text-generation", model="mistralai/Mistral-7B-Instruct-v0.3",
                      torch_dtype=torch.bfloat16, device_map="auto")
result = generator("Explain the attention mechanism", max_new_tokens=300)

# Text embeddings
extractor = pipeline("feature-extraction", model="BAAI/bge-large-en-v1.5",
                      device="cuda")
embedding = extractor("Hello world", return_tensors="pt")

from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model, TaskType
from trl import SFTTrainer, SFTConfig
from datasets import load_dataset

# 4-bit quantization
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3.1-8B",
    quantization_config=bnb_config,
    device_map="auto"
)

# LoRA config
lora = LoraConfig(
    r=16, lora_alpha=32, lora_dropout=0.05,
    target_modules=["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj","down_proj"],
    task_type=TaskType.CAUSAL_LM
)
model = get_peft_model(model, lora)

dataset = load_dataset("json", data_files="train.jsonl")["train"]

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    args=SFTConfig(
        output_dir="./fine_tuned",
        per_device_train_batch_size=4,
        gradient_accumulation_steps=4,
        learning_rate=2e-4,
        num_train_epochs=3,
        fp16=False, bf16=True,
        logging_steps=10,
        save_strategy="epoch"
    )
)
trainer.train()
trainer.save_model()