LLAMA.CPP
HANDBOOK

# Clone latest — repo now lives under the ggml-org GitHub org
git clone https://github.com/ggml-org/llama.cpp
cd llama.cpp

# Build with CUDA — GGML_CUDA=ON enables GPU support
cmake -B build \
  -DGGML_CUDA=ON \
  -DCMAKE_CUDA_ARCHITECTURES="89;120" \   # 89=Ada(40xx) 120=Blackwell(50xx)
  -DCMAKE_BUILD_TYPE=Release

cmake --build build --config Release -j$(nproc)

# Binaries land in ./build/bin/
ls build/bin/
# → llama-server  llama-cli  llama-bench  llama-quantize

# Download latest pre-built CUDA binary from GitHub Releases
# Releases use a "b" build-number scheme (e.g. b9976), not semver
RELEASE=$(curl -s https://api.github.com/repos/ggml-org/llama.cpp/releases/latest | grep tag_name | cut -d'"' -f4)

# Linux CUDA build
wget "https://github.com/ggml-org/llama.cpp/releases/download/${RELEASE}/llama-${RELEASE}-bin-ubuntu-x64.zip"
unzip llama-*.zip -d llama-cpp
export PATH="$PWD/llama-cpp:$PATH"

# Verify CUDA detected
./llama-cpp/llama-server --version
# Should show: CUDA available, GPU count: 1

# Official images published under ghcr.io/ggml-org
docker run --gpus all -v ~/models:/models -p 8080:8080 \
  ghcr.io/ggml-org/llama.cpp:server-cuda \
  -hf ggml-org/gemma-3-4b-it-GGUF --fit -c 8192

# CPU-only tag if no GPU passthrough available
docker run -v ~/models:/models -p 8080:8080 \
  ghcr.io/ggml-org/llama.cpp:server -hf ggml-org/Qwen3-4B-GGUF

# Pull a specific quant (recommended — be explicit)
llama-server -hf unsloth/Qwen3-4B-Instruct-2507-GGUF:Q4_K_M --fit

# Omit the quant tag and llama.cpp picks a sensible default (usually Q4_K_M)
llama-server -hf ggml-org/gemma-3-4b-it-GGUF --fit

# Vision model — mmproj file is auto-resolved and downloaded alongside
llama-server -hf ggml-org/Qwen3-VL-8B-Instruct-GGUF --fit

# Files land in the standard HF cache (~/.cache/llama.cpp or $HF_HOME)
# so re-runs are instant — no re-download

penalties → dry → top_n_sigma → top_k → typ_p → top_p → min_p → xtc → temperature

# Order matters: penalties and DRY run first (on the raw logit set),
# truncation samplers narrow the pool, temperature is applied last.
# Override the whole chain with --samplers "top_k;top_p;temp" if you want
# the old-style simple pipeline.

# Keep attention + shared/dense layers on GPU (always active, latency-critical)
# Push the sparse FFN expert tensors to CPU RAM (rarely all active at once)
llama-server -hf unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF:Q4_K_XL \
  -ngl 99 \
  --override-tensor "\.ffn_(up|down|gate)_exps\.=CPU" \
  -c 32768 -fa on

# Shorthand: many recipes now write this as --override-tensor 'exps=CPU'
# or the newer --cpu-moe / --n-cpu-moe N convenience flags that do the
# same regex under the hood without hand-writing it
llama-server -hf ggml-org/gpt-oss-20b-GGUF -ngl 99 --n-cpu-moe 999 --fit

# VRAM per layer ≈ model_file_size / total_layers
# Total VRAM ≈ (layers_on_gpu × vram_per_layer) + kv_cache + overhead

# Example: Qwen2.5-14B Q4_K_M (8.9 GB file, 48 layers)
vram_per_layer = 8900 MB / 48 = ~185 MB / layer
kv_cache_f16   = 2 × 48 × 40 × 128 × 8192 × 2 bytes ≈ ~3200 MB (8K ctx)
overhead       = ~500 MB (CUDA context, activations)

# For 12 GB VRAM:
available = 12000 - 3200 - 500 = 8300 MB for weights
max_layers = 8300 / 185 = ~44 layers → use -ngl 44

# With quantized KV cache (q8_0):
kv_cache_q8 = ~1600 MB → available = 9900 MB → -ngl 53 (close to all 48!)

./llama-server \
  -m ./Qwen3-14B-Instruct-Q4_K_M.gguf \
  -ngl 30 \                        # 30/48 layers on GPU
  -c 4096 \                        # Smaller context to save KV cache VRAM
  -b 512 \                         # Smaller batch to reduce peak VRAM
  -fa on \                         # Flash attention — saves attention VRAM
  --cache-type-k q8_0 \            # Quantize K cache — halves KV VRAM
  --cache-type-v q8_0 \
  -ot "token_embd=CPU" \           # Embedding to CPU (saves ~1.2 GB)
  -ot "output=CPU" \               # LM head to CPU
  -t 8 \                           # 8 threads for CPU layers
  --host 127.0.0.1 --port 8080 \
  --jinja -a "qwen14b"

./llama-server \
  -m ./Llama-3.3-70B-Instruct-Q4_K_M.gguf \
  -ngl 10 \                        # Only 10 layers on GPU (70B has 80 layers)
  -c 2048 \                        # Keep context small
  -t 16 \                          # 16 CPU threads for the 70 CPU layers
  -tb 16 \                         # 16 threads for prefill
  -b 512 \
  --no-mmap \                      # Load fully into RAM — avoids page faults
  -fa on \
  --cache-type-k q4_0 \            # Maximum KV compression
  --cache-type-v q4_0 \
  --host 127.0.0.1 --port 8080 \
  --jinja -a "llama3-70b"
# Expect ~3-6 tok/s — the PCIe + CPU compute limits throughput

./llama-server \
  -m ./Qwen3-8B-Instruct-Q4_K_M.gguf \
  -ngl 99 \                        # All layers to GPU
  -c 8192 \
  -fa on \
  --cache-type-k q8_0 \
  --cache-type-v q8_0 \
  -b 2048 \
  --cont-batching \
  -np 2 \                          # 2 parallel slots for multiple users
  --host 127.0.0.1 --port 8080 \
  --jinja \
  -a "qwen8b" \
  --defrag-thold 0.1               # Auto-defrag KV cache

./llama-server \
  -hf unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF:Q4_K_XL \
  -ngl 99 \                        # Attention + shared layers on GPU
  --n-cpu-moe 999 \                # Sparse expert tensors → CPU RAM
  --fit \                          # Auto-tune context/batch to remaining VRAM
  -fa on \
  --host 127.0.0.1 --port 8080 \
  --jinja -a "qwen3-coder-30b"
# Only ~3B active params per token → near-dense-7B speed
# despite a 30B total parameter checkpoint

# Interactive chat (GPU, Qwen2.5)
./llama-cli \
  -m ./Qwen2.5-7B-Instruct-Q4_K_M.gguf \
  -ngl 99 \
  -c 4096 \
  --temp 0.7 \
  --top-p 1.0 \
  --min-p 0.05 \
  --repeat-penalty 1.05 \
  --jinja \
  -i -if           # -i = interactive, -if = interactive first

# One-shot generation (pipe to output)
./llama-cli \
  -m ./model.gguf \
  -ngl 99 \
  -p "Explain ELSS mutual funds in 3 bullet points" \
  -n 256 \
  --temp 0.3 \
  --no-display-prompt \
  -s 42            # -s = seed (reproducible output)

# Benchmark to find optimal -ngl
./llama-bench \
  -m ./Qwen2.5-7B-Instruct-Q4_K_M.gguf \
  -ngl 0,10,20,28 \    # Test multiple values
  -p 512 -n 128 \
  -r 3                 # 3 repetitions per config

#!/bin/bash
# launch_server.sh — parameterized llama-server launch

MODEL="${MODEL:-./models/Qwen2.5-7B-Instruct-Q4_K_M.gguf}"
PORT="${PORT:-8080}"
CTX="${CTX:-8192}"
NGL="${NGL:-99}"
THREADS="${THREADS:-12}"
ALIAS="${ALIAS:-local-model}"

./llama-server \
  -m  "$MODEL" \
  -a  "$ALIAS" \
  -ngl $NGL \
  -c  $CTX \
  -t  $THREADS \
  -tb $THREADS \
  -b  2048 \
  -fa \
  --cache-type-k q8_0 \
  --cache-type-v q8_0 \
  --cont-batching \
  --defrag-thold 0.1 \
  --jinja \
  --host 127.0.0.1 \
  --port $PORT \
  --log-disable \            # Suppress verbose logs
  2>&1 | tee llama-server.log

# Usage:
# ./launch_server.sh
# MODEL=./14B.gguf NGL=30 CTX=4096 ./launch_server.sh

# Health check
curl http://localhost:8080/health

# List models
curl http://localhost:8080/v1/models | python3 -m json.tool

# Test completion
curl http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "local-model",
    "messages": [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user",   "content": "What is Section 80C?"}
    ],
    "temperature": 0.7,
    "max_tokens": 256
  }' | python3 -m json.tool

from openai import OpenAI

# llama-server is OpenAI API-compatible
client = OpenAI(
    base_url="http://localhost:8080/v1",
    api_key="not-needed",   # Required by SDK but ignored if no --api-key set
)

# ─── Non-streaming completion ─────────
response = client.chat.completions.create(
    model="local-model",
    messages=[
        {"role": "system",  "content": "You are Arthavidya, an Indian finance expert."},
        {"role": "user",    "content": "Explain ELSS tax saving mutual funds."},
    ],
    temperature=0.7,
    max_tokens=512,
    top_p=0.9,
    frequency_penalty=0.1,
    presence_penalty=0.05,
)
print(response.choices[0].message.content)

# ─── Streaming response ───────────────
stream = client.chat.completions.create(
    model="local-model",
    messages=[{"role": "user", "content": "Write a haiku about investing."}],
    stream=True,
    temperature=1.0,
    max_tokens=100,
)
for chunk in stream:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="", flush=True)
print()

import requests, json

def chat(messages: list, temperature: float = 0.7, max_tokens: int = 512) -> str:
    response = requests.post(
        "http://localhost:8080/v1/chat/completions",
        json={
            "model": "local-model",
            "messages": messages,
            "temperature": temperature,
            "max_tokens": max_tokens,
            "top_p": 0.9,
            "repeat_penalty": 1.05,   # llama.cpp native field
            "min_p": 0.05,              # llama.cpp extension
        },
        timeout=120,
    )
    response.raise_for_status()
    return response.json()["choices"][0]["message"]["content"]

# ─── Example usage ────────────────────
answer = chat([
    {"role": "system", "content": "You are a helpful finance assistant."},
    {"role": "user",   "content": "What is Section 80D deduction?"},
])
print(answer)

# ─── Check server info ────────────────
info = requests.get("http://localhost:8080/v1/models").json()
print(f"Loaded model: {info['data'][0]['id']}")

from langchain_openai import ChatOpenAI
from langchain_core.messages import SystemMessage, HumanMessage

# Point LangChain at local llama-server
llm = ChatOpenAI(
    model="local-model",
    base_url="http://localhost:8080/v1",
    api_key="not-needed",
    temperature=0.7,
    max_tokens=1024,
)

messages = [
    SystemMessage(content="You are an expert in Indian taxation."),
    HumanMessage(content="What is the difference between ELSS and PPF?"),
]

response = llm.invoke(messages)
print(response.content)

{
  "model": "./models/Qwen2.5-7B-Instruct-Q4_K_M.gguf",
  "alias": "qwen7b",
  "n_gpu_layers": 99,
  "ctx_size": 8192,
  "batch_size": 2048,
  "ubatch_size": 512,
  "threads": 12,
  "threads_batch": 12,
  "flash_attn": true,
  "cache_type_k": "q8_0",
  "cache_type_v": "q8_0",
  "cont_batching": true,
  "defrag_thold": 0.1,
  "jinja": true,
  "host": "127.0.0.1",
  "port": 8080,
  "n_parallel": 1,
  "temperature": 0.7,
  "top_p": 1.0,
  "min_p": 0.05,
  "repeat_penalty": 1.05
}

{
  "model": "./models/Qwen2.5-14B-Instruct-Q4_K_M.gguf",
  "alias": "qwen14b",
  "n_gpu_layers": 32,
  "ctx_size": 4096,
  "batch_size": 512,
  "ubatch_size": 256,
  "threads": 12,
  "threads_batch": 16,
  "flash_attn": true,
  "cache_type_k": "q4_0",
  "cache_type_v": "q4_0",
  "tensor_split": "",
  "override_tensor": "token_embd=CPU,output=CPU",
  "no_mmap": false,
  "jinja": true,
  "host": "127.0.0.1",
  "port": 8080
}

./llama-server --config config_7b_12gb.json

# Override specific values from config via CLI flags
./llama-server --config config_7b_12gb.json --port 8081 -c 16384

Raw logits from model
    ↓
Penalties               (--repeat-penalty / --presence-penalty / --frequency-penalty)
    ↓
DRY                     (--dry-multiplier — breaks phrase-level repetition loops)
    ↓
Top-N-Sigma             (--top-nsigma — off by default)
    ↓
Top-K filtering         (--top-k 0 to disable)
    ↓
Typical-P               (--typical-p — off by default)
    ↓
Top-P / nucleus         (--top-p)
    ↓
Min-P filtering         (--min-p)
    ↓
XTC                     (--xtc-probability — off by default, creative writing only)
    ↓
Temperature scaling     (--temp — applied last)
    ↓
Sample final token

# Override the full chain order with --samplers "top_k;top_p;temp" etc.