— Examples (sizing, axis choice, K8s)

Back to GPUResizing.md. Playbook: 10_cutover_playbook.md.


# --- 1) Binding resource: what would a resize even fix? ---
def binder(m: dict) -> str:
    """m = live metrics for ONE pool (generation XOR embed XOR ocr)."""
    if m["oom_events"] > 0 or m["kv_preempt_per_min"] > 1:
        return "vram_or_kv"          # S (ctx/seqs/quant) or V (more VRAM) or W (TP)
    if m["queue_depth"] > 8 and m["ttft_p95_ms"] > m["ttft_slo_ms"]:
        if m["tokens_per_s_per_gpu"] < m["expected_tokens_per_s"] * 0.5:
            return "slow_sku"        # V (newer gen / bandwidth) after confirming config
        return "need_replicas"       # H
    if m["gpu_util"] < 0.3 and m["kv_util"] < 0.3 and m["replicas"] > m["min_replicas"]:
        return "overprovisioned"     # scale H down
    return "hold"


# --- 2) VRAM → concurrency ---
def kv_bytes(layers, kv_heads, head_dim, ctx, seqs, kv_b=2) -> float:
    return 2 * layers * kv_heads * head_dim * kv_b * ctx * seqs  # K+V

def shape_ok(vram_gb, weights_gb, layers, kv_heads, head_dim, ctx, seqs, headroom=0.85, overhead_gb=2):
    kv_gb = kv_bytes(layers, kv_heads, head_dim, ctx, seqs) / 1e9
    used = weights_gb + kv_gb + overhead_gb
    return used <= vram_gb * headroom, {"used_gb": round(used, 1), "kv_gb": round(kv_gb, 1)}

# Llama-3.1-8B FP16 ~16 GB weights. Concurrent 64 @ 8k ctx on 24 GB vs 40 GB vs 80 GB:
# shape_ok(24, 16, 32, 8, 128, 8192, 64)  → False  → need quant, fewer seqs, or V
# shape_ok(40, 16, 32, 8, 128, 8192, 64)  → True   → H on this SKU
# 70B AWQ ~40 GB weights: single 40 GB card has almost no KV → V to 80 GB or W (TP=2)


# --- 3) Pick axis ---
def resize_plan(fit_current, fit_quantized, weights_gt_one_card, binder_id, peak_tps, tps_per_gpu):
    if binder_id == "overprovisioned":
        return {"axis": "H", "delta": "scale_down"}
    if not fit_current and fit_quantized:
        return {"axis": "S", "action": "quantise_or_cap_ctx_and_re_eval"}
    if weights_gt_one_card:
        return {"axis": "W", "action": "tensor_parallel", "tp": 2}  # or 4/8; prefer min TP that fits
    if binder_id == "need_replicas":
        return {"axis": "H", "replicas": -(-peak_tps // tps_per_gpu) + 1}  # ceil + N+1
    if binder_id in {"vram_or_kv", "slow_sku"}:
        return {"axis": "V", "action": "next_sku_same_or_less_TP"}
    return {"axis": "hold"}


# --- 4) Autoscale signal (thresholds are SKU-specific) ---
def hpa_signal(queue_depth, ttft_p95, slo_ms, q_target=4):
    if queue_depth > q_target * 2 or ttft_p95 > slo_ms:
        return "up"
    if queue_depth < 1:
        return "down_candidate"  # still require cooldown + minReplicas + not during peak
    return "hold"


# --- 5) Cutover gates ---
def canary_abort(canary, baseline, eval_delta, ttft_budget=1.15, err_budget=1.5):
    if canary["ttft_p95"] > baseline["ttft_p95"] * ttft_budget:
        return "abort_latency"
    if canary["error_rate"] > max(baseline["error_rate"] * err_budget, 0.01):
        return "abort_errors"
    if eval_delta["golden_drop"] > 0.02:
        return "abort_quality"
    return "continue"

Kubernetes sketch (generation pool exclusive GPU; resize = new node group + new deployment):

# Do not patch nvidia.com/gpu from 1 → 2 on a live Deployment of a TP=1 model
# and expect a resize. That schedules a new pod; TP flags must match.
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-gen-llama8b
spec:
  replicas: 3                          # H-axis; PDB minAvailable: 2
  template:
    spec:
      nodeSelector:
        pool: gpu-gen-l4               # SKU-pinned node group (V-axis = new pool)
      terminationGracePeriodSeconds: 180
      containers:
        - name: vllm
          resources:
            limits:
              nvidia.com/gpu: 1        # W-axis: 2 only if --tensor-parallel-size=2
          readinessProbe:
            httpGet: { path: /v1/models, port: 8000 }
            failureThreshold: 3
            periodSeconds: 10          # fail unready until weights are in VRAM