imjasonh/terraform-playground
1
0
Fork 0
mirror of https://github.com/imjasonh/terraform-playground synced 2026-07-07 23:35:16 +00:00
Code Activity

Add K8s scheduler game simulation core and tests

Browse source 
Pure, UI-agnostic simulation:
- types: instance types (general/ssd/mem/gpu/spot node pools with labels & taints)
  and Deployment-like app templates (selectors, tolerations, hard/soft pod
  anti-affinity, replica caps, lifetimes).
- scheduler: kube-scheduler-style predicate filtering (resources, nodeSelector,
  taints/tolerations, anti-affinity, cordon/ready) returning human-readable
  reasons, plus a MostAllocated bin-packing scoring phase.
- workload: seeded PRNG + steady/spike/gpu/chaos scenarios with bounded,
  Deployment-like arrival streams.
- engine: tick loop, node lifecycle (provision/cordon/drain/delete), scheduling,
  job/service lifetimes, scoring (utilization vs latency vs cost), and an
  auto-scheduler + responsive cluster autoscaler.
- tests: predicates, engine actions, autoscaler, determinism, plus soak &
  balance runs asserting no overcommit and a healthy steady state.

Co-authored-by: Jason Hall <imjasonh@users.noreply.github.com>
This commit is contained in:
Cursor Agent 2026-06-26 02:19:29 +00:00
parent 42fd405534
commit 94c2de4236
No known key found for this signature in database
5 changed files with 1211 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

413
k8s-scheduler-game/src/engine.js Normal file
View file

@ -0,0 +1,413 @@
// The simulation engine. Holds all game state, advances time one tick at a
// time, exposes the operator actions (add/cordon/drain/delete node, schedule
// pod), and computes the score. It is intentionally UI-agnostic: the UI reads
// `game.state` and calls these methods, then re-renders.
import { INSTANCE_TYPES, ZONES, TICKS_PER_SECOND, SLA_PENDING_TICKS } from "./types.js";
import { bestNodeFor, evaluateFit, summarizePendingReason } from "./scheduler.js";
import { SCENARIOS, makeRng, spawnForTick } from "./workload.js";
// --- Scoring weights -------------------------------------------------------
const UTIL_W = 30; // reward per tick at 100% cluster utilization
const PENDING_PEN = 1.5; // penalty per pending pod per tick (latency pressure)
const COST_PEN = 0.6; // multiplier applied to summed node $/hr each tick
const COMPLETE_BONUS = 8; // reward for finishing a job
const DRAIN_EVICT_PEN = 2; // graceful eviction (drain) penalty per pod
const FORCE_EVICT_PEN = 10; // hard eviction (delete live node) penalty per pod
const SLA_BREACH_PEN = 40; // one-time penalty when a pod blows its scheduling SLA
const MAX_EVENTS = 240;
export class Game {
constructor(scenarioId = "steady") {
this.reset(scenarioId);
}
reset(scenarioId = this.state?.scenarioId || "steady") {
const scenario = SCENARIOS[scenarioId];
this.scenario = scenario;
this.state = {
scenarioId,
tick: 0,
nodes: [],
pods: new Map(),
pendingIds: [],
nodeSeq: 0,
paused: true,
speed: 1,
autoSchedule: false,
autoScale: false,
events: [],
score: 0,
// running tallies for the score breakdown panel
breakdown: { util: 0, latency: 0, cost: 0, jobs: 0, sla: 0, disruption: 0 },
metrics: {
latencySum: 0,
latencyCount: 0,
scheduledTotal: 0,
completedTotal: 0,
retiredTotal: 0,
slaBreaches: 0,
evictions: 0,
spawnedTotal: 0,
},
lastUtil: 0,
scaleCooldown: 0,
};
this.rng = makeRng(scenario.seed);
for (const typeKey of scenario.startNodes) {
this.addNode(typeKey, undefined, /*instant*/ true);
}
this.log("info", `Scenario "${scenario.name}" loaded. Cluster ready.`);
return this.state;
}
// --- helpers -------------------------------------------------------------
nodeById(id) {
return this.state.nodes.find((n) => n.id === id) || null;
}
podById(id) {
return this.state.pods.get(id) || null;
}
podsOnNode(node) {
return node.podIds.map((id) => this.state.pods.get(id)).filter(Boolean);
}
pendingPods() {
return this.state.pendingIds.map((id) => this.state.pods.get(id)).filter(Boolean);
}
log(level, msg) {
this.state.events.push({ tick: this.state.tick, level, msg });
if (this.state.events.length > MAX_EVENTS) this.state.events.shift();
}
// --- node lifecycle ------------------------------------------------------
addNode(typeKey, zone, instant = false) {
const spec = INSTANCE_TYPES[typeKey];
if (!spec) throw new Error(`unknown instance type ${typeKey}`);
this.state.nodeSeq += 1;
const z = zone || ZONES[this.state.nodeSeq % ZONES.length];
const node = {
id: `node-${this.state.nodeSeq}`,
name: `node-${this.state.nodeSeq}`,
type: typeKey,
cpu: spec.cpu,
mem: spec.mem,
gpu: spec.gpu,
cost: spec.cost,
labels: { ...spec.labels, "topology.kubernetes.io/zone": z },
taints: spec.taints.map((t) => ({ ...t })),
status: instant ? "Ready" : "Provisioning",
provisioningTicksLeft: instant ? 0 : spec.bootTicks,
podIds: [],
idleTicks: 0,
createdTick: this.state.tick,
};
this.state.nodes.push(node);
if (!instant) this.log("info", `Provisioning ${node.name} (${typeKey}, ${z})…`);
return node;
}
cordon(nodeId) {
const node = this.nodeById(nodeId);
if (!node || node.status === "Provisioning") return;
if (node.status === "Cordoned") {
node.status = "Ready";
this.log("info", `Uncordoned ${node.name}.`);
} else {
node.status = "Cordoned";
this.log("warn", `Cordoned ${node.name} — marked unschedulable.`);
}
}
/** Graceful drain: cordon, then evict every pod back to the queue. */
drain(nodeId) {
const node = this.nodeById(nodeId);
if (!node) return;
const pods = this.podsOnNode(node);
for (const pod of pods) this.evictPod(pod, node, DRAIN_EVICT_PEN, "drain");
node.status = "Cordoned";
this.log("warn", `Drained ${node.name} (${pods.length} pod(s) rescheduled).`);
}
/** Terminate a node. Any still-running pods are forcibly evicted. */
deleteNode(nodeId) {
const node = this.nodeById(nodeId);
if (!node) return;
const pods = this.podsOnNode(node);
for (const pod of pods) this.evictPod(pod, node, FORCE_EVICT_PEN, "force-delete");
this.state.nodes = this.state.nodes.filter((n) => n.id !== nodeId);
const lvl = pods.length ? "error" : "info";
this.log(lvl, `Terminated ${node.name}${pods.length ? ` (force-killed ${pods.length} pod(s)!)` : ""}.`);
}
evictPod(pod, node, penalty, reason) {
node.podIds = node.podIds.filter((id) => id !== pod.id);
pod.status = "Pending";
pod.nodeId = null;
pod.scheduledTick = null;
pod.pendingTicks = 0;
pod.arrivalTick = this.state.tick; // fresh latency clock after disruption
pod.slaBreached = false;
this.state.pendingIds.push(pod.id);
this.state.score -= penalty;
this.state.breakdown.disruption -= penalty;
this.state.metrics.evictions += 1;
void reason;
}
// --- scheduling actions --------------------------------------------------
/**
* Attempt to bind a pod to a node. Returns { ok, reasons }.
*/
schedulePod(podId, nodeId) {
const pod = this.podById(podId);
const node = this.nodeById(nodeId);
if (!pod || !node) return { ok: false, reasons: ["pod or node not found"] };
if (pod.status !== "Pending") return { ok: false, reasons: ["pod is not pending"] };
const fit = evaluateFit(pod, node, this.podsOnNode(node));
if (!fit.ok) return fit;
pod.status = "Running";
pod.nodeId = node.id;
pod.scheduledTick = this.state.tick;
node.podIds.push(pod.id);
node.idleTicks = 0;
this.state.pendingIds = this.state.pendingIds.filter((id) => id !== podId);
const latency = pod.scheduledTick - pod.arrivalTick;
this.state.metrics.latencySum += latency;
this.state.metrics.latencyCount += 1;
this.state.metrics.scheduledTotal += 1;
return { ok: true, reasons: [] };
}
/** Place a single pending pod on its best feasible node, if any. */
autoPlaceOne(podId) {
const pod = this.podById(podId);
if (!pod || pod.status !== "Pending") return { ok: false, reasons: ["not pending"] };
const best = bestNodeFor(pod, this.schedulableNodes(), (nid) =>
this.podsOnNode(this.nodeById(nid))
);
if (!best) {
return { ok: false, reasons: [summarizePendingReason(pod, this.state.nodes, (nid) => this.podsOnNode(this.nodeById(nid)))] };
}
return this.schedulePod(podId, best.node.id);
}
schedulableNodes() {
return this.state.nodes.filter((n) => n.status === "Ready");
}
// --- automation ----------------------------------------------------------
runAutoScheduler() {
// Highest priority first, then oldest (FIFO) — like the real scheduler.
const queue = [...this.pendingPods()].sort(
(a, b) => b.priority - a.priority || a.arrivalTick - b.arrivalTick
);
for (const pod of queue) {
const best = bestNodeFor(pod, this.schedulableNodes(), (nid) =>
this.podsOnNode(this.nodeById(nid))
);
if (best) this.schedulePod(pod.id, best.node.id);
}
}
/** The instance type the autoscaler would provision to host this pod. */
scaleUpTypeForPod(pod) {
if (pod.gpu > 0) return "gpu-xlarge";
if (pod.nodeSelector && pod.nodeSelector.disktype === "ssd") {
return pod.mem > 8192 ? "mem-xlarge" : "ssd-large";
}
// batch jobs tolerate spot — grab cheap, interruptible capacity for them
if (pod.kind === "job" && (pod.tolerations || []).some((t) => t.key === "spot")) {
return "spot-medium";
}
return "general-large";
}
runAutoScaler() {
const s = this.state;
if (s.scaleCooldown > 0) s.scaleCooldown -= 1;
if (s.scaleCooldown === 0) {
const ready = this.schedulableNodes();
const unfittable = this.pendingPods().filter(
(pod) => !ready.some((n) => evaluateFit(pod, n, this.podsOnNode(n)).ok)
);
const provisioning = s.nodes.filter((n) => n.status === "Provisioning").length;
// ~8 generic pods fit on a fresh node; discount capacity already booting.
let budget = Math.min(3, Math.ceil(unfittable.length / 8) - provisioning);
if (unfittable.length > 0 && budget > 0) {
const added = [];
const usedSpecial = new Set();
// Highest priority first so scarce gpu/ssd workloads aren't starved.
const queue = [...unfittable].sort((a, b) => b.priority - a.priority);
for (const pod of queue) {
if (budget <= 0) break;
const type = this.scaleUpTypeForPod(pod);
if (type !== "general-large") {
// one specialized node hosts several such pods — don't over-add
if (usedSpecial.has(type)) continue;
usedSpecial.add(type);
}
this.addNode(type);
added.push(type);
budget -= 1;
}
if (added.length) {
this.log("info", `Autoscaler: scaling up (+${added.length}: ${[...new Set(added)].join(", ")}).`);
s.scaleCooldown = 3;
}
}
}
// Scale down: terminate a node that has been idle a while (keep at least one).
if (s.scaleCooldown === 0 && this.schedulableNodes().length > 1) {
const idle = this.state.nodes.find(
(n) => n.status === "Ready" && n.podIds.length === 0 && n.idleTicks > 20
);
if (idle) {
this.deleteNode(idle.id);
this.log("info", `Autoscaler: scaling down idle ${idle.name}.`);
s.scaleCooldown = 3;
}
}
}
// --- the main tick -------------------------------------------------------
tick() {
const s = this.state;
s.tick += 1;
// 1. node provisioning
for (const node of s.nodes) {
if (node.status === "Provisioning") {
node.provisioningTicksLeft -= 1;
if (node.provisioningTicksLeft <= 0) {
node.status = "Ready";
this.log("good", `${node.name} is now Ready.`);
}
}
}
// 2. spawn workload (respecting each app's replica cap)
const aliveByApp = {};
for (const p of s.pods.values()) {
if (p.status === "Pending" || p.status === "Running") {
aliveByApp[p.app] = (aliveByApp[p.app] || 0) + 1;
}
}
const spawned = spawnForTick(this.scenario, this.rng, s.tick, aliveByApp);
for (const pod of spawned) {
s.pods.set(pod.id, pod);
s.pendingIds.push(pod.id);
}
s.metrics.spawnedTotal += spawned.length;
// 3. automation
if (s.autoSchedule) this.runAutoScheduler();
if (s.autoScale) this.runAutoScaler();
// 4. advance running jobs, track idleness
for (const node of s.nodes) {
if (node.podIds.length === 0 && node.status === "Ready") node.idleTicks += 1;
else node.idleTicks = 0;
}
for (const pod of [...s.pods.values()]) {
if (pod.status === "Running" && pod.remainingTicks != null) {
pod.remainingTicks -= 1;
if (pod.remainingTicks <= 0) this.finishPod(pod);
}
}
// 5. pending accounting + SLA
let pendingCount = 0;
for (const id of s.pendingIds) {
const pod = s.pods.get(id);
if (!pod) continue;
pendingCount += 1;
pod.pendingTicks += 1;
if (!pod.slaBreached && pod.pendingTicks > SLA_PENDING_TICKS) {
pod.slaBreached = true;
s.metrics.slaBreaches += 1;
s.score -= SLA_BREACH_PEN;
s.breakdown.sla -= SLA_BREACH_PEN;
this.log("error", `SLA breach: ${pod.name} pending >${(SLA_PENDING_TICKS / TICKS_PER_SECOND).toFixed(0)}s.`);
}
}
// 6. scoring (utilization reward + latency/cost pressure)
const util = this.clusterUtilization();
s.lastUtil = util;
const utilDelta = util * UTIL_W;
const latDelta = pendingCount * PENDING_PEN;
const hourlyCost = this.hourlyCost();
const costDelta = hourlyCost * COST_PEN;
s.score += utilDelta - latDelta - costDelta;
s.breakdown.util += utilDelta;
s.breakdown.latency -= latDelta;
s.breakdown.cost -= costDelta;
return s;
}
/** A pod reached the end of its lifetime: free its resources. */
finishPod(pod) {
const node = this.nodeById(pod.nodeId);
if (node) node.podIds = node.podIds.filter((id) => id !== pod.id);
pod.status = "Completed";
this.state.pods.delete(pod.id);
if (pod.kind === "job") {
this.state.metrics.completedTotal += 1;
this.state.score += COMPLETE_BONUS;
this.state.breakdown.jobs += COMPLETE_BONUS;
this.log("good", `Job ${pod.name} completed and freed its resources.`);
} else {
// a service replica was rolled / scaled down — quietly frees capacity
this.state.metrics.retiredTotal += 1;
}
}
// --- metrics -------------------------------------------------------------
/** Average utilization (cpu+mem)/2 across nodes that are up (paying). */
clusterUtilization() {
const up = this.state.nodes.filter((n) =>
["Ready", "Cordoned", "Draining"].includes(n.status)
);
if (up.length === 0) return 0;
let sum = 0;
for (const node of up) {
const used = this.podsOnNode(node).reduce(
(a, p) => ({ cpu: a.cpu + p.cpu, mem: a.mem + p.mem }),
{ cpu: 0, mem: 0 }
);
const cpuFrac = node.cpu ? used.cpu / node.cpu : 0;
const memFrac = node.mem ? used.mem / node.mem : 0;
sum += (cpuFrac + memFrac) / 2;
}
return sum / up.length;
}
/** Sum of $/hr for every node currently powered on. */
hourlyCost() {
return this.state.nodes
.filter((n) => n.status !== "Terminating")
.reduce((s, n) => s + n.cost, 0);
}
avgLatencySeconds() {
const m = this.state.metrics;
if (m.latencyCount === 0) return 0;
return m.latencySum / m.latencyCount / TICKS_PER_SECOND;
}
runningCount() {
let n = 0;
for (const p of this.state.pods.values()) if (p.status === "Running") n += 1;
return n;
}
uptimeSeconds() {
return this.state.tick / TICKS_PER_SECOND;
}
}

166
k8s-scheduler-game/src/scheduler.js Normal file
View file

@ -0,0 +1,166 @@
// The scheduling "brain": predicate checks that mirror the kube-scheduler
// filtering + scoring phases. Everything here is pure so it can be unit tested
// and reused by both the manual UI and the auto-scheduler.
import { sumRequests } from "./types.js";
/**
* Does a set of tolerations tolerate a given taint?
* A toleration matches when the key matches (or operator Exists with no key),
* the value matches (or toleration has no value), and the effect matches
* (or toleration has no effect).
*/
export function tolerationMatches(toleration, taint) {
if (toleration.key && toleration.key !== taint.key) return false;
if (toleration.effect && toleration.effect !== taint.effect) return false;
if (toleration.operator === "Exists") return true;
if (toleration.value === undefined) return true; // treat missing value as wildcard
return toleration.value === taint.value;
}
export function tolerates(tolerations, taint) {
return (tolerations || []).some((t) => tolerationMatches(t, taint));
}
/** Every key/value in selector must be present in labels. */
export function selectorMatches(selector, labels) {
for (const [k, v] of Object.entries(selector || {})) {
if (labels[k] !== v) return false;
}
return true;
}
/** Resources currently requested by the pods on a node. */
export function nodeAllocation(nodePods) {
return sumRequests(nodePods);
}
/** Free capacity on a node given the pods running on it. */
export function freeResources(node, nodePods) {
const used = nodeAllocation(nodePods);
return {
cpu: node.cpu - used.cpu,
mem: node.mem - used.mem,
gpu: node.gpu - used.gpu,
};
}
/**
* Run all scheduling predicates for placing `pod` on `node`, where `nodePods`
* are the pods already running on that node.
*
* Returns { ok, reasons } where reasons is an array of short, k8s-flavored
* strings describing every failing predicate (empty when ok === true).
*
* Set opts.ignoreState to evaluate as if the node were Ready (used by the
* autoscaler when sizing brand-new nodes).
*/
export function evaluateFit(pod, node, nodePods, opts = {}) {
const reasons = [];
if (!opts.ignoreState && node.status !== "Ready") {
if (node.status === "Provisioning") reasons.push("node is still provisioning");
else if (node.status === "Cordoned") reasons.push("node is cordoned (unschedulable)");
else if (node.status === "Draining") reasons.push("node is draining");
else reasons.push(`node is ${node.status.toLowerCase()}`);
}
// Taints / tolerations (NoSchedule effect filters the node out).
for (const taint of node.taints || []) {
if (taint.effect === "NoSchedule" && !tolerates(pod.tolerations, taint)) {
reasons.push(`untolerated taint {${taint.key}=${taint.value}}`);
}
}
// Required node affinity / nodeSelector.
for (const [k, v] of Object.entries(pod.nodeSelector || {})) {
if (node.labels[k] !== v) {
reasons.push(`didn't match selector {${k}=${v}}`);
}
}
// Pod anti-affinity (hostname topology): no two same-app pods per node.
if (pod.antiAffinity && nodePods.some((p) => p.app === pod.app)) {
reasons.push(`anti-affinity: ${pod.app} already on node`);
}
// Resource fit (the filtering phase's NodeResourcesFit).
const free = freeResources(node, nodePods);
if (free.cpu < pod.cpu) {
reasons.push(`insufficient cpu (free ${free.cpu}m, need ${pod.cpu}m)`);
}
if (free.mem < pod.mem) {
reasons.push(`insufficient memory (free ${free.mem}Mi, need ${pod.mem}Mi)`);
}
if (pod.gpu > 0 && free.gpu < pod.gpu) {
reasons.push(`insufficient nvidia.com/gpu (free ${free.gpu}, need ${pod.gpu})`);
}
return { ok: reasons.length === 0, reasons };
}
/**
* Score a feasible node for a pod (the scoring phase). Higher is better.
* Uses a MostAllocated/bin-packing strategy so the cluster stays tightly
* packed, with a small bonus for honoring the pod's preferred zone.
*/
export function scoreNode(pod, node, nodePods) {
const used = nodeAllocation(nodePods);
const cpuFrac = (used.cpu + pod.cpu) / node.cpu;
const memFrac = (used.mem + pod.mem) / node.mem;
let score = ((cpuFrac + memFrac) / 2) * 100;
if (pod.preferredZone && node.labels["topology.kubernetes.io/zone"] === pod.preferredZone) {
score += 8;
}
// Soft (preferred) pod anti-affinity: discourage co-locating same-app replicas.
if (pod.softAntiAffinity && nodePods.some((p) => p.app === pod.app)) {
score -= 18;
}
// Gently prefer cheaper nodes when packing is otherwise equal.
score -= node.cost * 2;
// GPU nodes are scarce: avoid burning them on non-GPU pods.
if (pod.gpu === 0 && node.gpu > 0) score -= 25;
return score;
}
/**
* Pick the best feasible node for a pod.
* @param pod the pod to place
* @param nodes array of node objects
* @param podsByNode function(nodeId) -> pod[] running on that node
* @returns { node, score } or null when nothing fits.
*/
export function bestNodeFor(pod, nodes, podsByNode) {
let best = null;
for (const node of nodes) {
const nodePods = podsByNode(node.id);
const fit = evaluateFit(pod, node, nodePods);
if (!fit.ok) continue;
const score = scoreNode(pod, node, nodePods);
if (!best || score > best.score) best = { node, score };
}
return best;
}
/**
* Aggregate the reasons across all nodes into the single most common blocker,
* so the UI/event log can explain why a pod is stuck Pending.
*/
export function summarizePendingReason(pod, nodes, podsByNode) {
if (nodes.length === 0) return "no nodes in cluster";
const counts = new Map();
for (const node of nodes) {
const fit = evaluateFit(pod, node, podsByNode(node.id));
for (const r of fit.reasons) {
// Normalize the dynamic numbers out of resource messages for tallying.
const key = r.replace(/\d+/g, "N");
counts.set(key, (counts.get(key) || 0) + 1);
}
}
let top = null;
for (const [k, c] of counts) {
if (!top || c > top.c) top = { k, c };
}
return top ? top.k : "unschedulable";
}

265
k8s-scheduler-game/src/types.js Normal file
View file

@ -0,0 +1,265 @@
// Domain model for the Kubescheduler game.
//
// All CPU values are in millicores (m); 1000m == 1 vCPU.
// All memory values are in MiB; 1024 MiB == 1 GiB.
// These are the same units real Kubernetes resource requests use.
export const TICKS_PER_SECOND = 4; // simulation ticks per real second at 1x speed
export const SLA_PENDING_TICKS = 40; // a pod pending longer than this breaches SLA
/** Availability zones a node can be placed in. */
export const ZONES = ["us-east-1a", "us-east-1b", "us-east-1c"];
/**
* Node instance types ("node pools"). Each carries baked-in labels and taints,
* exactly like a managed node group in EKS/GKE. cost is a relative $/hour figure
* used purely for scoring.
*/
export const INSTANCE_TYPES = {
"general-medium": {
key: "general-medium",
family: "general",
cpu: 4000,
mem: 8192,
gpu: 0,
cost: 0.16,
bootTicks: 8,
labels: { "node.kubernetes.io/instance-type": "general-medium", disktype: "hdd" },
taints: [],
},
"general-large": {
key: "general-large",
family: "general",
cpu: 8000,
mem: 16384,
gpu: 0,
cost: 0.32,
bootTicks: 10,
labels: { "node.kubernetes.io/instance-type": "general-large", disktype: "hdd" },
taints: [],
},
"ssd-large": {
key: "ssd-large",
family: "ssd",
cpu: 8000,
mem: 16384,
gpu: 0,
cost: 0.42,
bootTicks: 10,
labels: { "node.kubernetes.io/instance-type": "ssd-large", disktype: "ssd" },
taints: [],
},
"mem-xlarge": {
key: "mem-xlarge",
family: "mem",
cpu: 8000,
mem: 65536,
gpu: 0,
cost: 0.55,
bootTicks: 12,
labels: { "node.kubernetes.io/instance-type": "mem-xlarge", disktype: "ssd" },
taints: [],
},
"gpu-xlarge": {
key: "gpu-xlarge",
family: "gpu",
cpu: 8000,
mem: 32768,
gpu: 4,
cost: 2.4,
bootTicks: 16,
labels: {
"node.kubernetes.io/instance-type": "gpu-xlarge",
disktype: "ssd",
accelerator: "nvidia-t4",
},
taints: [{ key: "nvidia.com/gpu", value: "present", effect: "NoSchedule" }],
},
"spot-medium": {
key: "spot-medium",
family: "spot",
cpu: 4000,
mem: 8192,
gpu: 0,
cost: 0.05,
bootTicks: 6,
labels: { "node.kubernetes.io/instance-type": "spot-medium", disktype: "hdd" },
taints: [{ key: "spot", value: "true", effect: "NoSchedule" }],
},
};
/** Per-app color used throughout the UI. */
export const APP_COLORS = {
frontend: "#38bdf8",
api: "#34d399",
cache: "#f472b6",
postgres: "#a78bfa",
batch: "#fbbf24",
"ml-train": "#fb7185",
};
/**
* Deployment / workload templates. Pods are minted from these.
* - kind "service": long-running, never completes on its own.
* - kind "job": runs for a bounded lifetime then completes and frees resources.
* - nodeSelector: hard label match (required node affinity).
* - tolerations: taints this pod can land on.
* - antiAffinity: HARD pod anti-affinity — two pods of the same app may never
* share a node (hostname topology). Only used on low-volume apps, since it
* caps the app at one replica per node.
* - softAntiAffinity: PREFERRED spread — influences scoring (the scheduler
* tries to spread replicas across nodes) but never blocks placement.
* - maxReplicas: like a Deployment's replica count — the workload generator
* won't mint a new pod for an app already at this many live replicas. This
* bounds the cluster and keeps hard anti-affinity satisfiable.
* - preferredZone: soft affinity used only for scoring/tie-breaks.
* - lifetime: [min, max] ticks the pod runs before it finishes. Services get
* long, variable lifetimes (think rollouts / scaling churn) so the cluster
* reaches a bounded steady state; jobs are short.
*/
export const APP_TEMPLATES = {
frontend: {
app: "frontend",
kind: "service",
cpu: 250,
mem: 256,
gpu: 0,
nodeSelector: {},
tolerations: [],
antiAffinity: false,
softAntiAffinity: true,
maxReplicas: 26,
priority: 100,
lifetime: [120, 300],
weight: 7,
},
api: {
app: "api",
kind: "service",
cpu: 500,
mem: 512,
gpu: 0,
nodeSelector: {},
tolerations: [],
antiAffinity: false,
softAntiAffinity: true,
maxReplicas: 18,
priority: 200,
lifetime: [140, 340],
weight: 6,
},
cache: {
app: "cache",
kind: "service",
cpu: 500,
mem: 2048,
gpu: 0,
nodeSelector: { disktype: "ssd" },
tolerations: [],
antiAffinity: false,
softAntiAffinity: false,
maxReplicas: 10,
priority: 150,
lifetime: [160, 360],
weight: 3,
},
postgres: {
app: "postgres",
kind: "service",
cpu: 1000,
mem: 6144,
gpu: 0,
nodeSelector: { disktype: "ssd" },
tolerations: [],
antiAffinity: true,
softAntiAffinity: false,
maxReplicas: 4,
priority: 400,
lifetime: [160, 300],
weight: 1,
},
batch: {
app: "batch",
kind: "job",
cpu: 1000,
mem: 1024,
gpu: 0,
nodeSelector: {},
tolerations: [{ key: "spot", value: "true", effect: "NoSchedule" }],
antiAffinity: false,
softAntiAffinity: false,
maxReplicas: 28,
priority: 50,
lifetime: [30, 90],
weight: 5,
},
"ml-train": {
app: "ml-train",
kind: "job",
cpu: 2000,
mem: 8192,
gpu: 1,
nodeSelector: { accelerator: "nvidia-t4" },
tolerations: [{ key: "nvidia.com/gpu", value: "present", effect: "NoSchedule" }],
antiAffinity: false,
softAntiAffinity: false,
maxReplicas: 6,
priority: 300,
lifetime: [40, 120],
weight: 2,
},
};
// ---------------------------------------------------------------------------
// Small pure helpers shared across modules.
// ---------------------------------------------------------------------------
/** Sum the resource requests of a list of pod objects. */
export function sumRequests(pods) {
return pods.reduce(
(acc, p) => {
acc.cpu += p.cpu;
acc.mem += p.mem;
acc.gpu += p.gpu;
return acc;
},
{ cpu: 0, mem: 0, gpu: 0 }
);
}
/** Capacity object for a node from its instance type. */
export function nodeCapacity(node) {
return { cpu: node.cpu, mem: node.mem, gpu: node.gpu };
}
/** True if a node is currently able to accept newly scheduled pods. */
export function isSchedulable(node) {
return node.status === "Ready";
}
/** Format millicores as a friendly core string. */
export function fmtCpu(m) {
if (m >= 1000) return `${(m / 1000).toFixed(m % 1000 === 0 ? 0 : 1)}`;
return `${(m / 1000).toFixed(2)}`;
}
/** Format MiB as Gi/Mi. */
export function fmtMem(mib) {
if (mib >= 1024) return `${(mib / 1024).toFixed(mib % 1024 === 0 ? 0 : 1)}Gi`;
return `${mib}Mi`;
}
let _seq = 0;
/** Monotonic id generator. */
export function nextId(prefix) {
_seq += 1;
return `${prefix}-${_seq.toString(36)}`;
}
/** Short random suffix that looks like a real replica hash. */
export function randHash(rng, n = 5) {
const chars = "0123456789abcdefghijklmnopqrstuvwxyz";
let s = "";
for (let i = 0; i < n; i++) s += chars[Math.floor(rng() * chars.length)];
return s;
}

148
k8s-scheduler-game/src/workload.js Normal file
View file

@ -0,0 +1,148 @@
// Workload generation. Each scenario describes a starting cluster and an
// arrival process that mints pods from the APP_TEMPLATES over time.
import { APP_TEMPLATES, APP_COLORS, ZONES, nextId, randHash } from "./types.js";
/** Deterministic, seedable PRNG (mulberry32) so runs are reproducible. */
export function makeRng(seed) {
let a = seed >>> 0;
return function () {
a |= 0;
a = (a + 0x6d2b79f5) | 0;
let t = Math.imul(a ^ (a >>> 15), 1 | a);
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}
function pick(rng, arr) {
return arr[Math.floor(rng() * arr.length)];
}
function randInt(rng, [lo, hi]) {
return lo + Math.floor(rng() * (hi - lo + 1));
}
/** Weighted choice from a {key: weight} map. */
function weightedPick(rng, weights) {
const entries = Object.entries(weights).filter(([, w]) => w > 0);
const total = entries.reduce((s, [, w]) => s + w, 0);
let r = rng() * total;
for (const [k, w] of entries) {
r -= w;
if (r <= 0) return k;
}
return entries[entries.length - 1][0];
}
/** Mint a concrete pod from a template. */
export function createPod(appName, rng, tick) {
const t = APP_TEMPLATES[appName];
const pod = {
id: nextId("pod"),
name: `${t.app}-${randHash(rng, 4)}-${randHash(rng, 4)}`,
app: t.app,
color: APP_COLORS[t.app] || "#94a3b8",
kind: t.kind,
cpu: t.cpu,
mem: t.mem,
gpu: t.gpu,
nodeSelector: { ...t.nodeSelector },
tolerations: t.tolerations.map((x) => ({ ...x })),
antiAffinity: !!t.antiAffinity,
softAntiAffinity: !!t.softAntiAffinity,
priority: t.priority,
preferredZone: pick(rng, ZONES),
status: "Pending",
nodeId: null,
arrivalTick: tick,
scheduledTick: null,
pendingTicks: 0,
slaBreached: false,
// every pod runs for a bounded time; remaining ticks only tick down while Running.
remainingTicks: randInt(rng, t.lifetime),
};
return pod;
}
/**
* Scenario catalogue. `arrival(tick, rng)` returns the expected number of pods
* to spawn this tick (fractional values spawn probabilistically). `weights`
* may be a function of tick to create waves.
*/
export const SCENARIOS = {
steady: {
id: "steady",
name: "Steady State",
blurb: "A balanced, predictable workload. Great for learning the ropes.",
startNodes: ["general-large", "general-large", "ssd-large"],
seed: 1337,
arrival: () => 0.55,
weights: () => ({ frontend: 7, api: 6, cache: 3, postgres: 1, batch: 4 }),
},
spike: {
id: "spike",
name: "Traffic Spike",
blurb: "Calm baseline punctuated by big frontend/api surges. Scale up fast, scale down after.",
startNodes: ["general-large", "ssd-large"],
seed: 7,
arrival: (tick) => {
const phase = tick % 240;
return phase < 60 ? 1.8 : 0.25; // ~15s storm every ~60s
},
weights: (tick) => {
const storm = tick % 240 < 60;
return storm
? { frontend: 12, api: 9, cache: 2, postgres: 0, batch: 1 }
: { frontend: 4, api: 3, cache: 2, postgres: 1, batch: 3 };
},
},
gpu: {
id: "gpu",
name: "GPU Crunch",
blurb: "Steady services plus periodic ML training jobs that demand GPU nodes you must provision.",
startNodes: ["general-large", "ssd-large"],
seed: 99,
arrival: (tick) => (tick % 200 < 30 ? 1.4 : 0.5),
weights: (tick) => {
const burst = tick % 200 < 30;
return burst
? { frontend: 3, api: 3, cache: 1, postgres: 0, batch: 2, "ml-train": 6 }
: { frontend: 6, api: 5, cache: 2, postgres: 1, batch: 3, "ml-train": 0 };
},
},
chaos: {
id: "chaos",
name: "Production Chaos",
blurb: "Everything, everywhere, all at once. High churn across every workload type. Hard mode.",
startNodes: ["general-large", "ssd-large"],
seed: 42,
arrival: (tick) => 0.9 + (tick % 150 < 40 ? 1.2 : 0),
weights: () => ({ frontend: 8, api: 7, cache: 4, postgres: 2, batch: 6, "ml-train": 3 }),
},
};
/**
* Spawn pods for a single tick. Returns an array of new pod objects.
*
* `aliveByApp` maps app -> current live replica count; apps already at their
* maxReplicas are excluded so each app behaves like a bounded Deployment.
*/
export function spawnForTick(scenario, rng, tick, aliveByApp = {}) {
const rate = scenario.arrival(tick, rng);
const weights = { ...scenario.weights(tick) };
// Zero out apps that have hit their replica cap.
for (const app of Object.keys(weights)) {
const cap = APP_TEMPLATES[app]?.maxReplicas ?? Infinity;
if ((aliveByApp[app] || 0) >= cap) weights[app] = 0;
}
if (Object.values(weights).every((w) => w <= 0)) return [];
let count = Math.floor(rate);
if (rng() < rate - count) count += 1; // fractional remainder -> probabilistic
const pods = [];
for (let i = 0; i < count; i++) {
pods.push(createPod(weightedPick(rng, weights), rng, tick));
}
return pods;
}

219
k8s-scheduler-game/test/scheduler.test.mjs Normal file
View file

@ -0,0 +1,219 @@
// Pure-logic tests for the scheduler predicates and the engine. Run with:
// node --test k8s-scheduler-game/test/scheduler.test.mjs
import test from "node:test";
import assert from "node:assert/strict";
import { evaluateFit, tolerates, selectorMatches, freeResources, bestNodeFor } from "../src/scheduler.js";
import { Game } from "../src/engine.js";
import { INSTANCE_TYPES } from "../src/types.js";
import { createPod, makeRng, SCENARIOS, spawnForTick } from "../src/workload.js";
function nodeFrom(typeKey, zone = "us-east-1a") {
const s = INSTANCE_TYPES[typeKey];
return {
id: "n1",
status: "Ready",
cpu: s.cpu,
mem: s.mem,
gpu: s.gpu,
cost: s.cost,
labels: { ...s.labels, "topology.kubernetes.io/zone": zone },
taints: s.taints.map((t) => ({ ...t })),
podIds: [],
};
}
test("tolerations match taints correctly", () => {
const taint = { key: "spot", value: "true", effect: "NoSchedule" };
assert.equal(tolerates([{ key: "spot", value: "true", effect: "NoSchedule" }], taint), true);
assert.equal(tolerates([{ key: "spot", operator: "Exists" }], taint), true);
assert.equal(tolerates([{ key: "nvidia.com/gpu" }], taint), false);
assert.equal(tolerates([], taint), false);
});
test("selectorMatches enforces every label", () => {
assert.equal(selectorMatches({ disktype: "ssd" }, { disktype: "ssd", zone: "a" }), true);
assert.equal(selectorMatches({ disktype: "ssd" }, { disktype: "hdd" }), false);
assert.equal(selectorMatches({}, { anything: "x" }), true);
});
test("resource fit blocks oversized pods", () => {
const node = nodeFrom("general-medium"); // 4000m / 8192Mi
const big = { cpu: 5000, mem: 1024, gpu: 0, nodeSelector: {}, tolerations: [], antiAffinity: false, app: "x" };
const fit = evaluateFit(big, node, []);
assert.equal(fit.ok, false);
assert.ok(fit.reasons.some((r) => r.includes("insufficient cpu")));
});
test("free resources subtract running pods", () => {
const node = nodeFrom("general-large"); // 8000m / 16384
const pods = [
{ cpu: 1000, mem: 2048, gpu: 0 },
{ cpu: 500, mem: 512, gpu: 0 },
];
const free = freeResources(node, pods);
assert.equal(free.cpu, 8000 - 1500);
assert.equal(free.mem, 16384 - 2560);
});
test("nodeSelector for ssd is enforced", () => {
const hdd = nodeFrom("general-large"); // disktype hdd
const ssd = nodeFrom("ssd-large"); // disktype ssd
const pod = { cpu: 500, mem: 512, gpu: 0, nodeSelector: { disktype: "ssd" }, tolerations: [], antiAffinity: false, app: "cache" };
assert.equal(evaluateFit(pod, hdd, []).ok, false);
assert.equal(evaluateFit(pod, ssd, []).ok, true);
});
test("gpu taint requires toleration and gpu capacity", () => {
const gpuNode = nodeFrom("gpu-xlarge");
const noTol = { cpu: 500, mem: 512, gpu: 0, nodeSelector: {}, tolerations: [], antiAffinity: false, app: "frontend" };
// frontend has no toleration -> blocked by taint
assert.equal(evaluateFit(noTol, gpuNode, []).ok, false);
const mlPod = {
cpu: 2000, mem: 8192, gpu: 1,
nodeSelector: { accelerator: "nvidia-t4" },
tolerations: [{ key: "nvidia.com/gpu", value: "present", effect: "NoSchedule" }],
antiAffinity: false, app: "ml-train",
};
assert.equal(evaluateFit(mlPod, gpuNode, []).ok, true);
});
test("anti-affinity prevents two same-app pods on one node", () => {
const node = nodeFrom("general-large");
const a = { cpu: 250, mem: 256, gpu: 0, nodeSelector: {}, tolerations: [], antiAffinity: true, app: "frontend" };
const b = { cpu: 250, mem: 256, gpu: 0, nodeSelector: {}, tolerations: [], antiAffinity: true, app: "frontend" };
assert.equal(evaluateFit(b, node, [a]).ok, false);
assert.ok(evaluateFit(b, node, [a]).reasons.some((r) => r.includes("anti-affinity")));
});
test("cordoned nodes are unschedulable", () => {
const node = nodeFrom("general-large");
node.status = "Cordoned";
const pod = { cpu: 250, mem: 256, gpu: 0, nodeSelector: {}, tolerations: [], antiAffinity: false, app: "frontend" };
assert.equal(evaluateFit(pod, node, []).ok, false);
});
test("bestNodeFor packs onto the fuller feasible node", () => {
const n1 = { ...nodeFrom("general-large"), id: "n1" };
const n2 = { ...nodeFrom("general-large"), id: "n2" };
const existing = { cpu: 4000, mem: 4096, gpu: 0, app: "api" };
const podsByNode = (id) => (id === "n1" ? [existing] : []);
const pod = { cpu: 500, mem: 512, gpu: 0, nodeSelector: {}, tolerations: [], antiAffinity: false, app: "frontend" };
const best = bestNodeFor(pod, [n1, n2], podsByNode);
assert.equal(best.node.id, "n1"); // MostAllocated -> fill n1 first
});
test("engine schedules a pod and records latency", () => {
const game = new Game("steady");
game.state.tick = 5;
const pod = createPod("frontend", makeRng(1), 2);
game.state.pods.set(pod.id, pod);
game.state.pendingIds.push(pod.id);
const node = game.schedulableNodes()[0];
const res = game.schedulePod(pod.id, node.id);
assert.equal(res.ok, true);
assert.equal(pod.status, "Running");
assert.equal(game.state.metrics.latencySum, 3); // 5 - 2
});
test("draining a node evicts its pods back to pending with a penalty", () => {
const game = new Game("steady");
const pod = createPod("api", makeRng(2), 0);
game.state.pods.set(pod.id, pod);
game.state.pendingIds.push(pod.id);
const node = game.schedulableNodes()[0];
game.schedulePod(pod.id, node.id);
const before = game.state.score;
game.drain(node.id);
assert.equal(pod.status, "Pending");
assert.equal(node.status, "Cordoned");
assert.ok(game.state.score < before);
assert.ok(game.state.pendingIds.includes(pod.id));
});
test("jobs complete and free resources after their lifetime", () => {
const game = new Game("steady");
const pod = createPod("batch", makeRng(3), 0);
pod.remainingTicks = 2;
game.state.pods.set(pod.id, pod);
game.state.pendingIds.push(pod.id);
const node = game.schedulableNodes()[0];
game.schedulePod(pod.id, node.id);
assert.ok(node.podIds.includes(pod.id));
game.tick();
game.tick();
game.tick();
assert.equal(game.state.pods.has(pod.id), false);
assert.equal(node.podIds.includes(pod.id), false);
assert.ok(game.state.metrics.completedTotal >= 1);
});
test("autoscaler provisions a GPU node when ml-train is stuck pending", () => {
const game = new Game("steady");
game.state.autoScale = true;
game.state.autoSchedule = true;
const pod = createPod("ml-train", makeRng(4), 0);
game.state.pods.set(pod.id, pod);
game.state.pendingIds.push(pod.id);
// No GPU node exists initially; autoscaler should add one within a few ticks.
let addedGpu = false;
for (let i = 0; i < 40 && !addedGpu; i++) {
game.tick();
addedGpu = game.state.nodes.some((n) => n.type === "gpu-xlarge");
}
assert.equal(addedGpu, true);
});
test("workload generator is deterministic for a seed", () => {
const a = spawnForTick(SCENARIOS.chaos, makeRng(42), 10).map((p) => p.app);
const b = spawnForTick(SCENARIOS.chaos, makeRng(42), 10).map((p) => p.app);
assert.deepEqual(a, b);
});
test("soak: every scenario runs 1200 ticks under full automation without overcommit", () => {
for (const id of Object.keys(SCENARIOS)) {
const game = new Game(id);
game.state.autoSchedule = true;
game.state.autoScale = true;
for (let i = 0; i < 1200; i++) game.tick();
assert.ok(Number.isFinite(game.state.score), `${id} score is finite`);
for (const node of game.state.nodes) {
const used = game.podsOnNode(node).reduce(
(a, p) => ({ cpu: a.cpu + p.cpu, mem: a.mem + p.mem, gpu: a.gpu + p.gpu }),
{ cpu: 0, mem: 0, gpu: 0 }
);
assert.ok(used.cpu <= node.cpu, `${id}: cpu overcommit on ${node.id}`);
assert.ok(used.mem <= node.mem, `${id}: mem overcommit on ${node.id}`);
assert.ok(used.gpu <= node.gpu, `${id}: gpu overcommit on ${node.id}`);
}
}
});
test("full automation keeps every scenario healthy (bounded queue, good utilization)", () => {
for (const id of Object.keys(SCENARIOS)) {
const game = new Game(id);
game.state.autoSchedule = true;
game.state.autoScale = true;
for (let i = 0; i < 1000; i++) game.tick();
const pending = game.state.pendingIds.length;
const util = game.clusterUtilization();
// Generous bounds: the auto policy should clearly keep up, not melt down.
assert.ok(pending < 35, `${id}: pending=${pending}`);
assert.ok(util > 0.35, `${id}: utilization=${util.toFixed(2)}`);
assert.ok(game.state.metrics.slaBreaches < 40, `${id}: sla=${game.state.metrics.slaBreaches}`);
assert.ok(game.state.score > 0, `${id}: score=${Math.round(game.state.score)}`);
}
});
test("finite pod lifetimes keep the cluster population bounded", () => {
const game = new Game("steady");
game.state.autoSchedule = true;
game.state.autoScale = true;
for (let i = 0; i < 1200; i++) game.tick();
// Services retire, so the total live pod count reaches a steady state instead
// of growing without bound (this also keeps the tick loop cheap).
assert.ok(game.state.pods.size < 400, `live pods=${game.state.pods.size}`);
assert.ok(game.state.metrics.retiredTotal > 0, "services should retire over time");
assert.ok(game.state.pendingIds.length < 60, `pending=${game.state.pendingIds.length}`);
});