diff --git a/k8s-scheduler-game/src/engine.js b/k8s-scheduler-game/src/engine.js new file mode 100644 index 0000000..fff3902 --- /dev/null +++ b/k8s-scheduler-game/src/engine.js @@ -0,0 +1,413 @@ +// The simulation engine. Holds all game state, advances time one tick at a +// time, exposes the operator actions (add/cordon/drain/delete node, schedule +// pod), and computes the score. It is intentionally UI-agnostic: the UI reads +// `game.state` and calls these methods, then re-renders. + +import { INSTANCE_TYPES, ZONES, TICKS_PER_SECOND, SLA_PENDING_TICKS } from "./types.js"; +import { bestNodeFor, evaluateFit, summarizePendingReason } from "./scheduler.js"; +import { SCENARIOS, makeRng, spawnForTick } from "./workload.js"; + +// --- Scoring weights ------------------------------------------------------- +const UTIL_W = 30; // reward per tick at 100% cluster utilization +const PENDING_PEN = 1.5; // penalty per pending pod per tick (latency pressure) +const COST_PEN = 0.6; // multiplier applied to summed node $/hr each tick +const COMPLETE_BONUS = 8; // reward for finishing a job +const DRAIN_EVICT_PEN = 2; // graceful eviction (drain) penalty per pod +const FORCE_EVICT_PEN = 10; // hard eviction (delete live node) penalty per pod +const SLA_BREACH_PEN = 40; // one-time penalty when a pod blows its scheduling SLA +const MAX_EVENTS = 240; + +export class Game { + constructor(scenarioId = "steady") { + this.reset(scenarioId); + } + + reset(scenarioId = this.state?.scenarioId || "steady") { + const scenario = SCENARIOS[scenarioId]; + this.scenario = scenario; + this.state = { + scenarioId, + tick: 0, + nodes: [], + pods: new Map(), + pendingIds: [], + nodeSeq: 0, + paused: true, + speed: 1, + autoSchedule: false, + autoScale: false, + events: [], + score: 0, + // running tallies for the score breakdown panel + breakdown: { util: 0, latency: 0, cost: 0, jobs: 0, sla: 0, disruption: 0 }, + metrics: { + latencySum: 0, + latencyCount: 0, + scheduledTotal: 0, + completedTotal: 0, + retiredTotal: 0, + slaBreaches: 0, + evictions: 0, + spawnedTotal: 0, + }, + lastUtil: 0, + scaleCooldown: 0, + }; + this.rng = makeRng(scenario.seed); + for (const typeKey of scenario.startNodes) { + this.addNode(typeKey, undefined, /*instant*/ true); + } + this.log("info", `Scenario "${scenario.name}" loaded. Cluster ready.`); + return this.state; + } + + // --- helpers ------------------------------------------------------------- + nodeById(id) { + return this.state.nodes.find((n) => n.id === id) || null; + } + podById(id) { + return this.state.pods.get(id) || null; + } + podsOnNode(node) { + return node.podIds.map((id) => this.state.pods.get(id)).filter(Boolean); + } + pendingPods() { + return this.state.pendingIds.map((id) => this.state.pods.get(id)).filter(Boolean); + } + log(level, msg) { + this.state.events.push({ tick: this.state.tick, level, msg }); + if (this.state.events.length > MAX_EVENTS) this.state.events.shift(); + } + + // --- node lifecycle ------------------------------------------------------ + addNode(typeKey, zone, instant = false) { + const spec = INSTANCE_TYPES[typeKey]; + if (!spec) throw new Error(`unknown instance type ${typeKey}`); + this.state.nodeSeq += 1; + const z = zone || ZONES[this.state.nodeSeq % ZONES.length]; + const node = { + id: `node-${this.state.nodeSeq}`, + name: `node-${this.state.nodeSeq}`, + type: typeKey, + cpu: spec.cpu, + mem: spec.mem, + gpu: spec.gpu, + cost: spec.cost, + labels: { ...spec.labels, "topology.kubernetes.io/zone": z }, + taints: spec.taints.map((t) => ({ ...t })), + status: instant ? "Ready" : "Provisioning", + provisioningTicksLeft: instant ? 0 : spec.bootTicks, + podIds: [], + idleTicks: 0, + createdTick: this.state.tick, + }; + this.state.nodes.push(node); + if (!instant) this.log("info", `Provisioning ${node.name} (${typeKey}, ${z})…`); + return node; + } + + cordon(nodeId) { + const node = this.nodeById(nodeId); + if (!node || node.status === "Provisioning") return; + if (node.status === "Cordoned") { + node.status = "Ready"; + this.log("info", `Uncordoned ${node.name}.`); + } else { + node.status = "Cordoned"; + this.log("warn", `Cordoned ${node.name} — marked unschedulable.`); + } + } + + /** Graceful drain: cordon, then evict every pod back to the queue. */ + drain(nodeId) { + const node = this.nodeById(nodeId); + if (!node) return; + const pods = this.podsOnNode(node); + for (const pod of pods) this.evictPod(pod, node, DRAIN_EVICT_PEN, "drain"); + node.status = "Cordoned"; + this.log("warn", `Drained ${node.name} (${pods.length} pod(s) rescheduled).`); + } + + /** Terminate a node. Any still-running pods are forcibly evicted. */ + deleteNode(nodeId) { + const node = this.nodeById(nodeId); + if (!node) return; + const pods = this.podsOnNode(node); + for (const pod of pods) this.evictPod(pod, node, FORCE_EVICT_PEN, "force-delete"); + this.state.nodes = this.state.nodes.filter((n) => n.id !== nodeId); + const lvl = pods.length ? "error" : "info"; + this.log(lvl, `Terminated ${node.name}${pods.length ? ` (force-killed ${pods.length} pod(s)!)` : ""}.`); + } + + evictPod(pod, node, penalty, reason) { + node.podIds = node.podIds.filter((id) => id !== pod.id); + pod.status = "Pending"; + pod.nodeId = null; + pod.scheduledTick = null; + pod.pendingTicks = 0; + pod.arrivalTick = this.state.tick; // fresh latency clock after disruption + pod.slaBreached = false; + this.state.pendingIds.push(pod.id); + this.state.score -= penalty; + this.state.breakdown.disruption -= penalty; + this.state.metrics.evictions += 1; + void reason; + } + + // --- scheduling actions -------------------------------------------------- + /** + * Attempt to bind a pod to a node. Returns { ok, reasons }. + */ + schedulePod(podId, nodeId) { + const pod = this.podById(podId); + const node = this.nodeById(nodeId); + if (!pod || !node) return { ok: false, reasons: ["pod or node not found"] }; + if (pod.status !== "Pending") return { ok: false, reasons: ["pod is not pending"] }; + const fit = evaluateFit(pod, node, this.podsOnNode(node)); + if (!fit.ok) return fit; + + pod.status = "Running"; + pod.nodeId = node.id; + pod.scheduledTick = this.state.tick; + node.podIds.push(pod.id); + node.idleTicks = 0; + this.state.pendingIds = this.state.pendingIds.filter((id) => id !== podId); + + const latency = pod.scheduledTick - pod.arrivalTick; + this.state.metrics.latencySum += latency; + this.state.metrics.latencyCount += 1; + this.state.metrics.scheduledTotal += 1; + return { ok: true, reasons: [] }; + } + + /** Place a single pending pod on its best feasible node, if any. */ + autoPlaceOne(podId) { + const pod = this.podById(podId); + if (!pod || pod.status !== "Pending") return { ok: false, reasons: ["not pending"] }; + const best = bestNodeFor(pod, this.schedulableNodes(), (nid) => + this.podsOnNode(this.nodeById(nid)) + ); + if (!best) { + return { ok: false, reasons: [summarizePendingReason(pod, this.state.nodes, (nid) => this.podsOnNode(this.nodeById(nid)))] }; + } + return this.schedulePod(podId, best.node.id); + } + + schedulableNodes() { + return this.state.nodes.filter((n) => n.status === "Ready"); + } + + // --- automation ---------------------------------------------------------- + runAutoScheduler() { + // Highest priority first, then oldest (FIFO) — like the real scheduler. + const queue = [...this.pendingPods()].sort( + (a, b) => b.priority - a.priority || a.arrivalTick - b.arrivalTick + ); + for (const pod of queue) { + const best = bestNodeFor(pod, this.schedulableNodes(), (nid) => + this.podsOnNode(this.nodeById(nid)) + ); + if (best) this.schedulePod(pod.id, best.node.id); + } + } + + /** The instance type the autoscaler would provision to host this pod. */ + scaleUpTypeForPod(pod) { + if (pod.gpu > 0) return "gpu-xlarge"; + if (pod.nodeSelector && pod.nodeSelector.disktype === "ssd") { + return pod.mem > 8192 ? "mem-xlarge" : "ssd-large"; + } + // batch jobs tolerate spot — grab cheap, interruptible capacity for them + if (pod.kind === "job" && (pod.tolerations || []).some((t) => t.key === "spot")) { + return "spot-medium"; + } + return "general-large"; + } + + runAutoScaler() { + const s = this.state; + if (s.scaleCooldown > 0) s.scaleCooldown -= 1; + + if (s.scaleCooldown === 0) { + const ready = this.schedulableNodes(); + const unfittable = this.pendingPods().filter( + (pod) => !ready.some((n) => evaluateFit(pod, n, this.podsOnNode(n)).ok) + ); + const provisioning = s.nodes.filter((n) => n.status === "Provisioning").length; + // ~8 generic pods fit on a fresh node; discount capacity already booting. + let budget = Math.min(3, Math.ceil(unfittable.length / 8) - provisioning); + + if (unfittable.length > 0 && budget > 0) { + const added = []; + const usedSpecial = new Set(); + // Highest priority first so scarce gpu/ssd workloads aren't starved. + const queue = [...unfittable].sort((a, b) => b.priority - a.priority); + for (const pod of queue) { + if (budget <= 0) break; + const type = this.scaleUpTypeForPod(pod); + if (type !== "general-large") { + // one specialized node hosts several such pods — don't over-add + if (usedSpecial.has(type)) continue; + usedSpecial.add(type); + } + this.addNode(type); + added.push(type); + budget -= 1; + } + if (added.length) { + this.log("info", `Autoscaler: scaling up (+${added.length}: ${[...new Set(added)].join(", ")}).`); + s.scaleCooldown = 3; + } + } + } + + // Scale down: terminate a node that has been idle a while (keep at least one). + if (s.scaleCooldown === 0 && this.schedulableNodes().length > 1) { + const idle = this.state.nodes.find( + (n) => n.status === "Ready" && n.podIds.length === 0 && n.idleTicks > 20 + ); + if (idle) { + this.deleteNode(idle.id); + this.log("info", `Autoscaler: scaling down idle ${idle.name}.`); + s.scaleCooldown = 3; + } + } + } + + // --- the main tick ------------------------------------------------------- + tick() { + const s = this.state; + s.tick += 1; + + // 1. node provisioning + for (const node of s.nodes) { + if (node.status === "Provisioning") { + node.provisioningTicksLeft -= 1; + if (node.provisioningTicksLeft <= 0) { + node.status = "Ready"; + this.log("good", `${node.name} is now Ready.`); + } + } + } + + // 2. spawn workload (respecting each app's replica cap) + const aliveByApp = {}; + for (const p of s.pods.values()) { + if (p.status === "Pending" || p.status === "Running") { + aliveByApp[p.app] = (aliveByApp[p.app] || 0) + 1; + } + } + const spawned = spawnForTick(this.scenario, this.rng, s.tick, aliveByApp); + for (const pod of spawned) { + s.pods.set(pod.id, pod); + s.pendingIds.push(pod.id); + } + s.metrics.spawnedTotal += spawned.length; + + // 3. automation + if (s.autoSchedule) this.runAutoScheduler(); + if (s.autoScale) this.runAutoScaler(); + + // 4. advance running jobs, track idleness + for (const node of s.nodes) { + if (node.podIds.length === 0 && node.status === "Ready") node.idleTicks += 1; + else node.idleTicks = 0; + } + for (const pod of [...s.pods.values()]) { + if (pod.status === "Running" && pod.remainingTicks != null) { + pod.remainingTicks -= 1; + if (pod.remainingTicks <= 0) this.finishPod(pod); + } + } + + // 5. pending accounting + SLA + let pendingCount = 0; + for (const id of s.pendingIds) { + const pod = s.pods.get(id); + if (!pod) continue; + pendingCount += 1; + pod.pendingTicks += 1; + if (!pod.slaBreached && pod.pendingTicks > SLA_PENDING_TICKS) { + pod.slaBreached = true; + s.metrics.slaBreaches += 1; + s.score -= SLA_BREACH_PEN; + s.breakdown.sla -= SLA_BREACH_PEN; + this.log("error", `SLA breach: ${pod.name} pending >${(SLA_PENDING_TICKS / TICKS_PER_SECOND).toFixed(0)}s.`); + } + } + + // 6. scoring (utilization reward + latency/cost pressure) + const util = this.clusterUtilization(); + s.lastUtil = util; + const utilDelta = util * UTIL_W; + const latDelta = pendingCount * PENDING_PEN; + const hourlyCost = this.hourlyCost(); + const costDelta = hourlyCost * COST_PEN; + s.score += utilDelta - latDelta - costDelta; + s.breakdown.util += utilDelta; + s.breakdown.latency -= latDelta; + s.breakdown.cost -= costDelta; + + return s; + } + + /** A pod reached the end of its lifetime: free its resources. */ + finishPod(pod) { + const node = this.nodeById(pod.nodeId); + if (node) node.podIds = node.podIds.filter((id) => id !== pod.id); + pod.status = "Completed"; + this.state.pods.delete(pod.id); + if (pod.kind === "job") { + this.state.metrics.completedTotal += 1; + this.state.score += COMPLETE_BONUS; + this.state.breakdown.jobs += COMPLETE_BONUS; + this.log("good", `Job ${pod.name} completed and freed its resources.`); + } else { + // a service replica was rolled / scaled down — quietly frees capacity + this.state.metrics.retiredTotal += 1; + } + } + + // --- metrics ------------------------------------------------------------- + /** Average utilization (cpu+mem)/2 across nodes that are up (paying). */ + clusterUtilization() { + const up = this.state.nodes.filter((n) => + ["Ready", "Cordoned", "Draining"].includes(n.status) + ); + if (up.length === 0) return 0; + let sum = 0; + for (const node of up) { + const used = this.podsOnNode(node).reduce( + (a, p) => ({ cpu: a.cpu + p.cpu, mem: a.mem + p.mem }), + { cpu: 0, mem: 0 } + ); + const cpuFrac = node.cpu ? used.cpu / node.cpu : 0; + const memFrac = node.mem ? used.mem / node.mem : 0; + sum += (cpuFrac + memFrac) / 2; + } + return sum / up.length; + } + + /** Sum of $/hr for every node currently powered on. */ + hourlyCost() { + return this.state.nodes + .filter((n) => n.status !== "Terminating") + .reduce((s, n) => s + n.cost, 0); + } + + avgLatencySeconds() { + const m = this.state.metrics; + if (m.latencyCount === 0) return 0; + return m.latencySum / m.latencyCount / TICKS_PER_SECOND; + } + + runningCount() { + let n = 0; + for (const p of this.state.pods.values()) if (p.status === "Running") n += 1; + return n; + } + + uptimeSeconds() { + return this.state.tick / TICKS_PER_SECOND; + } +} diff --git a/k8s-scheduler-game/src/scheduler.js b/k8s-scheduler-game/src/scheduler.js new file mode 100644 index 0000000..221db1e --- /dev/null +++ b/k8s-scheduler-game/src/scheduler.js @@ -0,0 +1,166 @@ +// The scheduling "brain": predicate checks that mirror the kube-scheduler +// filtering + scoring phases. Everything here is pure so it can be unit tested +// and reused by both the manual UI and the auto-scheduler. + +import { sumRequests } from "./types.js"; + +/** + * Does a set of tolerations tolerate a given taint? + * A toleration matches when the key matches (or operator Exists with no key), + * the value matches (or toleration has no value), and the effect matches + * (or toleration has no effect). + */ +export function tolerationMatches(toleration, taint) { + if (toleration.key && toleration.key !== taint.key) return false; + if (toleration.effect && toleration.effect !== taint.effect) return false; + if (toleration.operator === "Exists") return true; + if (toleration.value === undefined) return true; // treat missing value as wildcard + return toleration.value === taint.value; +} + +export function tolerates(tolerations, taint) { + return (tolerations || []).some((t) => tolerationMatches(t, taint)); +} + +/** Every key/value in selector must be present in labels. */ +export function selectorMatches(selector, labels) { + for (const [k, v] of Object.entries(selector || {})) { + if (labels[k] !== v) return false; + } + return true; +} + +/** Resources currently requested by the pods on a node. */ +export function nodeAllocation(nodePods) { + return sumRequests(nodePods); +} + +/** Free capacity on a node given the pods running on it. */ +export function freeResources(node, nodePods) { + const used = nodeAllocation(nodePods); + return { + cpu: node.cpu - used.cpu, + mem: node.mem - used.mem, + gpu: node.gpu - used.gpu, + }; +} + +/** + * Run all scheduling predicates for placing `pod` on `node`, where `nodePods` + * are the pods already running on that node. + * + * Returns { ok, reasons } where reasons is an array of short, k8s-flavored + * strings describing every failing predicate (empty when ok === true). + * + * Set opts.ignoreState to evaluate as if the node were Ready (used by the + * autoscaler when sizing brand-new nodes). + */ +export function evaluateFit(pod, node, nodePods, opts = {}) { + const reasons = []; + + if (!opts.ignoreState && node.status !== "Ready") { + if (node.status === "Provisioning") reasons.push("node is still provisioning"); + else if (node.status === "Cordoned") reasons.push("node is cordoned (unschedulable)"); + else if (node.status === "Draining") reasons.push("node is draining"); + else reasons.push(`node is ${node.status.toLowerCase()}`); + } + + // Taints / tolerations (NoSchedule effect filters the node out). + for (const taint of node.taints || []) { + if (taint.effect === "NoSchedule" && !tolerates(pod.tolerations, taint)) { + reasons.push(`untolerated taint {${taint.key}=${taint.value}}`); + } + } + + // Required node affinity / nodeSelector. + for (const [k, v] of Object.entries(pod.nodeSelector || {})) { + if (node.labels[k] !== v) { + reasons.push(`didn't match selector {${k}=${v}}`); + } + } + + // Pod anti-affinity (hostname topology): no two same-app pods per node. + if (pod.antiAffinity && nodePods.some((p) => p.app === pod.app)) { + reasons.push(`anti-affinity: ${pod.app} already on node`); + } + + // Resource fit (the filtering phase's NodeResourcesFit). + const free = freeResources(node, nodePods); + if (free.cpu < pod.cpu) { + reasons.push(`insufficient cpu (free ${free.cpu}m, need ${pod.cpu}m)`); + } + if (free.mem < pod.mem) { + reasons.push(`insufficient memory (free ${free.mem}Mi, need ${pod.mem}Mi)`); + } + if (pod.gpu > 0 && free.gpu < pod.gpu) { + reasons.push(`insufficient nvidia.com/gpu (free ${free.gpu}, need ${pod.gpu})`); + } + + return { ok: reasons.length === 0, reasons }; +} + +/** + * Score a feasible node for a pod (the scoring phase). Higher is better. + * Uses a MostAllocated/bin-packing strategy so the cluster stays tightly + * packed, with a small bonus for honoring the pod's preferred zone. + */ +export function scoreNode(pod, node, nodePods) { + const used = nodeAllocation(nodePods); + const cpuFrac = (used.cpu + pod.cpu) / node.cpu; + const memFrac = (used.mem + pod.mem) / node.mem; + let score = ((cpuFrac + memFrac) / 2) * 100; + + if (pod.preferredZone && node.labels["topology.kubernetes.io/zone"] === pod.preferredZone) { + score += 8; + } + // Soft (preferred) pod anti-affinity: discourage co-locating same-app replicas. + if (pod.softAntiAffinity && nodePods.some((p) => p.app === pod.app)) { + score -= 18; + } + // Gently prefer cheaper nodes when packing is otherwise equal. + score -= node.cost * 2; + // GPU nodes are scarce: avoid burning them on non-GPU pods. + if (pod.gpu === 0 && node.gpu > 0) score -= 25; + return score; +} + +/** + * Pick the best feasible node for a pod. + * @param pod the pod to place + * @param nodes array of node objects + * @param podsByNode function(nodeId) -> pod[] running on that node + * @returns { node, score } or null when nothing fits. + */ +export function bestNodeFor(pod, nodes, podsByNode) { + let best = null; + for (const node of nodes) { + const nodePods = podsByNode(node.id); + const fit = evaluateFit(pod, node, nodePods); + if (!fit.ok) continue; + const score = scoreNode(pod, node, nodePods); + if (!best || score > best.score) best = { node, score }; + } + return best; +} + +/** + * Aggregate the reasons across all nodes into the single most common blocker, + * so the UI/event log can explain why a pod is stuck Pending. + */ +export function summarizePendingReason(pod, nodes, podsByNode) { + if (nodes.length === 0) return "no nodes in cluster"; + const counts = new Map(); + for (const node of nodes) { + const fit = evaluateFit(pod, node, podsByNode(node.id)); + for (const r of fit.reasons) { + // Normalize the dynamic numbers out of resource messages for tallying. + const key = r.replace(/\d+/g, "N"); + counts.set(key, (counts.get(key) || 0) + 1); + } + } + let top = null; + for (const [k, c] of counts) { + if (!top || c > top.c) top = { k, c }; + } + return top ? top.k : "unschedulable"; +} diff --git a/k8s-scheduler-game/src/types.js b/k8s-scheduler-game/src/types.js new file mode 100644 index 0000000..6ac9a70 --- /dev/null +++ b/k8s-scheduler-game/src/types.js @@ -0,0 +1,265 @@ +// Domain model for the Kubescheduler game. +// +// All CPU values are in millicores (m); 1000m == 1 vCPU. +// All memory values are in MiB; 1024 MiB == 1 GiB. +// These are the same units real Kubernetes resource requests use. + +export const TICKS_PER_SECOND = 4; // simulation ticks per real second at 1x speed +export const SLA_PENDING_TICKS = 40; // a pod pending longer than this breaches SLA + +/** Availability zones a node can be placed in. */ +export const ZONES = ["us-east-1a", "us-east-1b", "us-east-1c"]; + +/** + * Node instance types ("node pools"). Each carries baked-in labels and taints, + * exactly like a managed node group in EKS/GKE. cost is a relative $/hour figure + * used purely for scoring. + */ +export const INSTANCE_TYPES = { + "general-medium": { + key: "general-medium", + family: "general", + cpu: 4000, + mem: 8192, + gpu: 0, + cost: 0.16, + bootTicks: 8, + labels: { "node.kubernetes.io/instance-type": "general-medium", disktype: "hdd" }, + taints: [], + }, + "general-large": { + key: "general-large", + family: "general", + cpu: 8000, + mem: 16384, + gpu: 0, + cost: 0.32, + bootTicks: 10, + labels: { "node.kubernetes.io/instance-type": "general-large", disktype: "hdd" }, + taints: [], + }, + "ssd-large": { + key: "ssd-large", + family: "ssd", + cpu: 8000, + mem: 16384, + gpu: 0, + cost: 0.42, + bootTicks: 10, + labels: { "node.kubernetes.io/instance-type": "ssd-large", disktype: "ssd" }, + taints: [], + }, + "mem-xlarge": { + key: "mem-xlarge", + family: "mem", + cpu: 8000, + mem: 65536, + gpu: 0, + cost: 0.55, + bootTicks: 12, + labels: { "node.kubernetes.io/instance-type": "mem-xlarge", disktype: "ssd" }, + taints: [], + }, + "gpu-xlarge": { + key: "gpu-xlarge", + family: "gpu", + cpu: 8000, + mem: 32768, + gpu: 4, + cost: 2.4, + bootTicks: 16, + labels: { + "node.kubernetes.io/instance-type": "gpu-xlarge", + disktype: "ssd", + accelerator: "nvidia-t4", + }, + taints: [{ key: "nvidia.com/gpu", value: "present", effect: "NoSchedule" }], + }, + "spot-medium": { + key: "spot-medium", + family: "spot", + cpu: 4000, + mem: 8192, + gpu: 0, + cost: 0.05, + bootTicks: 6, + labels: { "node.kubernetes.io/instance-type": "spot-medium", disktype: "hdd" }, + taints: [{ key: "spot", value: "true", effect: "NoSchedule" }], + }, +}; + +/** Per-app color used throughout the UI. */ +export const APP_COLORS = { + frontend: "#38bdf8", + api: "#34d399", + cache: "#f472b6", + postgres: "#a78bfa", + batch: "#fbbf24", + "ml-train": "#fb7185", +}; + +/** + * Deployment / workload templates. Pods are minted from these. + * - kind "service": long-running, never completes on its own. + * - kind "job": runs for a bounded lifetime then completes and frees resources. + * - nodeSelector: hard label match (required node affinity). + * - tolerations: taints this pod can land on. + * - antiAffinity: HARD pod anti-affinity — two pods of the same app may never + * share a node (hostname topology). Only used on low-volume apps, since it + * caps the app at one replica per node. + * - softAntiAffinity: PREFERRED spread — influences scoring (the scheduler + * tries to spread replicas across nodes) but never blocks placement. + * - maxReplicas: like a Deployment's replica count — the workload generator + * won't mint a new pod for an app already at this many live replicas. This + * bounds the cluster and keeps hard anti-affinity satisfiable. + * - preferredZone: soft affinity used only for scoring/tie-breaks. + * - lifetime: [min, max] ticks the pod runs before it finishes. Services get + * long, variable lifetimes (think rollouts / scaling churn) so the cluster + * reaches a bounded steady state; jobs are short. + */ +export const APP_TEMPLATES = { + frontend: { + app: "frontend", + kind: "service", + cpu: 250, + mem: 256, + gpu: 0, + nodeSelector: {}, + tolerations: [], + antiAffinity: false, + softAntiAffinity: true, + maxReplicas: 26, + priority: 100, + lifetime: [120, 300], + weight: 7, + }, + api: { + app: "api", + kind: "service", + cpu: 500, + mem: 512, + gpu: 0, + nodeSelector: {}, + tolerations: [], + antiAffinity: false, + softAntiAffinity: true, + maxReplicas: 18, + priority: 200, + lifetime: [140, 340], + weight: 6, + }, + cache: { + app: "cache", + kind: "service", + cpu: 500, + mem: 2048, + gpu: 0, + nodeSelector: { disktype: "ssd" }, + tolerations: [], + antiAffinity: false, + softAntiAffinity: false, + maxReplicas: 10, + priority: 150, + lifetime: [160, 360], + weight: 3, + }, + postgres: { + app: "postgres", + kind: "service", + cpu: 1000, + mem: 6144, + gpu: 0, + nodeSelector: { disktype: "ssd" }, + tolerations: [], + antiAffinity: true, + softAntiAffinity: false, + maxReplicas: 4, + priority: 400, + lifetime: [160, 300], + weight: 1, + }, + batch: { + app: "batch", + kind: "job", + cpu: 1000, + mem: 1024, + gpu: 0, + nodeSelector: {}, + tolerations: [{ key: "spot", value: "true", effect: "NoSchedule" }], + antiAffinity: false, + softAntiAffinity: false, + maxReplicas: 28, + priority: 50, + lifetime: [30, 90], + weight: 5, + }, + "ml-train": { + app: "ml-train", + kind: "job", + cpu: 2000, + mem: 8192, + gpu: 1, + nodeSelector: { accelerator: "nvidia-t4" }, + tolerations: [{ key: "nvidia.com/gpu", value: "present", effect: "NoSchedule" }], + antiAffinity: false, + softAntiAffinity: false, + maxReplicas: 6, + priority: 300, + lifetime: [40, 120], + weight: 2, + }, +}; + +// --------------------------------------------------------------------------- +// Small pure helpers shared across modules. +// --------------------------------------------------------------------------- + +/** Sum the resource requests of a list of pod objects. */ +export function sumRequests(pods) { + return pods.reduce( + (acc, p) => { + acc.cpu += p.cpu; + acc.mem += p.mem; + acc.gpu += p.gpu; + return acc; + }, + { cpu: 0, mem: 0, gpu: 0 } + ); +} + +/** Capacity object for a node from its instance type. */ +export function nodeCapacity(node) { + return { cpu: node.cpu, mem: node.mem, gpu: node.gpu }; +} + +/** True if a node is currently able to accept newly scheduled pods. */ +export function isSchedulable(node) { + return node.status === "Ready"; +} + +/** Format millicores as a friendly core string. */ +export function fmtCpu(m) { + if (m >= 1000) return `${(m / 1000).toFixed(m % 1000 === 0 ? 0 : 1)}`; + return `${(m / 1000).toFixed(2)}`; +} + +/** Format MiB as Gi/Mi. */ +export function fmtMem(mib) { + if (mib >= 1024) return `${(mib / 1024).toFixed(mib % 1024 === 0 ? 0 : 1)}Gi`; + return `${mib}Mi`; +} + +let _seq = 0; +/** Monotonic id generator. */ +export function nextId(prefix) { + _seq += 1; + return `${prefix}-${_seq.toString(36)}`; +} + +/** Short random suffix that looks like a real replica hash. */ +export function randHash(rng, n = 5) { + const chars = "0123456789abcdefghijklmnopqrstuvwxyz"; + let s = ""; + for (let i = 0; i < n; i++) s += chars[Math.floor(rng() * chars.length)]; + return s; +} diff --git a/k8s-scheduler-game/src/workload.js b/k8s-scheduler-game/src/workload.js new file mode 100644 index 0000000..26b75ff --- /dev/null +++ b/k8s-scheduler-game/src/workload.js @@ -0,0 +1,148 @@ +// Workload generation. Each scenario describes a starting cluster and an +// arrival process that mints pods from the APP_TEMPLATES over time. + +import { APP_TEMPLATES, APP_COLORS, ZONES, nextId, randHash } from "./types.js"; + +/** Deterministic, seedable PRNG (mulberry32) so runs are reproducible. */ +export function makeRng(seed) { + let a = seed >>> 0; + return function () { + a |= 0; + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +function pick(rng, arr) { + return arr[Math.floor(rng() * arr.length)]; +} + +function randInt(rng, [lo, hi]) { + return lo + Math.floor(rng() * (hi - lo + 1)); +} + +/** Weighted choice from a {key: weight} map. */ +function weightedPick(rng, weights) { + const entries = Object.entries(weights).filter(([, w]) => w > 0); + const total = entries.reduce((s, [, w]) => s + w, 0); + let r = rng() * total; + for (const [k, w] of entries) { + r -= w; + if (r <= 0) return k; + } + return entries[entries.length - 1][0]; +} + +/** Mint a concrete pod from a template. */ +export function createPod(appName, rng, tick) { + const t = APP_TEMPLATES[appName]; + const pod = { + id: nextId("pod"), + name: `${t.app}-${randHash(rng, 4)}-${randHash(rng, 4)}`, + app: t.app, + color: APP_COLORS[t.app] || "#94a3b8", + kind: t.kind, + cpu: t.cpu, + mem: t.mem, + gpu: t.gpu, + nodeSelector: { ...t.nodeSelector }, + tolerations: t.tolerations.map((x) => ({ ...x })), + antiAffinity: !!t.antiAffinity, + softAntiAffinity: !!t.softAntiAffinity, + priority: t.priority, + preferredZone: pick(rng, ZONES), + status: "Pending", + nodeId: null, + arrivalTick: tick, + scheduledTick: null, + pendingTicks: 0, + slaBreached: false, + // every pod runs for a bounded time; remaining ticks only tick down while Running. + remainingTicks: randInt(rng, t.lifetime), + }; + return pod; +} + +/** + * Scenario catalogue. `arrival(tick, rng)` returns the expected number of pods + * to spawn this tick (fractional values spawn probabilistically). `weights` + * may be a function of tick to create waves. + */ +export const SCENARIOS = { + steady: { + id: "steady", + name: "Steady State", + blurb: "A balanced, predictable workload. Great for learning the ropes.", + startNodes: ["general-large", "general-large", "ssd-large"], + seed: 1337, + arrival: () => 0.55, + weights: () => ({ frontend: 7, api: 6, cache: 3, postgres: 1, batch: 4 }), + }, + spike: { + id: "spike", + name: "Traffic Spike", + blurb: "Calm baseline punctuated by big frontend/api surges. Scale up fast, scale down after.", + startNodes: ["general-large", "ssd-large"], + seed: 7, + arrival: (tick) => { + const phase = tick % 240; + return phase < 60 ? 1.8 : 0.25; // ~15s storm every ~60s + }, + weights: (tick) => { + const storm = tick % 240 < 60; + return storm + ? { frontend: 12, api: 9, cache: 2, postgres: 0, batch: 1 } + : { frontend: 4, api: 3, cache: 2, postgres: 1, batch: 3 }; + }, + }, + gpu: { + id: "gpu", + name: "GPU Crunch", + blurb: "Steady services plus periodic ML training jobs that demand GPU nodes you must provision.", + startNodes: ["general-large", "ssd-large"], + seed: 99, + arrival: (tick) => (tick % 200 < 30 ? 1.4 : 0.5), + weights: (tick) => { + const burst = tick % 200 < 30; + return burst + ? { frontend: 3, api: 3, cache: 1, postgres: 0, batch: 2, "ml-train": 6 } + : { frontend: 6, api: 5, cache: 2, postgres: 1, batch: 3, "ml-train": 0 }; + }, + }, + chaos: { + id: "chaos", + name: "Production Chaos", + blurb: "Everything, everywhere, all at once. High churn across every workload type. Hard mode.", + startNodes: ["general-large", "ssd-large"], + seed: 42, + arrival: (tick) => 0.9 + (tick % 150 < 40 ? 1.2 : 0), + weights: () => ({ frontend: 8, api: 7, cache: 4, postgres: 2, batch: 6, "ml-train": 3 }), + }, +}; + +/** + * Spawn pods for a single tick. Returns an array of new pod objects. + * + * `aliveByApp` maps app -> current live replica count; apps already at their + * maxReplicas are excluded so each app behaves like a bounded Deployment. + */ +export function spawnForTick(scenario, rng, tick, aliveByApp = {}) { + const rate = scenario.arrival(tick, rng); + const weights = { ...scenario.weights(tick) }; + // Zero out apps that have hit their replica cap. + for (const app of Object.keys(weights)) { + const cap = APP_TEMPLATES[app]?.maxReplicas ?? Infinity; + if ((aliveByApp[app] || 0) >= cap) weights[app] = 0; + } + if (Object.values(weights).every((w) => w <= 0)) return []; + + let count = Math.floor(rate); + if (rng() < rate - count) count += 1; // fractional remainder -> probabilistic + const pods = []; + for (let i = 0; i < count; i++) { + pods.push(createPod(weightedPick(rng, weights), rng, tick)); + } + return pods; +} diff --git a/k8s-scheduler-game/test/scheduler.test.mjs b/k8s-scheduler-game/test/scheduler.test.mjs new file mode 100644 index 0000000..5a45aa1 --- /dev/null +++ b/k8s-scheduler-game/test/scheduler.test.mjs @@ -0,0 +1,219 @@ +// Pure-logic tests for the scheduler predicates and the engine. Run with: +// node --test k8s-scheduler-game/test/scheduler.test.mjs +import test from "node:test"; +import assert from "node:assert/strict"; + +import { evaluateFit, tolerates, selectorMatches, freeResources, bestNodeFor } from "../src/scheduler.js"; +import { Game } from "../src/engine.js"; +import { INSTANCE_TYPES } from "../src/types.js"; +import { createPod, makeRng, SCENARIOS, spawnForTick } from "../src/workload.js"; + +function nodeFrom(typeKey, zone = "us-east-1a") { + const s = INSTANCE_TYPES[typeKey]; + return { + id: "n1", + status: "Ready", + cpu: s.cpu, + mem: s.mem, + gpu: s.gpu, + cost: s.cost, + labels: { ...s.labels, "topology.kubernetes.io/zone": zone }, + taints: s.taints.map((t) => ({ ...t })), + podIds: [], + }; +} + +test("tolerations match taints correctly", () => { + const taint = { key: "spot", value: "true", effect: "NoSchedule" }; + assert.equal(tolerates([{ key: "spot", value: "true", effect: "NoSchedule" }], taint), true); + assert.equal(tolerates([{ key: "spot", operator: "Exists" }], taint), true); + assert.equal(tolerates([{ key: "nvidia.com/gpu" }], taint), false); + assert.equal(tolerates([], taint), false); +}); + +test("selectorMatches enforces every label", () => { + assert.equal(selectorMatches({ disktype: "ssd" }, { disktype: "ssd", zone: "a" }), true); + assert.equal(selectorMatches({ disktype: "ssd" }, { disktype: "hdd" }), false); + assert.equal(selectorMatches({}, { anything: "x" }), true); +}); + +test("resource fit blocks oversized pods", () => { + const node = nodeFrom("general-medium"); // 4000m / 8192Mi + const big = { cpu: 5000, mem: 1024, gpu: 0, nodeSelector: {}, tolerations: [], antiAffinity: false, app: "x" }; + const fit = evaluateFit(big, node, []); + assert.equal(fit.ok, false); + assert.ok(fit.reasons.some((r) => r.includes("insufficient cpu"))); +}); + +test("free resources subtract running pods", () => { + const node = nodeFrom("general-large"); // 8000m / 16384 + const pods = [ + { cpu: 1000, mem: 2048, gpu: 0 }, + { cpu: 500, mem: 512, gpu: 0 }, + ]; + const free = freeResources(node, pods); + assert.equal(free.cpu, 8000 - 1500); + assert.equal(free.mem, 16384 - 2560); +}); + +test("nodeSelector for ssd is enforced", () => { + const hdd = nodeFrom("general-large"); // disktype hdd + const ssd = nodeFrom("ssd-large"); // disktype ssd + const pod = { cpu: 500, mem: 512, gpu: 0, nodeSelector: { disktype: "ssd" }, tolerations: [], antiAffinity: false, app: "cache" }; + assert.equal(evaluateFit(pod, hdd, []).ok, false); + assert.equal(evaluateFit(pod, ssd, []).ok, true); +}); + +test("gpu taint requires toleration and gpu capacity", () => { + const gpuNode = nodeFrom("gpu-xlarge"); + const noTol = { cpu: 500, mem: 512, gpu: 0, nodeSelector: {}, tolerations: [], antiAffinity: false, app: "frontend" }; + // frontend has no toleration -> blocked by taint + assert.equal(evaluateFit(noTol, gpuNode, []).ok, false); + + const mlPod = { + cpu: 2000, mem: 8192, gpu: 1, + nodeSelector: { accelerator: "nvidia-t4" }, + tolerations: [{ key: "nvidia.com/gpu", value: "present", effect: "NoSchedule" }], + antiAffinity: false, app: "ml-train", + }; + assert.equal(evaluateFit(mlPod, gpuNode, []).ok, true); +}); + +test("anti-affinity prevents two same-app pods on one node", () => { + const node = nodeFrom("general-large"); + const a = { cpu: 250, mem: 256, gpu: 0, nodeSelector: {}, tolerations: [], antiAffinity: true, app: "frontend" }; + const b = { cpu: 250, mem: 256, gpu: 0, nodeSelector: {}, tolerations: [], antiAffinity: true, app: "frontend" }; + assert.equal(evaluateFit(b, node, [a]).ok, false); + assert.ok(evaluateFit(b, node, [a]).reasons.some((r) => r.includes("anti-affinity"))); +}); + +test("cordoned nodes are unschedulable", () => { + const node = nodeFrom("general-large"); + node.status = "Cordoned"; + const pod = { cpu: 250, mem: 256, gpu: 0, nodeSelector: {}, tolerations: [], antiAffinity: false, app: "frontend" }; + assert.equal(evaluateFit(pod, node, []).ok, false); +}); + +test("bestNodeFor packs onto the fuller feasible node", () => { + const n1 = { ...nodeFrom("general-large"), id: "n1" }; + const n2 = { ...nodeFrom("general-large"), id: "n2" }; + const existing = { cpu: 4000, mem: 4096, gpu: 0, app: "api" }; + const podsByNode = (id) => (id === "n1" ? [existing] : []); + const pod = { cpu: 500, mem: 512, gpu: 0, nodeSelector: {}, tolerations: [], antiAffinity: false, app: "frontend" }; + const best = bestNodeFor(pod, [n1, n2], podsByNode); + assert.equal(best.node.id, "n1"); // MostAllocated -> fill n1 first +}); + +test("engine schedules a pod and records latency", () => { + const game = new Game("steady"); + game.state.tick = 5; + const pod = createPod("frontend", makeRng(1), 2); + game.state.pods.set(pod.id, pod); + game.state.pendingIds.push(pod.id); + const node = game.schedulableNodes()[0]; + const res = game.schedulePod(pod.id, node.id); + assert.equal(res.ok, true); + assert.equal(pod.status, "Running"); + assert.equal(game.state.metrics.latencySum, 3); // 5 - 2 +}); + +test("draining a node evicts its pods back to pending with a penalty", () => { + const game = new Game("steady"); + const pod = createPod("api", makeRng(2), 0); + game.state.pods.set(pod.id, pod); + game.state.pendingIds.push(pod.id); + const node = game.schedulableNodes()[0]; + game.schedulePod(pod.id, node.id); + const before = game.state.score; + game.drain(node.id); + assert.equal(pod.status, "Pending"); + assert.equal(node.status, "Cordoned"); + assert.ok(game.state.score < before); + assert.ok(game.state.pendingIds.includes(pod.id)); +}); + +test("jobs complete and free resources after their lifetime", () => { + const game = new Game("steady"); + const pod = createPod("batch", makeRng(3), 0); + pod.remainingTicks = 2; + game.state.pods.set(pod.id, pod); + game.state.pendingIds.push(pod.id); + const node = game.schedulableNodes()[0]; + game.schedulePod(pod.id, node.id); + assert.ok(node.podIds.includes(pod.id)); + game.tick(); + game.tick(); + game.tick(); + assert.equal(game.state.pods.has(pod.id), false); + assert.equal(node.podIds.includes(pod.id), false); + assert.ok(game.state.metrics.completedTotal >= 1); +}); + +test("autoscaler provisions a GPU node when ml-train is stuck pending", () => { + const game = new Game("steady"); + game.state.autoScale = true; + game.state.autoSchedule = true; + const pod = createPod("ml-train", makeRng(4), 0); + game.state.pods.set(pod.id, pod); + game.state.pendingIds.push(pod.id); + // No GPU node exists initially; autoscaler should add one within a few ticks. + let addedGpu = false; + for (let i = 0; i < 40 && !addedGpu; i++) { + game.tick(); + addedGpu = game.state.nodes.some((n) => n.type === "gpu-xlarge"); + } + assert.equal(addedGpu, true); +}); + +test("workload generator is deterministic for a seed", () => { + const a = spawnForTick(SCENARIOS.chaos, makeRng(42), 10).map((p) => p.app); + const b = spawnForTick(SCENARIOS.chaos, makeRng(42), 10).map((p) => p.app); + assert.deepEqual(a, b); +}); + +test("soak: every scenario runs 1200 ticks under full automation without overcommit", () => { + for (const id of Object.keys(SCENARIOS)) { + const game = new Game(id); + game.state.autoSchedule = true; + game.state.autoScale = true; + for (let i = 0; i < 1200; i++) game.tick(); + assert.ok(Number.isFinite(game.state.score), `${id} score is finite`); + for (const node of game.state.nodes) { + const used = game.podsOnNode(node).reduce( + (a, p) => ({ cpu: a.cpu + p.cpu, mem: a.mem + p.mem, gpu: a.gpu + p.gpu }), + { cpu: 0, mem: 0, gpu: 0 } + ); + assert.ok(used.cpu <= node.cpu, `${id}: cpu overcommit on ${node.id}`); + assert.ok(used.mem <= node.mem, `${id}: mem overcommit on ${node.id}`); + assert.ok(used.gpu <= node.gpu, `${id}: gpu overcommit on ${node.id}`); + } + } +}); + +test("full automation keeps every scenario healthy (bounded queue, good utilization)", () => { + for (const id of Object.keys(SCENARIOS)) { + const game = new Game(id); + game.state.autoSchedule = true; + game.state.autoScale = true; + for (let i = 0; i < 1000; i++) game.tick(); + const pending = game.state.pendingIds.length; + const util = game.clusterUtilization(); + // Generous bounds: the auto policy should clearly keep up, not melt down. + assert.ok(pending < 35, `${id}: pending=${pending}`); + assert.ok(util > 0.35, `${id}: utilization=${util.toFixed(2)}`); + assert.ok(game.state.metrics.slaBreaches < 40, `${id}: sla=${game.state.metrics.slaBreaches}`); + assert.ok(game.state.score > 0, `${id}: score=${Math.round(game.state.score)}`); + } +}); + +test("finite pod lifetimes keep the cluster population bounded", () => { + const game = new Game("steady"); + game.state.autoSchedule = true; + game.state.autoScale = true; + for (let i = 0; i < 1200; i++) game.tick(); + // Services retire, so the total live pod count reaches a steady state instead + // of growing without bound (this also keeps the tick loop cheap). + assert.ok(game.state.pods.size < 400, `live pods=${game.state.pods.size}`); + assert.ok(game.state.metrics.retiredTotal > 0, "services should retire over time"); + assert.ok(game.state.pendingIds.length < 60, `pending=${game.state.pendingIds.length}`); +});