From 5c54c49ab867681cc4e3f6584355e15eaa719cdf Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 26 Jun 2026 13:26:16 +0000 Subject: [PATCH] Add daemonsets, spot interruptions, and node upgrades to sim core - DaemonSets: infra pods the controller reconciles onto every matching node (gpu plugin only on GPU nodes); counted as per-node overhead, kept on drain like kubectl, and recreated when a node joins or finishes an upgrade. - Spot nodes: a fluctuating spot price (mean-reverting random walk with occasional spikes) drives both billing and an unexpected-reclaim risk; reclaimed nodes give a short drain window before evicting their pods. - Cluster upgrades: nodes carry a version, periodic upgrade events bump the target, and upgradeNode() drains + reboots a node onto the new version. Outdated nodes leak score while pending; a full rollout pays a bonus. - Autopilot now recovers from spot reclaims and performs a safe one-at-a -time rolling upgrade. - Tests cover daemonset reconciliation/overhead, spot reclaim + price bounds, upgrade drain/reboot, rollout bonus, and auto rolling upgrade. Co-authored-by: Jason Hall --- k8s-scheduler-game/src/engine.js | 249 +++++++++++++++++++-- k8s-scheduler-game/src/types.js | 39 ++++ k8s-scheduler-game/src/workload.js | 36 +++ k8s-scheduler-game/test/scheduler.test.mjs | 136 +++++++++++ 4 files changed, 436 insertions(+), 24 deletions(-) diff --git a/k8s-scheduler-game/src/engine.js b/k8s-scheduler-game/src/engine.js index fff3902..e1a2821 100644 --- a/k8s-scheduler-game/src/engine.js +++ b/k8s-scheduler-game/src/engine.js @@ -3,9 +3,16 @@ // pod), and computes the score. It is intentionally UI-agnostic: the UI reads // `game.state` and calls these methods, then re-renders. -import { INSTANCE_TYPES, ZONES, TICKS_PER_SECOND, SLA_PENDING_TICKS } from "./types.js"; -import { bestNodeFor, evaluateFit, summarizePendingReason } from "./scheduler.js"; -import { SCENARIOS, makeRng, spawnForTick } from "./workload.js"; +import { + INSTANCE_TYPES, + ZONES, + TICKS_PER_SECOND, + SLA_PENDING_TICKS, + DAEMONSETS, + effectiveCost, +} from "./types.js"; +import { bestNodeFor, evaluateFit, selectorMatches, summarizePendingReason } from "./scheduler.js"; +import { SCENARIOS, makeRng, spawnForTick, createDaemonPod } from "./workload.js"; // --- Scoring weights ------------------------------------------------------- const UTIL_W = 30; // reward per tick at 100% cluster utilization @@ -15,6 +22,11 @@ const COMPLETE_BONUS = 8; // reward for finishing a job const DRAIN_EVICT_PEN = 2; // graceful eviction (drain) penalty per pod const FORCE_EVICT_PEN = 10; // hard eviction (delete live node) penalty per pod const SLA_BREACH_PEN = 40; // one-time penalty when a pod blows its scheduling SLA +const SPOT_RECLAIM_PEN = 1.5; // per workload pod lost to a (somewhat expected) spot interruption +const SPOT_WARNING_TICKS = 12; // ~3s interruption notice at 4x before a spot node is reclaimed +const SPOT_RECLAIM_BASE = 0.0035; // base per-tick probability a spot node is interrupted +const OUTDATED_PEN = 0.3; // per outdated, still-up node per tick while an upgrade is pending +const UPGRADE_DONE_BONUS = 35; // reward for completing a cluster-wide rollout const MAX_EVENTS = 240; export class Game { @@ -38,8 +50,14 @@ export class Game { autoScale: false, events: [], score: 0, + // cluster version: nodes carry a minor version; an upgrade bumps the target. + clusterMinor: 30, + upgradePending: false, + // spot market: a fluctuating multiplier applied to spot node $/hr. + spotPrice: 1, + nextUpgradeTick: scenario.upgradeEvery || 0, // running tallies for the score breakdown panel - breakdown: { util: 0, latency: 0, cost: 0, jobs: 0, sla: 0, disruption: 0 }, + breakdown: { util: 0, latency: 0, cost: 0, jobs: 0, sla: 0, disruption: 0, upgrade: 0 }, metrics: { latencySum: 0, latencyCount: 0, @@ -49,6 +67,8 @@ export class Game { slaBreaches: 0, evictions: 0, spawnedTotal: 0, + spotReclaims: 0, + nodesUpgraded: 0, }, lastUtil: 0, scaleCooldown: 0, @@ -93,19 +113,50 @@ export class Game { mem: spec.mem, gpu: spec.gpu, cost: spec.cost, + spot: spec.family === "spot", + minor: this.state.clusterMinor, // new nodes join on the current version labels: { ...spec.labels, "topology.kubernetes.io/zone": z }, taints: spec.taints.map((t) => ({ ...t })), status: instant ? "Ready" : "Provisioning", provisioningTicksLeft: instant ? 0 : spec.bootTicks, + spotWarnTicksLeft: 0, podIds: [], idleTicks: 0, createdTick: this.state.tick, }; this.state.nodes.push(node); - if (!instant) this.log("info", `Provisioning ${node.name} (${typeKey}, ${z})…`); + if (instant) this.reconcileDaemonSets(node); + else this.log("info", `Provisioning ${node.name} (${typeKey}, ${z})…`); return node; } + /** Ensure every applicable DaemonSet has a pod on this (Ready) node. */ + reconcileDaemonSets(node) { + if (node.status !== "Ready") return; + for (const ds of DAEMONSETS) { + if (ds.nodeSelector && !selectorMatches(ds.nodeSelector, node.labels)) continue; + const present = node.podIds.some((id) => this.state.pods.get(id)?.daemonOf === ds.name); + if (!present) { + const pod = createDaemonPod(ds, node, this.state.tick); + this.state.pods.set(pod.id, pod); + node.podIds.push(pod.id); + } + } + } + + /** Pods on a node that are real workload (excludes DaemonSet/infra pods). */ + workloadPodsOnNode(node) { + return this.podsOnNode(node).filter((p) => p.kind !== "daemon"); + } + + /** Delete a node's DaemonSet pods from the pod map (used when a node leaves). */ + purgeDaemonPods(node) { + for (const id of node.podIds) { + const p = this.state.pods.get(id); + if (p && p.kind === "daemon") this.state.pods.delete(id); + } + } + cordon(nodeId) { const node = this.nodeById(nodeId); if (!node || node.status === "Provisioning") return; @@ -118,25 +169,28 @@ export class Game { } } - /** Graceful drain: cordon, then evict every pod back to the queue. */ + /** Graceful drain: cordon, then evict workload pods back to the queue. + * DaemonSet pods are left running, exactly like `kubectl drain`. */ drain(nodeId) { const node = this.nodeById(nodeId); if (!node) return; - const pods = this.podsOnNode(node); + const pods = this.workloadPodsOnNode(node); for (const pod of pods) this.evictPod(pod, node, DRAIN_EVICT_PEN, "drain"); - node.status = "Cordoned"; - this.log("warn", `Drained ${node.name} (${pods.length} pod(s) rescheduled).`); + if (node.status !== "Reclaiming" && node.status !== "Upgrading") node.status = "Cordoned"; + this.log("warn", `Drained ${node.name} (${pods.length} pod(s) rescheduled; daemonsets kept).`); } - /** Terminate a node. Any still-running pods are forcibly evicted. */ + /** Terminate a node. Workload pods still on it are forcibly evicted. */ deleteNode(nodeId) { const node = this.nodeById(nodeId); if (!node) return; - const pods = this.podsOnNode(node); + const pods = this.workloadPodsOnNode(node); for (const pod of pods) this.evictPod(pod, node, FORCE_EVICT_PEN, "force-delete"); + this.purgeDaemonPods(node); this.state.nodes = this.state.nodes.filter((n) => n.id !== nodeId); const lvl = pods.length ? "error" : "info"; this.log(lvl, `Terminated ${node.name}${pods.length ? ` (force-killed ${pods.length} pod(s)!)` : ""}.`); + this.checkUpgradeComplete(); } evictPod(pod, node, penalty, reason) { @@ -154,6 +208,97 @@ export class Game { void reason; } + // --- spot interruptions -------------------------------------------------- + /** The cloud reclaims a spot node: its workload is evicted and it's gone. */ + reclaimSpotNode(node) { + const pods = this.workloadPodsOnNode(node); + for (const pod of pods) this.evictPod(pod, node, SPOT_RECLAIM_PEN, "spot-reclaim"); + this.purgeDaemonPods(node); + this.state.nodes = this.state.nodes.filter((n) => n.id !== node.id); + this.state.metrics.spotReclaims += 1; + this.log("error", `Spot interruption: ${node.name} reclaimed by the cloud (${pods.length} pod(s) evicted).`); + this.checkUpgradeComplete(); + } + + // --- cluster upgrades ---------------------------------------------------- + /** Control-plane upgrade event: bump the target version; nodes now lag. */ + triggerUpgrade() { + this.state.clusterMinor += 1; + this.state.upgradePending = true; + this.log( + "error", + `Control plane upgraded to v1.${this.state.clusterMinor}. Drain & restart every node to match!` + ); + } + + /** + * Responsibly restart a node onto the current version: gracefully drain its + * workload, drop its daemonset pods, then reboot (it returns Ready on the new + * version after its boot time). Force-restarting a node full of pods is the + * irresponsible way and is discouraged by the drain penalty. + */ + upgradeNode(nodeId) { + const node = this.nodeById(nodeId); + if (!node) return { ok: false, reason: "node not found" }; + if (node.minor >= this.state.clusterMinor) return { ok: false, reason: "already up to date" }; + if (["Provisioning", "Upgrading", "Reclaiming"].includes(node.status)) { + return { ok: false, reason: `node is ${node.status.toLowerCase()}` }; + } + const pods = this.workloadPodsOnNode(node); + for (const pod of pods) this.evictPod(pod, node, DRAIN_EVICT_PEN, "upgrade"); + this.purgeDaemonPods(node); + node.status = "Upgrading"; + node.provisioningTicksLeft = INSTANCE_TYPES[node.type].bootTicks; + this.log("warn", `Upgrading ${node.name} → v1.${this.state.clusterMinor} (${pods.length} pod(s) drained).`); + return { ok: true }; + } + + /** When every node is on the current version, the rollout is complete. */ + checkUpgradeComplete() { + if (!this.state.upgradePending) return; + const outdated = this.state.nodes.some((n) => n.minor < this.state.clusterMinor); + if (!outdated) { + this.state.upgradePending = false; + this.state.score += UPGRADE_DONE_BONUS; + this.state.breakdown.upgrade += UPGRADE_DONE_BONUS; + this.log("good", `Cluster fully upgraded to v1.${this.state.clusterMinor}. Clean rollout! +${UPGRADE_DONE_BONUS}`); + } + } + + /** + * Advance the spot market one tick: a mean-reverting random walk with the odd + * price spike, then roll interruption dice for each spot node. Warned nodes + * count down a short notice (during which you can drain them) before the cloud + * reclaims them. Higher prices mean more interruptions. + */ + updateSpotMarket() { + const s = this.state; + const r = this.rng; + let sp = s.spotPrice + (1 - s.spotPrice) * 0.03 + (r() - 0.5) * 0.08; + if (r() < 0.004) sp += 0.8 + r(); // occasional price spike + s.spotPrice = Math.max(0.2, Math.min(3, sp)); + + for (const node of [...s.nodes]) { + if (!node.spot) continue; + if (node.status === "Reclaiming") { + node.spotWarnTicksLeft -= 1; + if (node.spotWarnTicksLeft <= 0) this.reclaimSpotNode(node); + continue; + } + if (node.status === "Ready" || node.status === "Cordoned") { + const p = SPOT_RECLAIM_BASE * (0.5 + s.spotPrice); + if (r() < p) { + node.status = "Reclaiming"; + node.spotWarnTicksLeft = SPOT_WARNING_TICKS; + this.log( + "warn", + `Spot notice: ${node.name} will be reclaimed in ${(SPOT_WARNING_TICKS / TICKS_PER_SECOND).toFixed(0)}s.` + ); + } + } + } + } + // --- scheduling actions -------------------------------------------------- /** * Attempt to bind a pod to a node. Returns { ok, reasons }. @@ -228,6 +373,21 @@ export class Game { const s = this.state; if (s.scaleCooldown > 0) s.scaleCooldown -= 1; + // Automated rolling upgrade: bring outdated nodes up one at a time. We only + // start a new one when nothing is already booting/upgrading, which keeps the + // rollout safe (never tears down the whole cluster at once). + if (s.upgradePending) { + // One upgrade at a time keeps the rollout safe, but don't wait on routine + // provisioning (spot replacements etc.) or the rollout can stall. + const upgrading = s.nodes.some((n) => n.status === "Upgrading"); + if (!upgrading) { + const outdated = s.nodes + .filter((n) => n.status === "Ready" && n.minor < s.clusterMinor) + .sort((a, b) => this.workloadPodsOnNode(a).length - this.workloadPodsOnNode(b).length)[0]; + if (outdated) this.upgradeNode(outdated.id); + } + } + if (s.scaleCooldown === 0) { const ready = this.schedulableNodes(); const unfittable = this.pendingPods().filter( @@ -261,10 +421,14 @@ export class Game { } } - // Scale down: terminate a node that has been idle a while (keep at least one). + // Scale down: terminate a node idle of *workload* a while (keep at least one). if (s.scaleCooldown === 0 && this.schedulableNodes().length > 1) { const idle = this.state.nodes.find( - (n) => n.status === "Ready" && n.podIds.length === 0 && n.idleTicks > 20 + (n) => + n.status === "Ready" && + n.minor >= s.clusterMinor && + this.workloadPodsOnNode(n).length === 0 && + n.idleTicks > 20 ); if (idle) { this.deleteNode(idle.id); @@ -279,17 +443,35 @@ export class Game { const s = this.state; s.tick += 1; - // 1. node provisioning + // 1. node provisioning + upgrade completion for (const node of s.nodes) { - if (node.status === "Provisioning") { + if (node.status === "Provisioning" || node.status === "Upgrading") { node.provisioningTicksLeft -= 1; if (node.provisioningTicksLeft <= 0) { + const wasUpgrade = node.status === "Upgrading"; node.status = "Ready"; - this.log("good", `${node.name} is now Ready.`); + if (wasUpgrade) { + node.minor = s.clusterMinor; + s.metrics.nodesUpgraded += 1; + this.log("good", `${node.name} back online on v1.${node.minor}.`); + } else { + this.log("good", `${node.name} is now Ready.`); + } + this.reconcileDaemonSets(node); + this.checkUpgradeComplete(); } } } + // 1b. cluster upgrade events + if (s.nextUpgradeTick && s.tick >= s.nextUpgradeTick) { + this.triggerUpgrade(); + s.nextUpgradeTick = s.tick + (this.scenario.upgradeEvery || 700); + } + + // 1c. spot market + interruptions + this.updateSpotMarket(); + // 2. spawn workload (respecting each app's replica cap) const aliveByApp = {}; for (const p of s.pods.values()) { @@ -308,9 +490,13 @@ export class Game { if (s.autoSchedule) this.runAutoScheduler(); if (s.autoScale) this.runAutoScaler(); - // 4. advance running jobs, track idleness + // 4. advance running jobs, track idleness (daemonset-only nodes count idle) for (const node of s.nodes) { - if (node.podIds.length === 0 && node.status === "Ready") node.idleTicks += 1; + const hasWorkload = node.podIds.some((id) => { + const p = s.pods.get(id); + return p && p.kind !== "daemon"; + }); + if (!hasWorkload && node.status === "Ready") node.idleTicks += 1; else node.idleTicks = 0; } for (const pod of [...s.pods.values()]) { @@ -336,17 +522,25 @@ export class Game { } } - // 6. scoring (utilization reward + latency/cost pressure) + // 6. scoring (utilization reward + latency/cost pressure + upgrade tech-debt) const util = this.clusterUtilization(); s.lastUtil = util; const utilDelta = util * UTIL_W; const latDelta = pendingCount * PENDING_PEN; const hourlyCost = this.hourlyCost(); const costDelta = hourlyCost * COST_PEN; - s.score += utilDelta - latDelta - costDelta; + let outdatedDelta = 0; + if (s.upgradePending) { + const outdated = s.nodes.filter( + (n) => n.minor < s.clusterMinor && ["Ready", "Cordoned"].includes(n.status) + ).length; + outdatedDelta = outdated * OUTDATED_PEN; + } + s.score += utilDelta - latDelta - costDelta - outdatedDelta; s.breakdown.util += utilDelta; s.breakdown.latency -= latDelta; s.breakdown.cost -= costDelta; + s.breakdown.upgrade -= outdatedDelta; return s; } @@ -372,7 +566,7 @@ export class Game { /** Average utilization (cpu+mem)/2 across nodes that are up (paying). */ clusterUtilization() { const up = this.state.nodes.filter((n) => - ["Ready", "Cordoned", "Draining"].includes(n.status) + ["Ready", "Cordoned", "Draining", "Reclaiming"].includes(n.status) ); if (up.length === 0) return 0; let sum = 0; @@ -388,11 +582,11 @@ export class Game { return sum / up.length; } - /** Sum of $/hr for every node currently powered on. */ + /** Sum of $/hr for every node currently powered on (spot at live price). */ hourlyCost() { return this.state.nodes .filter((n) => n.status !== "Terminating") - .reduce((s, n) => s + n.cost, 0); + .reduce((s, n) => s + effectiveCost(n, this.state.spotPrice), 0); } avgLatencySeconds() { @@ -403,10 +597,17 @@ export class Game { runningCount() { let n = 0; - for (const p of this.state.pods.values()) if (p.status === "Running") n += 1; + for (const p of this.state.pods.values()) { + if (p.status === "Running" && p.kind !== "daemon") n += 1; + } return n; } + /** Count of nodes not yet on the current cluster version. */ + outdatedNodeCount() { + return this.state.nodes.filter((n) => n.minor < this.state.clusterMinor).length; + } + uptimeSeconds() { return this.state.tick / TICKS_PER_SECOND; } diff --git a/k8s-scheduler-game/src/types.js b/k8s-scheduler-game/src/types.js index 6ac9a70..9f463a9 100644 --- a/k8s-scheduler-game/src/types.js +++ b/k8s-scheduler-game/src/types.js @@ -86,8 +86,39 @@ export const INSTANCE_TYPES = { labels: { "node.kubernetes.io/instance-type": "spot-medium", disktype: "hdd" }, taints: [{ key: "spot", value: "true", effect: "NoSchedule" }], }, + "spot-large": { + key: "spot-large", + family: "spot", + cpu: 8000, + mem: 16384, + gpu: 0, + cost: 0.09, + bootTicks: 6, + labels: { "node.kubernetes.io/instance-type": "spot-large", disktype: "hdd" }, + taints: [{ key: "spot", value: "true", effect: "NoSchedule" }], + }, }; +/** + * DaemonSets: infrastructure pods the daemonset controller runs on EVERY node + * (or every node matching nodeSelector). They tolerate all taints, can't be + * hand-scheduled or moved, and consume capacity on every node — so they're pure + * per-node overhead that rewards running fewer, larger nodes. A node-local agent + * is recreated automatically whenever a node joins or finishes an upgrade. + */ +export const DAEMONSETS = [ + { name: "node-exporter", cpu: 75, mem: 96, color: "#64748b" }, + { name: "fluent-bit", cpu: 150, mem: 256, color: "#475569" }, + { name: "kube-proxy", cpu: 75, mem: 96, color: "#52525b" }, + { + name: "nvidia-device-plugin", + cpu: 50, + mem: 128, + color: "#3f6212", + nodeSelector: { accelerator: "nvidia-t4" }, + }, +]; + /** Per-app color used throughout the UI. */ export const APP_COLORS = { frontend: "#38bdf8", @@ -232,6 +263,14 @@ export function nodeCapacity(node) { return { cpu: node.cpu, mem: node.mem, gpu: node.gpu }; } +/** + * A node's current $/hr. Spot nodes are billed at the live, fluctuating spot + * price (a multiplier on their base cost); on-demand nodes are flat. + */ +export function effectiveCost(node, spotPrice = 1) { + return node.cost * (node.spot ? spotPrice : 1); +} + /** True if a node is currently able to accept newly scheduled pods. */ export function isSchedulable(node) { return node.status === "Ready"; diff --git a/k8s-scheduler-game/src/workload.js b/k8s-scheduler-game/src/workload.js index 26b75ff..12c79b1 100644 --- a/k8s-scheduler-game/src/workload.js +++ b/k8s-scheduler-game/src/workload.js @@ -65,6 +65,38 @@ export function createPod(appName, rng, tick) { return pod; } +/** + * Mint a DaemonSet pod bound to a specific node. These are placed directly by + * the "daemonset controller" (bypassing the scheduler), tolerate every taint, + * and never appear in the pending queue. + */ +export function createDaemonPod(ds, node, tick) { + return { + id: nextId("ds"), + name: `${ds.name}-${node.name.replace("node-", "")}`, + app: ds.name, + color: ds.color || "#64748b", + kind: "daemon", + daemonOf: ds.name, + cpu: ds.cpu, + mem: ds.mem, + gpu: 0, + nodeSelector: {}, + tolerations: [], + antiAffinity: false, + softAntiAffinity: false, + priority: 2000, + preferredZone: null, + status: "Running", + nodeId: node.id, + arrivalTick: tick, + scheduledTick: tick, + pendingTicks: 0, + slaBreached: false, + remainingTicks: null, + }; +} + /** * Scenario catalogue. `arrival(tick, rng)` returns the expected number of pods * to spawn this tick (fractional values spawn probabilistically). `weights` @@ -77,6 +109,7 @@ export const SCENARIOS = { blurb: "A balanced, predictable workload. Great for learning the ropes.", startNodes: ["general-large", "general-large", "ssd-large"], seed: 1337, + upgradeEvery: 900, arrival: () => 0.55, weights: () => ({ frontend: 7, api: 6, cache: 3, postgres: 1, batch: 4 }), }, @@ -86,6 +119,7 @@ export const SCENARIOS = { blurb: "Calm baseline punctuated by big frontend/api surges. Scale up fast, scale down after.", startNodes: ["general-large", "ssd-large"], seed: 7, + upgradeEvery: 700, arrival: (tick) => { const phase = tick % 240; return phase < 60 ? 1.8 : 0.25; // ~15s storm every ~60s @@ -103,6 +137,7 @@ export const SCENARIOS = { blurb: "Steady services plus periodic ML training jobs that demand GPU nodes you must provision.", startNodes: ["general-large", "ssd-large"], seed: 99, + upgradeEvery: 750, arrival: (tick) => (tick % 200 < 30 ? 1.4 : 0.5), weights: (tick) => { const burst = tick % 200 < 30; @@ -117,6 +152,7 @@ export const SCENARIOS = { blurb: "Everything, everywhere, all at once. High churn across every workload type. Hard mode.", startNodes: ["general-large", "ssd-large"], seed: 42, + upgradeEvery: 550, arrival: (tick) => 0.9 + (tick % 150 < 40 ? 1.2 : 0), weights: () => ({ frontend: 8, api: 7, cache: 4, postgres: 2, batch: 6, "ml-train": 3 }), }, diff --git a/k8s-scheduler-game/test/scheduler.test.mjs b/k8s-scheduler-game/test/scheduler.test.mjs index 5a45aa1..e3ce574 100644 --- a/k8s-scheduler-game/test/scheduler.test.mjs +++ b/k8s-scheduler-game/test/scheduler.test.mjs @@ -171,6 +171,142 @@ test("workload generator is deterministic for a seed", () => { assert.deepEqual(a, b); }); +function bootNode(game, node) { + for (let i = 0; i < INSTANCE_TYPES[node.type].bootTicks + 1; i++) game.tick(); +} + +test("daemonsets run one pod per node, match selectors, and never queue", () => { + const game = new Game("steady"); + // Start nodes are non-GPU: node-exporter + fluent-bit + kube-proxy = 3 each. + const node = game.schedulableNodes()[0]; + const daemons = game.podsOnNode(node).filter((p) => p.kind === "daemon"); + assert.equal(daemons.length, 3); + assert.ok(daemons.every((p) => p.daemonOf)); + assert.equal(game.state.pendingIds.length, 0); // daemonsets are not scheduled via the queue + + const gpu = game.addNode("gpu-xlarge"); + bootNode(game, gpu); + const gpuReady = game.nodeById(gpu.id); + const names = game.podsOnNode(gpuReady).filter((p) => p.kind === "daemon").map((p) => p.daemonOf); + assert.ok(names.includes("nvidia-device-plugin"), "GPU node gets the device plugin daemonset"); + assert.equal(names.length, 4); +}); + +test("daemonsets are overhead: they reduce schedulable capacity", () => { + const game = new Game("steady"); + const node = game.schedulableNodes()[0]; + const free = freeResources(node, game.podsOnNode(node)); + assert.ok(free.cpu < node.cpu, "daemonsets consume cpu"); + assert.ok(free.mem < node.mem, "daemonsets consume memory"); +}); + +test("drain keeps daemonset pods; delete purges them with no pod leak", () => { + const game = new Game("steady"); + const node = game.schedulableNodes()[0]; + const pod = createPod("frontend", makeRng(7), 0); + game.state.pods.set(pod.id, pod); + game.state.pendingIds.push(pod.id); + game.schedulePod(pod.id, node.id); + + const daemonsBefore = game.podsOnNode(node).filter((p) => p.kind === "daemon").length; + assert.ok(daemonsBefore >= 3); + game.drain(node.id); + assert.equal(pod.status, "Pending"); // workload evicted + assert.equal(game.podsOnNode(node).filter((p) => p.kind === "daemon").length, daemonsBefore); // daemons stay + + const daemonIds = game.podsOnNode(node).filter((p) => p.kind === "daemon").map((p) => p.id); + game.deleteNode(node.id); + for (const id of daemonIds) assert.equal(game.state.pods.has(id), false); +}); + +test("spot reclamation evicts workload, drops the node, and is counted", () => { + const game = new Game("steady"); + const spot = game.addNode("spot-medium"); + bootNode(game, spot); + const node = game.nodeById(spot.id); + assert.equal(node.status, "Ready"); + + const pod = createPod("batch", makeRng(9), game.state.tick); // batch tolerates spot + game.state.pods.set(pod.id, pod); + game.state.pendingIds.push(pod.id); + assert.equal(game.schedulePod(pod.id, node.id).ok, true); + + const daemonIds = game.podsOnNode(node).filter((p) => p.kind === "daemon").map((p) => p.id); + game.reclaimSpotNode(node); + assert.ok(!game.nodeById(spot.id)); // node is gone + assert.equal(pod.status, "Pending"); + assert.ok(game.state.pendingIds.includes(pod.id)); + assert.equal(game.state.metrics.spotReclaims, 1); + for (const id of daemonIds) assert.equal(game.state.pods.has(id), false); +}); + +test("spot price fluctuates over time but stays in bounds", () => { + const game = new Game("steady"); + const seen = new Set(); + for (let i = 0; i < 300; i++) { + game.tick(); + assert.ok(game.state.spotPrice >= 0.2 && game.state.spotPrice <= 3); + seen.add(game.state.spotPrice.toFixed(3)); + } + assert.ok(seen.size > 5, "spot price should vary"); +}); + +test("upgradeNode drains workload and reboots onto the new version", () => { + const game = new Game("steady"); + const node = game.schedulableNodes()[0]; + const pod = createPod("api", makeRng(11), 0); + game.state.pods.set(pod.id, pod); + game.state.pendingIds.push(pod.id); + game.schedulePod(pod.id, node.id); + + game.triggerUpgrade(); + const target = game.state.clusterMinor; + assert.equal(game.state.upgradePending, true); + assert.ok(node.minor < target); + + const r = game.upgradeNode(node.id); + assert.equal(r.ok, true); + assert.equal(node.status, "Upgrading"); + assert.equal(pod.status, "Pending"); // workload gracefully drained + assert.equal(game.podsOnNode(node).filter((p) => p.kind === "daemon").length, 0); + + bootNode(game, node); + assert.equal(node.status, "Ready"); + assert.equal(node.minor, target); + assert.ok(game.podsOnNode(node).filter((p) => p.kind === "daemon").length >= 3); // daemons recreated + assert.ok(game.state.metrics.nodesUpgraded >= 1); +}); + +test("completing a version rollout awards the bonus exactly once", () => { + const game = new Game("steady"); + game.triggerUpgrade(); + const m = game.state.clusterMinor; + const before = game.state.breakdown.upgrade; + for (const n of game.state.nodes) n.minor = m; // pretend every node was restarted + game.checkUpgradeComplete(); + assert.equal(game.state.upgradePending, false); + assert.equal(game.state.breakdown.upgrade - before, 35); + game.checkUpgradeComplete(); // idempotent + assert.equal(game.state.breakdown.upgrade - before, 35); +}); + +test("auto rolling upgrade brings the whole fleet current under automation", () => { + const game = new Game("steady"); + game.state.autoSchedule = true; + game.state.autoScale = true; + for (let i = 0; i < 60; i++) game.tick(); // warm up + game.triggerUpgrade(); + const target = game.state.clusterMinor; + let done = false; + for (let i = 0; i < 250 && !done; i++) { + game.tick(); + done = !game.state.upgradePending; + } + assert.equal(done, true, "rollout should finish"); + assert.ok(game.state.nodes.every((n) => n.minor >= target)); + assert.ok(game.state.metrics.nodesUpgraded >= 1); +}); + test("soak: every scenario runs 1200 ticks under full automation without overcommit", () => { for (const id of Object.keys(SCENARIOS)) { const game = new Game(id);