profile-workspace.js

/**
 * Run node ./dev/ide/capture-ws-top.js to profile cpu and memory usage of the workspace.
 *
 */
//@ts-check

const { execSync } = require("child_process");
const { promises } = require("fs");
const fetch = require("node-fetch");
const path = require("path");

/**
 * @param {number[]} arr
 * @returns {number}
 */
const q90 = (arr) => {
    const sorted = arr.sort((a, b) => a - b);
    const pos = (sorted.length - 1) * 0.9;
    const base = Math.floor(pos);
    const rest = pos - base;
    if (sorted[base + 1] !== undefined) {
        return sorted[base] + rest * (sorted[base + 1] - sorted[base]);
    } else {
        return sorted[base];
    }
};

(async () => {
    let wsPodName;
    let creationTimestamp;
    while (!wsPodName) {
        const segments = execSync(
            "kubectl get pod -l component=workspace -o=custom-columns=:metadata.name,:metadata.creationTimestamp",
            {
                encoding: "utf8",
            },
        )
            .trim()
            .split(/\s+/);
        wsPodName = segments[0];
        creationTimestamp = new Date(segments[1]);
        await new Promise((r) => setTimeout(r, 1000));
    }
    console.log(wsPodName);
    console.log(creationTimestamp);

    const query = async (prefix) => {
        const age = ((new Date().getTime() - creationTimestamp.getTime()) / 1000).toFixed(0);
        const response = await fetch.default(
            encodeURI(`http://localhost:9090/api/v1/query?query=${prefix}{pod="${wsPodName}"}[${age}s])/(1024*1024)`),
            {
                method: "GET",
            },
        );
        if (!response.ok) {
            console.error(`${prefix}: ${response.statusText} (${response.status})`);
            return "N/A";
        }
        /**
         * @type {{data: {result: { value: [number, string] }[]}}}
         */
        const body = await response.json();
        return Number(body.data.result[0].value[1]).toFixed(2);
    };

    const perfLogPath = path.resolve(__dirname, "perf.log");
    console.log(perfLogPath);
    const start = new Date().getTime();
    await promises.writeFile(perfLogPath, wsPodName + "\n", { encoding: "utf8" });
    let measurements = 0;
    const cores = [];
    const mems = [];
    let avgCores = 0;
    let maxCores = 0;
    let sumCores = 0;
    let avgMemory = 0;
    let maxMemory = 0;
    let sumMemory = 0;
    while (true) {
        measurements++;

        /**
         * @type {{memory: {used: number, limit: number}, cpu: {used: number, limit: number}} | undefined}
         */
        let top;
        try {
            const content = execSync(`kubectl exec -t ${wsPodName} -- /.supervisor/supervisor top -sj`, {
                encoding: "utf8",
            }).trim();
            const value = JSON.parse(content);
            if ("memory" in value && "cpu" in value) {
                top = value;
            }
        } catch (e) {
            console.error(e);
        }

        if (top) {
            cores.push(top.cpu.used);
            sumCores += top.cpu.used;
            avgCores = sumCores / measurements;
            maxCores = Math.max(maxCores, top.cpu.used);

            const mem = top.memory.used / (1024 * 1024);
            mems.push(mem);
            sumMemory += mem;
            avgMemory = sumMemory / measurements;
            maxMemory = Math.max(maxMemory, mem);

            const [avgMax, maxMax, q90Max, avgUsed, maxUsed, q90user] = (
                await Promise.allSettled([
                    query("avg_over_time(gitpod_jb_backend_memory_max_bytes"),
                    query("max_over_time(gitpod_jb_backend_memory_max_bytes"),
                    query("quantile_over_time(0.9, gitpod_jb_backend_memory_max_bytes"),
                    query("avg_over_time(gitpod_jb_backend_memory_used_bytes"),
                    query("max_over_time(gitpod_jb_backend_memory_used_bytes"),
                    query("quantile_over_time(0.9, gitpod_jb_backend_memory_used_bytes"),
                ])
            ).map((v) => {
                if (v.status === "fulfilled") {
                    return v.value;
                }
                console.error(v.reason);
                return "N/A";
            });

            await promises.appendFile(
                perfLogPath,
                `${((new Date().getTime() - start) / 1000).toFixed(2)}s, cpu(m) ${top.cpu.used.toFixed(
                    2,
                )}/${avgCores.toFixed(2)}/${maxCores.toFixed(2)}/${q90(cores).toFixed(2)}, memory(Mi) ${mem.toFixed(
                    2,
                )}/${avgMemory.toFixed(2)}/${maxMemory.toFixed(2)}/${q90(mems).toFixed(
                    2,
                )}, allocated(M) ${avgMax}/${maxMax}/${q90Max}, used(M) ${avgUsed}/${maxUsed}/${q90user}\n`,
            );
        }
        await new Promise((r) => setTimeout(r, 1000));
    }
})();