[Node.js] Availability and Zero-downtime Restarts

It might be possible for our node server has some downtime, no matter it is because server update or simply some crashs in the code. We want to minizie the downtime as much as possible.

 

1. In case of cluster worker crash, we want master worker fork a new worker:

const http = require('http');
const cluster = require('cluster');
const os = require('os');

if (cluster.isMaster) {
    const cpus = os.cpus().length;

    console.log(`Forking for ${cpus} CPUs`);
    for (let i = 0; i < cpus; i++) {
        cluster.fork();
    }

    cluster.on('exit', (worker, code, signal) => {
        if (code !== 0 && !worker.exitedAfterDisconnect) {
            console.log(`Worker ${worker.id} crashed. Starting a new wroker`);
            cluster.fork();
        }
    })
} else {
    require('./server');
}

It is important to check 'worker.exitedAfterDisconnect' to see whether is is because crash or because we want to exit one worker.

 

2. In case of upgrade, we want to restart each worker one by one, to make zero downtime:

    // kill -SIGUSR2 <MASTER_PID>
    // In case to upgrade, we want to restart each worker one by one
    process.on('SIGUSR2', () => {
        const workers = Object.values(cluster.workers);
        const restartWorker = (workerIndex) => {
            const worker = cluster.workers[workerIndex];
            if (!worker) return;

            // On worker exit, we want to restart it, then continue 
            // with next worker
            worker.on('exit', () => {
                // If it is because crash, we don't continue
                if (!worker.exitedAfterDisconnect) return;
                console.log(`Exited process ${worker.process.pid}`);
                cluster.fork().on('listening', () => {
                    restartWorker(workerIndex + 1);
                });

                worker.disconnect();
            });
        }
        // Calling restartWorker recursively
        restartWorker(0);
    });

 

In really production, we don't actually need to code cluster by ourselve, we can use PM2 package. but it is important to understand what's happening under hood.

 

---

 

const cluster = require('cluster');
const http = require('http');
const os = require('os');

// For runing for the first time,
// Master worker will get started
// Then we can fork our new workers
if (cluster.isMaster) {
    const cpus = os.cpus().length;

    console.log(`Forking for ${cpus} CPUs`);
    for (let i = 0; i < cpus; i++) {
        cluster.fork();
    }

    // In case of crash, we want to strat a new worker
    cluster.on('exit', (worker, code, signal) => {
        if (code !== 0 && !worker.exitedAfterDisconnect) {
            console.log(`Worker ${worker.id} crashed. Starting a new wroker`);
            cluster.fork();
        }
    })

    // kill -SIGUSR2 <MASTER_PID>
    // In case to upgrade, we want to restart each worker one by one
    process.on('SIGUSR2', () => {
        const workers = Object.values(cluster.workers);
        const restartWorker = (workerIndex) => {
            const worker = cluster.workers[workerIndex];
            if (!worker) return;

            // On worker exit, we want to restart it, then continue 
            // with next worker
            worker.on('exit', () => {
                // If it is because crash, we don't continue
                if (!worker.exitedAfterDisconnect) return;
                console.log(`Exited process ${worker.process.pid}`);
                cluster.fork().on('listening', () => {
                    restartWorker(workerIndex + 1);
                });

                worker.disconnect();
            });
        }
        // Calling restartWorker recursively
        restartWorker(0);
    });
} else {
    require('./server');
}