Cluster mode and load balancing

Cluster Mode and Load Balancing

As a lightweight Node.js framework, Koa2 can become a performance bottleneck when handling high-concurrency requests in single-process mode. By utilizing cluster mode and load balancing techniques, we can fully leverage multi-core CPU resources, significantly improving application throughput and stability.

Basic Implementation of Cluster Mode

Node.js's cluster module allows creating child processes that share the same port. The master process manages worker processes, with each worker being an independent V8 instance. The basic implementation is as follows:

const cluster = require('cluster');
const os = require('os');
const Koa = require('koa');

if (cluster.isMaster) {
  const cpuCount = os.cpus().length;
  
  for (let i = 0; i < cpuCount; i++) {
    cluster.fork();
  }

  cluster.on('exit', (worker) => {
    console.log(`Worker ${worker.id} died`);
    cluster.fork();
  });
} else {
  const app = new Koa();
  
  app.use(async ctx => {
    ctx.body = `Worker ${cluster.worker.id} handled this request`;
  });

  app.listen(3000);
}

In this mode, the operating system distributes requests to worker processes in a round-robin fashion. When a worker crashes, the master process immediately restarts a new worker instance.

Advanced Load Balancing Strategies

Beyond the default round-robin strategy, more sophisticated load control can be implemented:

1. Dynamic Allocation Based on Connection Count

// Master process
const workers = {};
const createWorker = () => {
  const worker = cluster.fork();
  workers[worker.id] = { 
    conns: 0,
    instance: worker
  };
  
  worker.on('message', (msg) => {
    if (msg.type === 'updateConn') {
      workers[worker.id].conns = msg.count;
    }
  });
};

// Select worker with fewest connections
const getWorker = () => {
  return Object.values(workers)
    .sort((a, b) => a.conns - b.conns)[0]
    .instance;
};

2. Weighted Allocation Based on Response Time

// Worker process
let responseTimes = [];
setInterval(() => {
  process.send({
    type: 'perfMetrics',
    avgTime: responseTimes.reduce((a,b) => a+b, 0) / responseTimes.length || 0
  });
  responseTimes = [];
}, 5000);

app.use(async (ctx, next) => {
  const start = Date.now();
  await next();
  const duration = Date.now() - start;
  responseTimes.push(duration);
  ctx.set('X-Response-Time', `${duration}ms`);
});

Inter-Process Communication Optimization

Special attention is needed for state sharing in cluster mode:

Using Redis for Session Sharing

const session = require('koa-session');
const RedisStore = require('koa-redis');

app.keys = ['some secret key'];
app.use(session({
  store: new RedisStore({
    host: '127.0.0.1',
    port: 6379,
    ttl: 86400 * 30
  }),
  key: 'koa:sess'
}, app));

Event Broadcasting Mechanism

// Master process
cluster.on('message', (worker, message) => {
  if (message.type === 'broadcast') {
    for (const id in cluster.workers) {
      cluster.workers[id].send(message);
    }
  }
});

// Worker process
process.on('message', (msg) => {
  if (msg.event === 'configUpdate') {
    reloadConfig(msg.data);
  }
});

function broadcast(data) {
  process.send({
    type: 'broadcast',
    event: 'configUpdate',
    data: data
  });
}

Zero-Downtime Deployment Solution

Seamless restarts can be achieved through load balancers:

// Graceful worker shutdown
process.on('SIGTERM', () => {
  server.close(() => {
    process.exit(0);
  });
  
  setTimeout(() => {
    process.exit(1);
  }, 5000);
});

// Rolling restart strategy
let restartQueue = [];
function rollingRestart() {
  const workers = Object.values(cluster.workers);
  restartQueue = [...workers];
  
  const restartNext = () => {
    const worker = restartQueue.pop();
    if (!worker) return;
    
    worker.once('disconnect', () => {
      const newWorker = cluster.fork();
      newWorker.once('listening', restartNext);
    });
    
    worker.send('shutdown');
  };
  
  restartNext();
}

Performance Monitoring and Tuning

A comprehensive monitoring system is crucial for cluster management:

// Collect worker metrics
const stats = {};
setInterval(() => {
  for (const worker of Object.values(cluster.workers)) {
    worker.send({ type: 'getStats' });
  }
}, 10000);

cluster.on('message', (worker, msg) => {
  if (msg.type === 'stats') {
    stats[worker.id] = {
      memory: msg.memory,
      load: msg.load,
      uptime: msg.uptime
    };
  }
});

// Worker-side implementation
const os = require('os');
setInterval(() => {
  const mem = process.memoryUsage();
  const load = os.loadavg()[0];
  
  process.send({
    type: 'stats',
    memory: {
      rss: mem.rss,
      heapTotal: mem.heapTotal,
      heapUsed: mem.heapUsed
    },
    load: load,
    uptime: process.uptime()
  });
}, 5000);

Containerized Deployment Considerations

Special handling is required when running clusters in Docker environments:

# Example Dockerfile
FROM node:14
WORKDIR /app
COPY package*.json ./
RUN npm install
COPY . .
CMD ["node", "cluster.js"]

Note CPU limits when starting:

# Set according to actual CPU cores
docker run -e NODE_CLUSTER_SCHED_POLICY='rr' -e NODE_CLUSTER_WORKERS='max' -p 3000:3000 app

Common Problem Solutions

Port Conflict Issues

// Use SO_REUSEPORT option (Node.js 13.0+)
const net = require('net');
const server = net.createServer({ reusePort: true });
server.listen(3000);

File Descriptor Limits

# Check current limit
ulimit -n

# Temporarily increase limit
ulimit -n 100000

Memory Leak Troubleshooting

const heapdump = require('heapdump');
process.on('SIGUSR2', () => {
  const filename = `/tmp/heapdump-${process.pid}-${Date.now()}.heapsnapshot`;
  heapdump.writeSnapshot(filename);
  console.log(`Heap dump written to ${filename}`);
});