Performance bottleneck analysis and optimization

Performance Bottleneck Analysis and Optimization

Mongoose, as a widely used MongoDB object modeling tool in Node.js, may encounter various performance issues when handling complex data operations. Through systematic bottleneck identification and targeted optimization, application throughput and response speed can be significantly improved.

Query Performance Analysis

Slow queries are the primary factor in MongoDB performance issues. Use Mongoose's set() method to enable query logging:

mongoose.set('debug', function(collectionName, method, query, doc) {
  console.log(`Mongoose: ${collectionName}.${method}`, 
              JSON.stringify(query), doc);
});

Example of a problematic query:

// Problematic query: Full collection scan
const users = await User.find({ 
  age: { $gt: 18 } 
}).sort({ createdAt: -1 });

// Optimized: Add a compound index
User.schema.index({ age: 1, createdAt: -1 });

Query execution plan analysis:

const explain = await User.find({ age: { $gt: 18 } })
                         .sort({ createdAt: -1 })
                         .explain('executionStats');
console.log(explain.executionStats);

Connection Pool Tuning

Mongoose's default connection pool size is 5, which may need adjustment in high-concurrency scenarios:

const options = {
  poolSize: 50, // Maximum connections
  socketTimeoutMS: 30000,
  connectTimeoutMS: 30000,
  maxPoolSize: 100,
  minPoolSize: 10
};

mongoose.connect(uri, options);

Monitor connection status:

const conn = mongoose.connection;
conn.on('connected', () => console.log('Connection established'));
conn.on('disconnected', () => console.log('Connection disconnected'));

Bulk Operation Optimization

Avoid N+1 query issues:

// Inefficient approach
const orders = await Order.find({ userId: user.id });
for (const order of orders) {
  const product = await Product.findById(order.productId);
}

// Efficient approach: Use $in operator
const productIds = orders.map(o => o.productId);
const products = await Product.find({ 
  _id: { $in: productIds } 
});

Bulk write optimization:

// Single insert
for (const item of items) {
  await new Model(item).save();
}

// Bulk insert
await Model.insertMany(items, { ordered: false });

Middleware Performance Impact

Mongoose middleware can become a performance bottleneck:

UserSchema.pre('save', async function(next) {
  // Complex password hashing
  this.password = await bcrypt.hash(this.password, 12);
  next();
});

// Optimized: Hash only when modified
UserSchema.pre('save', async function(next) {
  if (!this.isModified('password')) return next();
  this.password = await bcrypt.hash(this.password, 12);
  next();
});

Index Strategy Optimization

Compound index design principles:

// Query pattern: find({status}).sort({createdAt})
UserSchema.index({ status: 1, createdAt: -1 });

// Covering index
UserSchema.index({ 
  email: 1, 
  name: 1 
}, { 
  unique: true,
  partialFilterExpression: { email: { $exists: true } }
});

Index maintenance:

// Get collection indexes
const indexes = await User.collection.getIndexes();

// Rebuild index
await User.collection.reIndex();

Data Model Design Optimization

Use nested documents appropriately to reduce query frequency:

// Original design
const OrderSchema = new Schema({
  userId: { type: Schema.Types.ObjectId, ref: 'User' }
});

// Optimized design: Embed frequently used data
const OrderSchema = new Schema({
  user: {
    _id: Schema.Types.ObjectId,
    name: String,
    email: String
  }
});

Reference data loading strategies:

// Eager loading
const order = await Order.findById(id).populate('user');

// Lazy loading
const order = await Order.findById(id);
const user = await User.findById(order.userId);

Streaming Large Data

Use cursors to process large datasets:

const stream = User.find({ age: { $gt: 18 } })
                  .cursor()
                  .on('data', (doc) => {
                    // Process individual document
                  })
                  .on('end', () => {
                    // Processing complete
                  });

Pagination optimization techniques:

// Traditional pagination
const page = await User.find()
                     .skip((pageNum - 1) * pageSize)
                     .limit(pageSize);

// ID-based pagination (more efficient)
const lastId = '...'; // Last record ID from previous page
const page = await User.find({ _id: { $gt: lastId } })
                     .limit(pageSize);

Caching Strategy Implementation

Query result caching example:

const cache = new Map();

async function getCachedUser(userId) {
  if (cache.has(userId)) {
    return cache.get(userId);
  }
  const user = await User.findById(userId);
  cache.set(userId, user);
  return user;
}

Redis integration:

const redis = require('redis');
const client = redis.createClient();

async function getUser(userId) {
  const key = `user:${userId}`;
  const cached = await client.get(key);
  if (cached) return JSON.parse(cached);
  
  const user = await User.findById(userId);
  await client.setEx(key, 3600, JSON.stringify(user));
  return user;
}

Monitoring and Diagnostic Tools

Use Mongoose's built-in monitoring:

mongoose.set('debug', (collection, method, query, doc) => {
  logger.debug(`Mongoose: ${collection}.${method}`, {
    query: JSON.stringify(query),
    doc
  });
});

Performance metric collection:

const start = Date.now();
await User.find({ /* query */ });
const duration = Date.now() - start;
metrics.track('query.duration', duration);

Transaction Performance Considerations

Transaction usage best practices:

const session = await mongoose.startSession();
try {
  session.startTransaction();
  
  await User.updateOne(
    { _id: userId },
    { $inc: { balance: -amount } },
    { session }
  );

  await Payment.create([{
    userId,
    amount,
    status: 'completed'
  }], { session });

  await session.commitTransaction();
} catch (err) {
  await session.abortTransaction();
  throw err;
} finally {
  session.endSession();
}

Aggregation Pipeline Optimization

Complex aggregation query example:

const results = await Order.aggregate([
  { $match: { status: 'completed' } },
  { $group: { 
    _id: '$userId', 
    total: { $sum: '$amount' } 
  }},
  { $sort: { total: -1 } },
  { $limit: 10 }
]).allowDiskUse(true);

Aggregation stage optimization techniques:

// Filter early
{ $match: { createdAt: { $gt: new Date('2023-01-01') } } }

// Reduce fields
{ $project: { _id: 1, name: 1 } }

// Use indexes
{ $sort: { indexedField: 1 } }

Connection Management Strategy

Multi-database connection configuration:

const primaryDB = mongoose.createConnection(primaryURI, {
  readPreference: 'primary',
  poolSize: 20
});

const replicaDB = mongoose.createConnection(replicaURI, {
  readPreference: 'secondary',
  poolSize: 30
});

Connection health checks:

setInterval(async () => {
  try {
    await mongoose.connection.db.admin().ping();
  } catch (err) {
    reconnectDatabase();
  }
}, 30000);

Document Size Control

Large document handling solutions:

// Original large document
const BlogPost = new Schema({
  title: String,
  content: String, // Potentially very large
  comments: [CommentSchema]
});

// Optimized solution: Separate large fields
const BlogPost = new Schema({
  title: String,
  contentRef: { type: Schema.Types.ObjectId } // Points to separate collection
});

GridFS integration:

const mongoose = require('mongoose');
const Grid = require('gridfs-stream');

const conn = mongoose.createConnection();
const gfs = Grid(conn.db, mongoose.mongo);

const writeStream = gfs.createWriteStream({
  filename: 'large-file.txt'
});

fs.createReadStream('./large-file.txt').pipe(writeStream);

Prefetching and Lazy Loading Strategies

Data loading pattern selection:

// Prefetch all related data
const order = await Order.findById(id)
  .populate('user')
  .populate('products');

// On-demand loading
const order = await Order.findById(id);
const user = order.userId && await User.findById(order.userId);

Virtual field optimization:

UserSchema.virtual('profile').get(function() {
  return {
    name: this.name,
    avatar: this.avatarUrl
  };
}).set(function(v) {
  this.name = v.name;
  this.avatarUrl = v.avatar;
});

Sharded Cluster Adaptation

Sharded collection configuration:

const userSchema = new Schema({
  _id: { type: String, required: true },
  email: { type: String, unique: true },
  tenantId: { type: String, required: true }
});

// Shard by tenant
userSchema.index({ tenantId: 1, _id: 1 });

Cross-shard transactions:

const session = await mongoose.startSession();
session.startTransaction();

try {
  await Order.updateOne(
    { _id: orderId },
    { $set: { status: 'shipped' } },
    { session }
  );

  await Inventory.updateOne(
    { productId },
    { $inc: { quantity: -1 } },
    { session }
  );

  await session.commitTransaction();
} catch (err) {
  await session.abortTransaction();
  throw err;
}

Read/Write Separation Configuration

Read preference settings:

const readOptions = {
  readPreference: 'secondary',
  readPreferenceTags: [{ region: 'east' }]
};

const users = await User.find({}, null, readOptions);

Connection-level configuration:

const replicaConn = mongoose.createConnection(replicaURI, {
  readPreference: 'secondaryPreferred',
  replicaSet: 'rs0'
});

Performance Testing Methodology

Benchmark implementation:

const { performance } = require('perf_hooks');

async function runBenchmark() {
  const start = performance.now();
  
  // Test query
  await User.find({ age: { $gt: 30 } });
  
  const duration = performance.now() - start;
  console.log(`Query duration: ${duration.toFixed(2)}ms`);
}

// Stress test
for (let i = 0; i < 1000; i++) {
  runBenchmark();
}

Comparative testing framework:

const benchmark = require('benchmark');
const suite = new benchmark.Suite();

suite.add('findById', {
  defer: true,
  fn: async (deferred) => {
    await User.findById('507f1f77bcf86cd799439011');
    deferred.resolve();
  }
});

suite.add('findOne', {
  defer: true,
  fn: async (deferred) => {
    await User.findOne({ _id: '507f1f77bcf86cd799439011' });
    deferred.resolve();
  }
});

suite.on('cycle', event => {
  console.log(String(event.target));
}).run();