Security auditing and log recording

The Importance of Security Auditing and Logging

Security auditing and logging are critical components of ensuring system security. By recording various events during system operation, developers can track abnormal behavior, analyze security vulnerabilities, and quickly identify the root cause when issues occur. Koa2, as a lightweight Node.js framework, provides a flexible middleware mechanism that facilitates the implementation of security auditing and logging functionalities.

Basic Implementation of Logging

In Koa2, request logging can be implemented using middleware. Below is a simple example of a logging middleware:

const fs = require('fs');  
const path = require('path');  

async function logger(ctx, next) {  
  const start = Date.now();  
  await next();  
  const ms = Date.now() - start;  

  const logEntry = `${new Date().toISOString()} - ${ctx.method} ${ctx.url} - ${ctx.status} - ${ms}ms\n`;  

  fs.appendFileSync(  
    path.join(__dirname, 'access.log'),  
    logEntry,  
    { flag: 'a' }  
  );  

  console.log(logEntry.trim());  
}  

// Usage in a Koa2 application  
app.use(logger);  

This middleware records each request's method, URL, status code, and response time, writing the data to both a file and the console.

Key Points of Security Auditing

Security auditing should focus on the following critical aspects:

1. Sensitive Operation Logging: User logins, permission changes, data deletions, etc.
2. Anomaly Detection: Frequent failed login attempts, unusual request patterns.
3. Data Integrity Verification: Modification records of critical data.

Example Implementation of Audit Logging

Below is a more comprehensive audit logging middleware specifically designed to record sensitive operations:

const { createHmac } = require('crypto');  

function auditLogger(category) {  
  return async (ctx, next) => {  
    try {  
      await next();  

      if (shouldAudit(ctx)) {  
        const auditData = {  
          timestamp: new Date().toISOString(),  
          userId: ctx.state.user?.id || 'anonymous',  
          action: `${ctx.method} ${ctx.path}`,  
          category,  
          status: ctx.status,  
          clientIP: ctx.ip,  
          userAgent: ctx.get('User-Agent'),  
          metadata: getRelevantMetadata(ctx)  
        };  

        const logEntry = JSON.stringify(auditData);  
        const hash = createHmac('sha256', process.env.AUDIT_SECRET)  
          .update(logEntry)  
          .digest('hex');  

        storeAuditLog({ ...auditData, hash });  
      }  
    } catch (err) {  
      // Log error scenarios  
      const errorEntry = {  
        timestamp: new Date().toISOString(),  
        error: err.message,  
        stack: err.stack,  
        context: getErrorContext(ctx)  
      };  
      storeErrorLog(errorEntry);  
      throw err;  
    }  
  };  
}  

// Example usage  
app.use(auditLogger('authentication'));  

Log Storage and Rotation Strategies

In a production environment, log storage and rotation must be considered:

const { createGzip } = require('zlib');  
const { pipeline } = require('stream');  
const rotateLogs = require('file-stream-rotator');  

// Configure log rotation  
const accessLogStream = rotateLogs.getStream({  
  filename: 'logs/access-%DATE%.log',  
  frequency: 'daily',  
  date_format: 'YYYY-MM-DD',  
  size: '10M',  
  max_logs: '30d',  
  audit_file: 'logs/access-audit.json',  
  extension: '.log.gz',  
  create_symlink: true,  
  symlink_name: 'access.log'  
});  

// Compress logs  
accessLogStream.on('rotate', (oldFile, newFile) => {  
  const gzip = createGzip();  
  const source = fs.createReadStream(oldFile);  
  const destination = fs.createWriteStream(`${oldFile}.gz`);  

  pipeline(source, gzip, destination, (err) => {  
    if (!err) fs.unlinkSync(oldFile);  
  });  
});  

Security Log Analysis

After collecting logs, real-time analysis can be performed to detect anomalies:

const tail = require('tail').Tail;  
const anomalyDetector = new (require('./anomaly-detector'))();  

const logTail = new tail('logs/access.log');  

logTail.on('line', (line) => {  
  try {  
    const logEntry = JSON.parse(line);  

    // Detect abnormal login attempts  
    if (logEntry.category === 'authentication' && logEntry.status === 401) {  
      anomalyDetector.recordFailedLogin(logEntry.userId, logEntry.clientIP);  
    }  

    // Detect abnormal request rates  
    anomalyDetector.analyzeRequestPattern(logEntry);  

  } catch (err) {  
    console.error('Log parsing error:', err);  
  }  
});  

// Example anomaly detector  
class AnomalyDetector {  
  constructor() {  
    this.failedLogins = new Map();  
    this.requestRates = new Map();  
  }  

  recordFailedLogin(userId, ip) {  
    const key = `${userId || 'anonymous'}_${ip}`;  
    const count = (this.failedLogins.get(key) || 0) + 1;  
    this.failedLogins.set(key, count);  

    if (count > 5) {  
      triggerSecurityAlert(`Multiple login failures: ${key}`);  
    }  
  }  

  analyzeRequestPattern(logEntry) {  
    // Implement request pattern analysis logic  
  }  
}  

Log Access Control

Ensuring the security of logs themselves is equally important:

const secureLogs = require('koa-helmet');  

// Protect log file access  
app.use(async (ctx, next) => {  
  if (ctx.path.startsWith('/logs/')) {  
    if (!ctx.state.user || !ctx.state.user.isAdmin) {  
      ctx.throw(403, 'Unauthorized to access logs');  
    }  
  }  
  await next();  
});  

// Use HTTPS for log transmission  
app.use(secureLogs({  
  hsts: {  
    maxAge: 31536000,  
    includeSubDomains: true,  
    preload: true  
  }  
}));  

Performance Considerations and Optimization

Extensive logging may impact performance, necessitating optimization:

const { Worker } = require('worker_threads');  

// Use worker threads for log processing  
function createLogWorker() {  
  return new Worker(`  
    const { parentPort } = require('worker_threads');  
    const fs = require('fs');  

    parentPort.on('message', (logEntry) => {  
      fs.appendFile('logs/application.log',  
        JSON.stringify(logEntry) + '\\n',  
        { flag: 'a' }, () => {});  
    });  
  `);  
}  

const logWorker = createLogWorker();  

// Use worker threads in middleware  
app.use(async (ctx, next) => {  
  const start = Date.now();  
  await next();  
  const duration = Date.now() - start;  

  logWorker.postMessage({  
    timestamp: new Date(),  
    method: ctx.method,  
    path: ctx.path,  
    status: ctx.status,  
    duration,  
    ip: ctx.ip  
  });  
});  

Compliance Requirements

Depending on compliance requirements (e.g., GDPR, PCI DSS), logging must meet specific standards:

// GDPR-compliant logging  
function createGdprCompliantLogger() {  
  return async (ctx, next) => {  
    await next();  

    const logData = {  
      timestamp: new Date(),  
      event: `${ctx.method} ${ctx.path}`,  
      status: ctx.status,  
      metadata: {  
        // Avoid logging personally identifiable information  
        userId: ctx.state.user ? hashUserId(ctx.state.user.id) : null,  
        ip: anonymizeIP(ctx.ip)  
      }  
    };  

    if (ctx.path.includes('/user/') && ctx.method === 'DELETE') {  
      logData.gdprAction = 'user_deletion';  
      logData.retentionPeriod = '30d'; // Set retention period  
    }  

    storeCompliantLog(logData);  
  };  
}  

function hashUserId(id) {  
  return createHmac('sha256', process.env.PII_SALT)  
    .update(id)  
    .digest('hex');  
}  

function anonymizeIP(ip) {  
  if (ip.includes(':')) {  
    // IPv6  
    return ip.split(':').slice(0, 3).join(':') + '::';  
  }  
  // IPv4  
  return ip.split('.').slice(0, 2).join('.') + '.0.0';  
}  

Distributed System Logging

In a microservices architecture, distributed tracing must be considered:

const { v4: uuidv4 } = require('uuid');  
const { Kafka } = require('kafkajs');  

const kafka = new Kafka({  
  clientId: 'api-gateway',  
  brokers: ['kafka1:9092', 'kafka2:9092']  
});  

const producer = kafka.producer();  

// Distributed tracing middleware  
app.use(async (ctx, next) => {  
  const traceId = ctx.get('X-Trace-ID') || uuidv4();  
  const spanId = uuidv4();  

  ctx.state.trace = { traceId, spanId };  
  ctx.set('X-Trace-ID', traceId);  

  await producer.connect();  

  try {  
    await next();  

    await producer.send({  
      topic: 'audit-logs',  
      messages: [{  
        value: JSON.stringify({  
          traceId,  
          spanId,  
          timestamp: new Date(),  
          service: 'api-gateway',  
          event: 'request',  
          method: ctx.method,  
          path: ctx.path,  
          status: ctx.status,  
          duration: Date.now() - ctx.state.startTime  
        })  
      }]  
    });  
  } finally {  
    await producer.disconnect();  
  }  
});  

Log Query and Analysis Interface

Provide administrators with log query functionality:

const { createServer } = require('http');  
const { parse } = require('url');  
const next = require('next');  

const dev = process.env.NODE_ENV !== 'production';  
const app = next({ dev });  
const handle = app.getRequestHandler();  

// Log query API  
app.prepare().then(() => {  
  const server = new Koa();  

  server.use(async (ctx) => {  
    const parsedUrl = parse(ctx.url, true);  

    if (parsedUrl.pathname === '/admin/logs') {  
      if (!ctx.state.user?.isAdmin) {  
        ctx.status = 403;  
        return;  
      }  

      const { query, dateFrom, dateTo } = ctx.query;  
      const logs = await searchLogs({ query, dateFrom, dateTo });  

      ctx.body = logs;  
      ctx.type = 'application/json';  
    } else {  
      await handle(ctx.req, ctx.res, parsedUrl);  
      ctx.respond = false;  
    }  
  });  

  function searchLogs({ query, dateFrom, dateTo }) {  
    // Implement log search logic  
    // Professional tools like Elasticsearch can be used  
  }  
});