Koa + CLS + Log4js 实现全链路日志系统

什么是全链路日志

在线上项目运行期间，经常会出现各种莫名其妙的 bug，而且一个请求往往会经过多个项目的接口调用。比如电商中的下订单，可能会调用到商品服务、优惠券满减服务、会员服务之类的。假如某一时刻下单失败，前端报了个系统异常，怎么样快速定位到底是哪个服务发生了异常，以及定位发生异常的服务具体是报了什么异常日志呢。

这就是全链路日志要做的事情，它把这个请求内调用到的所有请求通过全局 ID 串起来，通过全局 ID 可以把所有涉及到的系统日志都快速地定位出来。

日志开发架构图

之后 Logstash 后可能需要加 Queue 为后续大数据做准备

技术栈

功能	技术栈
日志	Log4js
异步资源追踪	cls-hooked
Web 框架	Koa
Node	12
REST - 传递 header[trace-id]	Axios

思路

请求打进来，由网关下发全局唯一 ID
Koa 框架可以获取同步请求的上下文，但在异步中需要上下文持久的问题（比如 async_hook）
需要保证当前项目所有 category 的 log 可用，且打印 trace-id
需要考虑解耦，避免大量修改

开发

封装 log 包

config.json

{
  "levels": {
    "acc": {
      "value": 5000,
      "colour": "green"
    },
    "biz": {
      "value": 10000,
      "colour": "cyan"
    },
    "debug": {
      "value": 20000,
      "colour": "blue"
    },
    "warn": {
      "value": 30000,
      "colour": "yellow"
    },
    "error": {
      "value": 40000,
      "colour": "red"
    },
    "sql": {
      "value": 50000,
      "colour": "grey"
    }
  },
  "appenders": {
    "console": {
      "type": "console",
      "layout": {
        "type": "pattern",
        "pattern": "%[[%d{yyyy-MM-dd hh:mm:ss.SSS}] [%p]%] ~ %m"
      }
    },
    "acc": {
      "type": "dateFile",
      "filename": "log/acc/acc",
      "pattern": ".yyyy-MM-dd.log",
      "alwaysIncludePattern": true,
      "compress": true,
      "daysToKeep": 1,
      "layout": {
        "type": "pattern",
        "pattern": "[%d{yyyy-MM-dd hh:mm:ss.SSS}] [%p] ~ %m"
      }
    },
    "biz": {
      "type": "dateFile",
      "filename": "log/biz/biz",
      "pattern": ".yyyy-MM-dd.log",
      "alwaysIncludePattern": true,
      "compress": true,
      "daysToKeep": 1,
      "layout": {
        "type": "pattern",
        "pattern": "[%d{yyyy-MM-dd hh:mm:ss.SSS}] [%p] ~ %m"
      }
    },
    "debug": {
      "type": "dateFile",
      "filename": "log/app/debug",
      "pattern": ".yyyy-MM-dd.log",
      "alwaysIncludePattern": true,
      "compress": true,
      "daysToKeep": 1,
      "layout": {
        "type": "pattern",
        "pattern": "[%d{yyyy-MM-dd hh:mm:ss.SSS}] [%p] ~ %m"
      }
    },
    "warn": {
      "type": "dateFile",
      "filename": "log/app/warn",
      "pattern": ".yyyy-MM-dd.log",
      "alwaysIncludePattern": true,
      "compress": true,
      "daysToKeep": 1,
      "layout": {
        "type": "pattern",
        "pattern": "[%d{yyyy-MM-dd hh:mm:ss.SSS}] [%p] ~ %m"
      }
    },
    "error": {
      "type": "dateFile",
      "filename": "log/app/error",
      "pattern": ".yyyy-MM-dd.log",
      "alwaysIncludePattern": true,
      "compress": true,
      "daysToKeep": 1,
      "layout": {
        "type": "pattern",
        "pattern": "[%d{yyyy-MM-dd hh:mm:ss.SSS}] [%p] ~ %m"
      }
    },
    "sql": {
      "type": "dateFile",
      "filename": "log/sql/sql",
      "pattern": ".yyyy-MM-dd.log",
      "alwaysIncludePattern": true,
      "compress": true,
      "daysToKeep": 1,
      "layout": {
        "type": "pattern",
        "pattern": "[%d{yyyy-MM-dd hh:mm:ss.SSS}] [%p] ~ %m"
      }
    }
  },
  "categories": {
    "default": {
      "appenders": [
        "console"
      ],
      "level": "all"
    },
    "acc": {
      "appenders": [
        "acc", "console"
      ],
      "level": "acc"
    },
    "biz": {
      "appenders": [
        "biz", "console"
      ],
      "level": "biz"
    },
    "debug": {
      "appenders": [
        "debug", "console"
      ],
      "level": "debug"
    },
    "warn": {
      "appenders": [
        "warn", "console"
      ],
      "level": "warn"
    },
    "error": {
      "appenders": [
        "error", "console"
      ],
      "level": "error"
    },
    "sql": {
      "appenders": [
        "sql", "console"
      ],
      "level": "sql"
    }
  }
}

Log 包核心类

const util = require('util')
const { configure, getLogger } = require('log4js')
const config = require('./config.json')
const { createNamespace } = require('cls-hooked')
const session = createNamespace('xxx')

const localIp = getLocalIp()

function getLocalIp() {
  const os = require('os');
  const ifaces = os.networkInterfaces();
  let ip = '';
  for (let dev in ifaces) {
    for (let i = 0; i < ifaces[dev].length; i++) {
      if (!ifaces[dev][i].internal && ifaces[dev][i].family === 'IPv4' && !ifaces[dev][i].address.includes('::') && ifaces[dev][i].address !== '127.0.0.1') {
        ip = ifaces[dev][i].address;
        break;
      }
    }
  }
  return ip;
}

class Logger {
  constructor(ctx, opts) {
    configure(config);
    const self = this;
    const functions = ['acc', 'biz', 'debug', 'warn', 'error', 'sql'];
    for (const fn of functions) {
      const logger = getLogger(fn);
      Logger.prototype[fn] = function(...args) {
        for (const [i, arg] of args.entries()) {
          if (typeof arg !== "object") continue;
          args[i] = util.inspect(arg, { depth: Infinity });
        }
        // appName: module名
        logger[fn].call(logger, `${opts.appName} ~ ${session.get('requestId') || '-'} ~ ${localIp} ~`, ...args);
      };
    }
    // 声明到全局
    ctx.log = this;
  }
}

module.exports = (context, opts) => {
  const log = new Logger(context, opts);
  return async (ctx, next) => {
    ctx.log = log;
    await session.runPromise(async () => {
      // 获取 requestId
      const requestId = ctx.header['trace-id'] || '-'
      ctx.res.setHeader('trace-id', requestId)
      // CLS 中设置 requestId
      session.set('requestId', requestId)
      // 也可以去掉 async，这里直接返回 next()
    })
    try {
      await next();
    } catch (e) {
      ctx.log.error(e.message);
    }
  };
}

REST 包

const axios = require('axios')
const { getNamespace } = require('cls-hooked')
const session = getNamespace('sopei')
...
client.get(url, { params, headers: { 'trace-id': session.get('requestId') } })
  .then(res => resolve(res.data))
  .catch((err) => {
    self.ctx.log.error(err.message, 'GET', url, logParams);
    reject(err);
  });

Koa 应用

const app = new Koa();
// 确保 logger 中间件先执行
app.use(logger(app.context, { appName: xxx }));
app.use(rest(app.context))