Skip to content

Commit d4990c2

Browse files
carbonrobotcarbonrobot
authored
feat: allow customizing log formats (#21)
* feat: allow customizing log formats * chore: update documentation * chore: fix typo in documentation on formatters * refactor!: fix spelling of interface BREAKING CHANGE: fix the spelling of the LambdaEvent interface
1 parent 65f3089 commit d4990c2

File tree

10 files changed

+311
-146
lines changed

10 files changed

+311
-146
lines changed

‎README.md‎

Lines changed: 61 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ pino-lambda
66

77
A lightweight drop-in decorator for [pino](https://github.com/pinojs/pino) that takes advantage of the unique environment in AWS Lambda functions.
88

9-
This wrapper reformats the default pino output so it matches the existing Cloudwatch format. The default pino configuration [loses some of the built in support for request ID tracing](https://github.com/pinojs/pino/issues/648) that lambda has built into Cloudwatch insights.
9+
By default, this wrapper reformats the log output so it matches the existing Cloudwatch format. The default pino configuration [loses some of the built in support for request ID tracing](https://github.com/pinojs/pino/issues/648) that lambda has built into Cloudwatch insights. This can be disabled or customized as needed.
1010

1111
It also tracks the request id, correlation ids, and xray tracing from upstream services, and can be set to debug mode by upstream services on a per-request basis.
1212

@@ -44,7 +44,7 @@ other Cloudwatch aware tools such as Datadog and Splunk.
4444
"x-correlation-id": "238da608-0542-11e9-8eb2-f2801f1b9fd1",
4545
"x-correlation-trace-id": "Root=1-5c1bcbd2-9cce3b07143efd5bea1224f2;Parent=07adc05e4e92bf13;Sampled=1",
4646
"level": 30,
47-
"message": "Some A log message",
47+
"message": "A log message",
4848
"data": "Some data"
4949
}
5050
```
@@ -89,7 +89,7 @@ Cloudwatch Output
8989
"x-correlation-id": "238da608-0542-11e9-8eb2-f2801f1b9fd1",
9090
"x-correlation-trace-id": "Root=1-5c1bcbd2-9cce3b07143efd5bea1224f2;Parent=07adc05e4e92bf13;Sampled=1",
9191
"level": 30,
92-
"message": "Some A log message",
92+
"message": "A log message",
9393
"data": "Some data"
9494
}
9595
```
@@ -151,7 +151,64 @@ Output
151151
"level": 30,
152152
"host": "www.host.com",
153153
"brand": "famicom",
154-
"message": "Some A log message",
154+
"message": "A log message",
155+
"data": "Some data"
156+
}
157+
```
158+
159+
## Customize output format
160+
161+
If you want the request tracing features, but don't need the Cloudwatch format, you can use the default pino formatter, or supply your own formatter.
162+
163+
```ts
164+
// default Pino formatter for logs
165+
import pino, { PinoLogFormatter } from 'pino-lambda';
166+
const logger = pino({
167+
formatter: new PinoLogFormatter(),
168+
});
169+
```
170+
171+
Output
172+
173+
```
174+
{
175+
"awsRequestId": "6fccb00e-0479-11e9-af91-d7ab5c8fe19e",
176+
"x-correlation-trace-id": "Root=1-5c1bcbd2-9cce3b07143efd5bea1224f2;Parent=07adc05e4e92bf13;Sampled=1",
177+
"level": 30,
178+
"host": "www.host.com",
179+
"brand": "famicom",
180+
"message": "A log message",
181+
"data": "Some data"
182+
}
183+
```
184+
185+
The formatter function can be replaced with any custom implementation you require by using the supplied interface.
186+
187+
```ts
188+
import pino, { ExtendedPinoLambdaOptions, ILogFormatter } from 'pino-lambda';
189+
190+
class BananaLogFormatter implements ILogFormatter {
191+
format(buffer: string, options: ExtendedPinoLambdaOptions) {
192+
return `[BANANA] ${buffer}`;
193+
}
194+
}
195+
196+
const logger = pino({
197+
formatter: new BananaLogFormatter(),
198+
});
199+
```
200+
201+
Output
202+
203+
```
204+
[BANANA]
205+
{
206+
"awsRequestId": "6fccb00e-0479-11e9-af91-d7ab5c8fe19e",
207+
"x-correlation-trace-id": "Root=1-5c1bcbd2-9cce3b07143efd5bea1224f2;Parent=07adc05e4e92bf13;Sampled=1",
208+
"level": 30,
209+
"host": "www.host.com",
210+
"brand": "famicom",
211+
"message": "A log message",
155212
"data": "Some data"
156213
}
157214
```

‎src/formatters/cloudwatch.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
import pino from 'pino';
2+
import { GlobalContextStorageProvider } from '../context';
3+
import { ILogFormatter, ExtendedPinoOptions } from '../types';
4+
5+
const formatLevel = (level: string | number): string => {
6+
if (typeof level === 'string') {
7+
return level.toLocaleUpperCase();
8+
} else if (typeof level === 'number') {
9+
return pino.levels.labels[level]?.toLocaleUpperCase();
10+
}
11+
return level;
12+
};
13+
14+
/**
15+
* Formats the log in native cloudwatch format.
16+
* Default format for pino-lambda
17+
*/
18+
export class CloudwatchLogFormatter implements ILogFormatter {
19+
format(buffer: string, options: ExtendedPinoOptions): string {
20+
/**
21+
* Writes to stdout using the same method that AWS lambda uses
22+
* under the hood for console.log
23+
* This preserves the default log format of cloudwatch
24+
*/
25+
let output = buffer;
26+
const { level, msg } = JSON.parse(buffer);
27+
const storageProvider = options.storageProvider || GlobalContextStorageProvider;
28+
const { awsRequestId } = storageProvider.getContext() || {};
29+
const time = new Date().toISOString();
30+
const levelTag = formatLevel(level);
31+
32+
output = `${time}${awsRequestId ? `\t${awsRequestId}` : ''}\t${levelTag}\t${msg}\t${buffer}`;
33+
return output;
34+
}
35+
}

‎src/formatters/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
export * from './cloudwatch';
2+
export * from './pino';

‎src/formatters/pino.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import { ILogFormatter } from '../types';
2+
3+
/**
4+
* Formats the log in native pino format
5+
*/
6+
export class PinoLogFormatter implements ILogFormatter {
7+
format(buffer: string): string {
8+
return buffer;
9+
}
10+
}

‎src/index.ts‎

Lines changed: 13 additions & 141 deletions
Original file line numberDiff line numberDiff line change
@@ -1,94 +1,17 @@
1-
import pino, { DestinationStream, LevelMapping, LoggerOptions, Logger } from 'pino';
2-
import { GlobalContextStorageProvider, ContextStorageProvider, ContextMap } from './context';
3-
4-
export interface ExtendedPinoOptions extends LoggerOptions {
5-
requestMixin?: (
6-
event: LamdbaEvent,
7-
context: LambdaContext,
8-
) => { [key: string]: string | undefined };
9-
storageProvider?: ContextStorageProvider;
10-
streamWriter?: (str: string | Uint8Array) => boolean;
11-
}
12-
13-
interface LambdaContext {
14-
awsRequestId: string;
15-
16-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
17-
[key: string]: any;
18-
}
19-
20-
interface LamdbaEvent {
21-
headers?: {
22-
[key: string]: string | undefined;
23-
};
24-
25-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
26-
[key: string]: any;
27-
}
28-
29-
interface PinoLambdaExtensionOptions {
30-
levels: LevelMapping;
31-
options: ExtendedPinoOptions;
32-
}
33-
34-
const AMAZON_TRACE_ID = '_X_AMZN_TRACE_ID';
35-
const CORRELATION_HEADER = 'x-correlation-';
36-
const CORRELATION_ID = `${CORRELATION_HEADER}id`;
37-
const CORRELATION_TRACE_ID = `${CORRELATION_HEADER}trace-id`;
38-
const CORRELATION_DEBUG = `${CORRELATION_HEADER}debug`;
39-
40-
const formatLevel = (level: string | number, levels: LevelMapping): string => {
41-
if (typeof level === 'string') {
42-
return level.toLocaleUpperCase();
43-
} else if (typeof level === 'number') {
44-
return levels.labels[level]?.toLocaleUpperCase();
45-
}
46-
return level;
47-
};
48-
49-
/**
50-
* Custom destination stream for Pino
51-
* @param options Pino options
52-
* @param storageProvider Global storage provider for request values
53-
*/
54-
const pinolambda = ({ levels, options }: PinoLambdaExtensionOptions): DestinationStream => ({
55-
write(buffer: string) {
56-
let output = buffer;
57-
if (!options.prettyPrint) {
58-
/**
59-
* Writes to stdout using the same method that AWS lambda uses
60-
* under the hood for console.log
61-
* This preserves the default log format of cloudwatch
62-
*/
63-
const { level, msg } = JSON.parse(buffer);
64-
const storageProvider = options.storageProvider || GlobalContextStorageProvider;
65-
const { awsRequestId } = storageProvider.getContext() || {};
66-
const time = new Date().toISOString();
67-
const levelTag = formatLevel(level, levels);
68-
69-
output = `${time}${awsRequestId ? `\t${awsRequestId}` : ''}\t${levelTag}\t${msg}\t${buffer}`;
70-
output = output.replace(/\n/, '\r');
71-
output += '\n';
72-
}
73-
74-
if (options.streamWriter) {
75-
return options.streamWriter(output);
76-
}
77-
return process.stdout.write(output);
78-
},
79-
});
80-
81-
export type PinoLambdaLogger = Logger & {
82-
withRequest: (event: LamdbaEvent, context: LambdaContext) => void;
83-
};
1+
import pino from 'pino';
2+
import { GlobalContextStorageProvider } from './context';
3+
import { ExtendedPinoOptions, PinoLambdaLogger } from './types';
4+
import { withRequest } from './request';
5+
import { createStream } from './stream';
846

857
/**
868
* Exports a default constructor with an extended instance of Pino
879
* that provides convinience methods for use with AWS Lambda
8810
*/
8911
export default (extendedPinoOptions?: ExtendedPinoOptions): PinoLambdaLogger => {
9012
const options = extendedPinoOptions ?? {};
91-
const storageProvider = extendedPinoOptions?.storageProvider || GlobalContextStorageProvider;
13+
const storageProvider = (options.storageProvider =
14+
extendedPinoOptions?.storageProvider || GlobalContextStorageProvider);
9215

9316
// attach request values to logs
9417
const pinoOptions = {
@@ -104,65 +27,14 @@ export default (extendedPinoOptions?: ExtendedPinoOptions): PinoLambdaLogger =>
10427
};
10528

10629
// construct a pino logger and set its destination
107-
const logger = (pino(
108-
pinoOptions,
109-
pinolambda({ options: pinoOptions, levels: pino.levels }),
110-
) as unknown) as PinoLambdaLogger;
111-
112-
// keep a reference to the original logger level
113-
const configuredLevel = logger.level;
30+
const logger = (pino(pinoOptions, createStream(pinoOptions)) as unknown) as PinoLambdaLogger;
11431

11532
// extend the base logger
116-
logger.withRequest = (event: LamdbaEvent, context: LambdaContext): void => {
117-
const ctx: ContextMap = {
118-
awsRequestId: context.awsRequestId,
119-
};
120-
121-
// capture api gateway request ID
122-
const apiRequestId = event.requestContext?.requestId;
123-
if (apiRequestId) {
124-
ctx.apiRequestId = apiRequestId;
125-
}
126-
127-
// capture any correlation headers sent from upstream callers
128-
if (event.headers) {
129-
Object.keys(event.headers).forEach((header) => {
130-
if (header.toLowerCase().startsWith(CORRELATION_HEADER)) {
131-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
132-
ctx[header] = event.headers![header] as string;
133-
}
134-
});
135-
}
136-
137-
// capture the xray trace id if its enabled
138-
if (process.env[AMAZON_TRACE_ID]) {
139-
ctx[CORRELATION_TRACE_ID] = process.env[AMAZON_TRACE_ID] as string;
140-
}
33+
logger.withRequest = withRequest(logger, pinoOptions);
14134

142-
// set the correlation id if not already set by upstream callers
143-
if (!ctx[CORRELATION_ID]) {
144-
ctx[CORRELATION_ID] = context.awsRequestId;
145-
}
146-
147-
// if an upstream service requests DEBUG mode,
148-
// dynamically modify the logging level
149-
if (ctx[CORRELATION_DEBUG] === 'true') {
150-
logger.level = 'debug';
151-
} else {
152-
logger.level = configuredLevel;
153-
}
154-
155-
// handle custom request level mixins
156-
if (pinoOptions.requestMixin) {
157-
const result = pinoOptions.requestMixin(event, context);
158-
for (const key in result) {
159-
// Cast this to string for typescript
160-
// when the JSON serializer runs, by default it omits undefined properties
161-
ctx[key] = result[key] as string;
162-
}
163-
}
164-
165-
storageProvider.setContext(ctx);
166-
};
16735
return logger;
16836
};
37+
38+
// reexport all public types
39+
export * from './formatters';
40+
export * from './types';

‎src/request.ts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import { ContextMap } from './context';
2+
import { LambdaEvent, LambdaContext, PinoLambdaLogger, ExtendedPinoOptions } from './types';
3+
4+
const AMAZON_TRACE_ID = '_X_AMZN_TRACE_ID';
5+
const CORRELATION_HEADER = 'x-correlation-';
6+
const CORRELATION_ID = `${CORRELATION_HEADER}id`;
7+
const CORRELATION_TRACE_ID = `${CORRELATION_HEADER}trace-id`;
8+
const CORRELATION_DEBUG = `${CORRELATION_HEADER}debug`;
9+
10+
export const withRequest = (logger: PinoLambdaLogger, options: ExtendedPinoOptions) => (
11+
event: LambdaEvent,
12+
context: LambdaContext,
13+
): void => {
14+
// keep a reference to the original logger level
15+
const configuredLevel = logger.level;
16+
17+
const ctx: ContextMap = {
18+
awsRequestId: context.awsRequestId,
19+
};
20+
21+
// capture api gateway request ID
22+
const apiRequestId = event.requestContext?.requestId;
23+
if (apiRequestId) {
24+
ctx.apiRequestId = apiRequestId;
25+
}
26+
27+
// capture any correlation headers sent from upstream callers
28+
if (event.headers) {
29+
Object.keys(event.headers).forEach((header) => {
30+
if (header.toLowerCase().startsWith(CORRELATION_HEADER)) {
31+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
32+
ctx[header] = event.headers![header] as string;
33+
}
34+
});
35+
}
36+
37+
// capture the xray trace id if its enabled
38+
if (process.env[AMAZON_TRACE_ID]) {
39+
ctx[CORRELATION_TRACE_ID] = process.env[AMAZON_TRACE_ID] as string;
40+
}
41+
42+
// set the correlation id if not already set by upstream callers
43+
if (!ctx[CORRELATION_ID]) {
44+
ctx[CORRELATION_ID] = context.awsRequestId;
45+
}
46+
47+
// if an upstream service requests DEBUG mode,
48+
// dynamically modify the logging level
49+
if (ctx[CORRELATION_DEBUG] === 'true') {
50+
logger.level = 'debug';
51+
} else {
52+
logger.level = configuredLevel;
53+
}
54+
55+
// handle custom request level mixins
56+
if (options.requestMixin) {
57+
const result = options.requestMixin(event, context);
58+
for (const key in result) {
59+
// Cast this to string for typescript
60+
// when the JSON serializer runs, by default it omits undefined properties
61+
ctx[key] = result[key] as string;
62+
}
63+
}
64+
65+
if (options.storageProvider) {
66+
options.storageProvider.setContext(ctx);
67+
}
68+
};

0 commit comments

Comments
 (0)