🚨 Design workflows of slog handlers: pipeline, middleware, fanout, routing, failover, load balancing...
Design workflows of slog handlers:
log.Record
to multiple slog.Handler
in parallellog.Record
on the fly (eg: for privacy reason)log.Record
to all matching slog.Handler
log.Record
to the first available slog.Handler
log.Record
to a pool of slog.Handler
Here a simple workflow with both pipeline and fanout:
See also:
slog.Handler
chaining, fanout, routing, failover, load balancing...slog
attribute formattingslog
sampling policyslog
loggerslog
loggerslog
loggerslog
handler for Datadog
slog
handler for Rollbar
slog
handler for Sentry
slog
handler for Syslog
slog
handler for Logstash
slog
handler for Fluentd
slog
handler for Graylog
slog
handler for Loki
slog
handler for Slack
slog
handler for Telegram
slog
handler for Mattermost
slog
handler for Microsoft Teams
slog
handler for Webhook
slog
handler for Kafka
go get github.com/samber/slog-multi
Compatibility: go >= 1.20.3
This library is v0 and follows SemVer strictly. On slog
final release (go 1.21), this library will go v1.
No breaking changes will be made to exported APIs before v1.0.0.
⚠️ Use this library carefully, log processing can be very costly (!)
GoDoc: https://pkg.go.dev/github.com/samber/slog-multi
slogmulti.Fanout()
Distribute logs to multiple slog.Handler
in parallel.
import (
slogmulti "github.com/samber/slog-multi"
"golang.org/x/exp/slog"
)
func main() {
logstash, _ := net.Dial("tcp", "logstash.acme:4242") // use github.com/netbrain/goautosocket for auto-reconnect
stderr := os.Stderr
logger := slog.New(
slogmulti.Fanout(
slog.HandlerOptions{}.NewJSONHandler(logstash), // pass to first handler: logstash over tcp
slog.HandlerOptions{}.NewTextHandler(stderr), // then to second handler: stderr
// ...
),
)
logger.
With(
slog.Group("user",
slog.String("id", "user-123"),
slog.Time("created_at", time.Now()),
),
).
With("environment", "dev").
With("error", fmt.Errorf("an error")).
Error("A message")
}
Stderr output:
time=2023-04-10T14:00:0.000000+00:00 level=ERROR msg="A message" user.id=user-123 user.created_at=2023-04-10T14:00:0.000000+00:00 environment=dev error="an error"
Netcat output:
{
"time":"2023-04-10T14:00:0.000000+00:00",
"level":"ERROR",
"msg":"A message",
"user":{
"id":"user-123",
"created_at":"2023-04-10T14:00:0.000000+00:00"
},
"environment":"dev",
"error":"an error"
}
slogmulti.Router()
Distribute logs to all matching slog.Handler
in parallel.
import (
slogmulti "github.com/samber/slog-multi"
slogslack "github.com/samber/slog-slack"
"golang.org/x/exp/slog"
)
func main() {
slackChannelUS := slogslack.Option{Level: slog.LevelError, WebhookURL: "xxx", Channel: "supervision-us"}.NewSlackHandler()
slackChannelEU := slogslack.Option{Level: slog.LevelError, WebhookURL: "xxx", Channel: "supervision-eu"}.NewSlackHandler()
slackChannelAPAC := slogslack.Option{Level: slog.LevelError, WebhookURL: "xxx", Channel: "supervision-apac"}.NewSlackHandler()
logger := slog.New(
slogmulti.Router().
Add(slackChannelUS, recordMatchRegion("us")).
Add(slackChannelEU, recordMatchRegion("eu")).
Add(slackChannelAPAC, recordMatchRegion("apac")).
Handler(),
)
logger.
With("region", "us").
With("pool", "us-east-1").
Error("Server desynchronized")
}
func recordMatchRegion(region string) func(ctx context.Context, r slog.Record) bool {
return func(ctx context.Context, r slog.Record) bool {
ok := false
r.Attrs(func(attr slog.Attr) bool {
if attr.Key == "region" && attr.Value.Kind() == slog.KindString && attr.Value.String() == region {
ok = true
return false
}
return true
})
return ok
}
}
slogmulti.Failover()
List multiple targets for a slog.Record
instead of retrying on the same unavailable log management system.
import (
"net"
slogmulti "github.com/samber/slog-multi"
"golang.org/x/exp/slog"
)
func main() {
// ncat -l 1000 -k
// ncat -l 1001 -k
// ncat -l 1002 -k
// list AZs
// use github.com/netbrain/goautosocket for auto-reconnect
logstash1, _ := net.Dial("tcp", "logstash.eu-west-3a.internal:1000")
logstash2, _ := net.Dial("tcp", "logstash.eu-west-3b.internal:1000")
logstash3, _ := net.Dial("tcp", "logstash.eu-west-3c.internal:1000")
logger := slog.New(
slogmulti.Failover()(
slog.HandlerOptions{}.NewJSONHandler(logstash1), // send to this instance first
slog.HandlerOptions{}.NewJSONHandler(logstash2), // then this instance in case of failure
slog.HandlerOptions{}.NewJSONHandler(logstash3), // and finally this instance in case of double failure
),
)
logger.
With(
slog.Group("user",
slog.String("id", "user-123"),
slog.Time("created_at", time.Now()),
),
).
With("environment", "dev").
With("error", fmt.Errorf("an error")).
Error("A message")
}
slogmulti.Pool()
Increase log bandwidth by sending log.Record
to a pool of slog.Handler
.
import (
"net"
slogmulti "github.com/samber/slog-multi"
"golang.org/x/exp/slog"
)
func main() {
// ncat -l 1000 -k
// ncat -l 1001 -k
// ncat -l 1002 -k
// list AZs
// use github.com/netbrain/goautosocket for auto-reconnect
logstash1, _ := net.Dial("tcp", "logstash.eu-west-3a.internal:1000")
logstash2, _ := net.Dial("tcp", "logstash.eu-west-3b.internal:1000")
logstash3, _ := net.Dial("tcp", "logstash.eu-west-3c.internal:1000")
logger := slog.New(
slogmulti.Pool()(
// a random handler will be picked
slog.HandlerOptions{}.NewJSONHandler(logstash1),
slog.HandlerOptions{}.NewJSONHandler(logstash2),
slog.HandlerOptions{}.NewJSONHandler(logstash3),
),
)
logger.
With(
slog.Group("user",
slog.String("id", "user-123"),
slog.Time("created_at", time.Now()),
),
).
With("environment", "dev").
With("error", fmt.Errorf("an error")).
Error("A message")
}
slogmulti.Pipe()
Rewrite log.Record
on the fly (eg: for privacy reason).
func main() {
// first middleware: format go `error` type into an object {error: "*myCustomErrorType", message: "could not reach https://a.b/c"}
errorFormattingMiddleware := slogmulti.NewHandleInlineMiddleware(errorFormattingMiddleware)
// second middleware: remove PII
gdprMiddleware := NewGDPRMiddleware()
// final handler
sink := slog.HandlerOptions{}.NewJSONHandler(os.Stderr)
logger := slog.New(
slogmulti.
Pipe(errorFormattingMiddleware).
Pipe(gdprMiddleware).
// ...
Handler(sink),
)
logger.
With(
slog.Group("user",
slog.String("id", "user-123"),
slog.String("email", "user-123"),
slog.Time("created_at", time.Now()),
),
).
With("environment", "dev").
Error("A message",
slog.String("foo", "bar"),
slog.Any("error", fmt.Errorf("an error")),
)
}
Stderr output:
{
"time":"2023-04-10T14:00:0.000000+00:00",
"level":"ERROR",
"msg":"A message",
"user":{
"id":"*******",
"email":"*******",
"created_at":"*******"
},
"environment":"dev",
"foo":"bar",
"error":{
"type":"*myCustomErrorType",
"message":"an error"
}
}
Middleware must match the following prototype:
type Middleware func(slog.Handler) slog.Handler
The example above uses:
Note: WithAttrs
and WithGroup
methods of custom middleware must return a new instance, instead of this
.
An "inline middleware" (aka. lambda), is a shortcut to middleware implementation, that hooks a single method and proxies others.
// hook `logger.Enabled` method
mdw := slogmulti.NewEnabledInlineMiddleware(func(ctx context.Context, level slog.Level, next func(context.Context, slog.Level) bool) bool{
// [...]
return next(ctx, level)
})
// hook `logger.Handle` method
mdw := slogmulti.NewHandleInlineMiddleware(func(ctx context.Context, record slog.Record, next func(context.Context, slog.Record) error) error {
// [...]
return next(ctx, record)
})
// hook `logger.WithAttrs` method
mdw := slogmulti.NewWithAttrsInlineMiddleware(func(attrs []slog.Attr, next func([]slog.Attr) slog.Handler) slog.Handler{
// [...]
return next(attrs)
})
// hook `logger.WithGroup` method
mdw := slogmulti.NewWithGroupInlineMiddleware(func(name string, next func(string) slog.Handler) slog.Handler{
// [...]
return next(name)
})
A super inline middleware that hooks all methods.
Warning: you would rather implement your own middleware.
mdw := slogmulti.NewInlineMiddleware(
func(ctx context.Context, level slog.Level, next func(context.Context, slog.Level) bool) bool{
// [...]
return next(ctx, level)
},
func(ctx context.Context, record slog.Record, next func(context.Context, slog.Record) error) error{
// [...]
return next(ctx, record)
},
func(attrs []slog.Attr, next func([]slog.Attr) slog.Handler) slog.Handler{
// [...]
return next(attrs)
},
func(name string, next func(string) slog.Handler) slog.Handler{
// [...]
return next(name)
},
)
Don't hesitate ;)
# Install some dev dependencies
make tools
# Run tests
make test
# or
make watch-test
Give a ⭐️ if this project helped you!
Copyright © 2023 Samuel Berthe.
This project is MIT licensed.