zeromicro/go-zero
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add fields with logx methods, support using third party logging libs. (#1847)

Browse Source 
* backup

* simplify

* chore: remove unused pool

* chore: fix lint errors

* chore: use strings.Builder instead of bytes.Buffer

* test: add more tests

* chore: fix reviewdog

* test: fix data race

* feat: make logger customizable

* chore: fix reviewdog

* test: fix fails

* chore: fix set writer twice

* chore: use context instead of golang.org context

* chore: specify uint32 for level types
This commit is contained in:
Kevin Wan
2022-05-03 17:34:26 +08:00
committed by GitHub
parent ae2c76765c
commit ac321fc146
21 changed files with 1551 additions and 792 deletions
Download Patch File Download Diff File Expand all files Collapse all files

540
core/logx/logs.go
View File

@@ -1,93 +1,29 @@
package logx
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path"
"runtime"
"runtime/debug"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/zeromicro/go-zero/core/iox"
"github.com/zeromicro/go-zero/core/sysx"
"github.com/zeromicro/go-zero/core/timex"
)
const (
// InfoLevel logs everything
InfoLevel = iota
// ErrorLevel includes errors, slows, stacks
ErrorLevel
// SevereLevel only log severe messages
SevereLevel
)
const (
jsonEncodingType = iota
plainEncodingType
jsonEncoding = "json"
plainEncoding = "plain"
plainEncodingSep = '\t'
)
const (
accessFilename = "access.log"
errorFilename = "error.log"
severeFilename = "severe.log"
slowFilename = "slow.log"
statFilename = "stat.log"
consoleMode = "console"
volumeMode = "volume"
levelAlert = "alert"
levelInfo = "info"
levelError = "error"
levelSevere = "severe"
levelFatal = "fatal"
levelSlow = "slow"
levelStat = "stat"
backupFileDelimiter = "-"
callerInnerDepth = 5
flags = 0x0
)
const callerDepth = 5
var (
// ErrLogPathNotSet is an error that indicates the log path is not set.
ErrLogPathNotSet = errors.New("log path must be set")
// ErrLogNotInitialized is an error that log is not initialized.
ErrLogNotInitialized = errors.New("log not initialized")
// ErrLogServiceNameNotSet is an error that indicates that the service name is not set.
ErrLogServiceNameNotSet = errors.New("log service name must be set")
timeFormat = "2006-01-02T15:04:05.000Z07:00"
writeConsole bool
logLevel uint32
encoding uint32 = jsonEncodingType
timeFormat = "2006-01-02T15:04:05.000Z07:00"
logLevel uint32
encoding uint32 = jsonEncodingType
// use uint32 for atomic operations
disableStat uint32
infoLog io.WriteCloser
errorLog io.WriteCloser
severeLog io.WriteCloser
slowLog io.WriteCloser
statLog io.WriteCloser
stackLog io.Writer
once sync.Once
initialized uint32
options logOptions
options logOptions
writer = new(atomicWriter)
)
type (
@@ -95,109 +31,37 @@ type (
Timestamp string `json:"@timestamp"`
Level string `json:"level"`
Duration string `json:"duration,omitempty"`
Caller string `json:"caller,omitempty"`
Content interface{} `json:"content"`
}
logEntryWithFields map[string]interface{}
logOptions struct {
gzipEnabled bool
logStackCooldownMills int
keepDays int
}
// LogField is a key-value pair that will be added to the log entry.
LogField struct {
Key string
Value interface{}
}
// LogOption defines the method to customize the logging.
LogOption func(options *logOptions)
// A Logger represents a logger.
Logger interface {
Error(...interface{})
Errorf(string, ...interface{})
Errorv(interface{})
Info(...interface{})
Infof(string, ...interface{})
Infov(interface{})
Slow(...interface{})
Slowf(string, ...interface{})
Slowv(interface{})
WithDuration(time.Duration) Logger
}
)
// MustSetup sets up logging with given config c. It exits on error.
func MustSetup(c LogConf) {
Must(SetUp(c))
}
// SetUp sets up the logx. If already set up, just return nil.
// we allow SetUp to be called multiple times, because for example
// we need to allow different service frameworks to initialize logx respectively.
// the same logic for SetUp
func SetUp(c LogConf) error {
if len(c.TimeFormat) > 0 {
timeFormat = c.TimeFormat
}
switch c.Encoding {
case plainEncoding:
atomic.StoreUint32(&encoding, plainEncodingType)
default:
atomic.StoreUint32(&encoding, jsonEncodingType)
}
switch c.Mode {
case consoleMode:
setupWithConsole(c)
return nil
case volumeMode:
return setupWithVolume(c)
default:
return setupWithFiles(c)
}
}
// Alert alerts v in alert level, and the message is written to error log.
func Alert(v string) {
outputText(errorLog, levelAlert, v)
getWriter().Alert(v)
}
// Close closes the logging.
func Close() error {
if writeConsole {
return nil
}
if atomic.LoadUint32(&initialized) == 0 {
return ErrLogNotInitialized
}
atomic.StoreUint32(&initialized, 0)
if infoLog != nil {
if err := infoLog.Close(); err != nil {
return err
}
}
if errorLog != nil {
if err := errorLog.Close(); err != nil {
return err
}
}
if severeLog != nil {
if err := severeLog.Close(); err != nil {
return err
}
}
if slowLog != nil {
if err := slowLog.Close(); err != nil {
return err
}
}
if statLog != nil {
if err := statLog.Close(); err != nil {
return err
}
if w := writer.Swap(nil); w != nil {
return w.(io.Closer).Close()
}
return nil
@@ -205,16 +69,7 @@ func Close() error {
// Disable disables the logging.
func Disable() {
once.Do(func() {
atomic.StoreUint32(&initialized, 1)
infoLog = iox.NopCloser(ioutil.Discard)
errorLog = iox.NopCloser(ioutil.Discard)
severeLog = iox.NopCloser(ioutil.Discard)
slowLog = iox.NopCloser(ioutil.Discard)
statLog = iox.NopCloser(ioutil.Discard)
stackLog = ioutil.Discard
})
writer.Store(nopWriter{})
}
// DisableStat disables the stat logs.
@@ -224,22 +79,12 @@ func DisableStat() {
// Error writes v into error log.
func Error(v ...interface{}) {
ErrorCaller(1, v...)
}
// ErrorCaller writes v with context into error log.
func ErrorCaller(callDepth int, v ...interface{}) {
errorTextSync(fmt.Sprint(v...), callDepth+callerInnerDepth)
}
// ErrorCallerf writes v with context in format into error log.
func ErrorCallerf(callDepth int, format string, v ...interface{}) {
errorTextSync(fmt.Errorf(format, v...).Error(), callDepth+callerInnerDepth)
errorTextSync(fmt.Sprint(v...))
}
// Errorf writes v with format into error log.
func Errorf(format string, v ...interface{}) {
ErrorCallerf(1, format, v...)
errorTextSync(fmt.Errorf(format, v...).Error())
}
// ErrorStack writes v along with call stack into error log.
@@ -260,6 +105,21 @@ func Errorv(v interface{}) {
errorAnySync(v)
}
// Errorw writes msg along with fields into error log.
func Errorw(msg string, fields ...LogField) {
errorFieldsSync(msg, fields...)
}
// Field returns a LogField for the given key and value.
func Field(key string, value interface{}) LogField {
switch val := value.(type) {
case time.Duration:
return LogField{Key: key, Value: fmt.Sprint(val)}
default:
return LogField{Key: key, Value: val}
}
}
// Info writes v into access log.
func Info(v ...interface{}) {
infoTextSync(fmt.Sprint(v...))
@@ -275,14 +135,32 @@ func Infov(v interface{}) {
infoAnySync(v)
}
// Infow writes msg along with fields into access log.
func Infow(msg string, fields ...LogField) {
infoFieldsSync(msg, fields...)
}
// Must checks if err is nil, otherwise logs the error and exits.
func Must(err error) {
if err != nil {
msg := formatWithCaller(err.Error(), 3)
log.Print(msg)
outputText(severeLog, levelFatal, msg)
os.Exit(1)
if err == nil {
return
}
msg := err.Error()
log.Print(msg)
getWriter().Severe(msg)
os.Exit(1)
}
// MustSetup sets up logging with given config c. It exits on error.
func MustSetup(c LogConf) {
Must(SetUp(c))
}
// Reset clears the writer and resets the log level.
func Reset() Writer {
SetLevel(InfoLevel)
return writer.Swap(nil)
}
// SetLevel sets the logging level. It can be used to suppress some logs.
@@ -290,6 +168,43 @@ func SetLevel(level uint32) {
atomic.StoreUint32(&logLevel, level)
}
// SetWriter sets the logging writer. It can be used to customize the logging.
// Call Reset before calling SetWriter again.
func SetWriter(w Writer) {
if writer.Load() == nil {
writer.Store(w)
}
}
// SetUp sets up the logx. If already set up, just return nil.
// we allow SetUp to be called multiple times, because for example
// we need to allow different service frameworks to initialize logx respectively.
// the same logic for SetUp
func SetUp(c LogConf) error {
setupLogLevel(c)
if len(c.TimeFormat) > 0 {
timeFormat = c.TimeFormat
}
switch c.Encoding {
case plainEncoding:
atomic.StoreUint32(&encoding, plainEncodingType)
default:
atomic.StoreUint32(&encoding, jsonEncodingType)
}
switch c.Mode {
case consoleMode:
setupWithConsole()
return nil
case volumeMode:
return setupWithVolume(c)
default:
return setupWithFiles(c)
}
}
// Severe writes v into severe log.
func Severe(v ...interface{}) {
severeSync(fmt.Sprint(v...))
@@ -315,6 +230,11 @@ func Slowv(v interface{}) {
slowAnySync(v)
}
// Sloww writes msg along with fields into slow log.
func Sloww(msg string, fields ...LogField) {
slowFieldsSync(msg, fields...)
}
// Stat writes v into stat log.
func Stat(v ...interface{}) {
statSync(fmt.Sprint(v...))
@@ -357,52 +277,30 @@ func createOutput(path string) (io.WriteCloser, error) {
func errorAnySync(v interface{}) {
if shallLog(ErrorLevel) {
outputAny(errorLog, levelError, v)
getWriter().Error(v)
}
}
func errorTextSync(msg string, callDepth int) {
func errorFieldsSync(content string, fields ...LogField) {
if shallLog(ErrorLevel) {
outputError(errorLog, msg, callDepth)
getWriter().Error(content, fields...)
}
}
func formatWithCaller(msg string, callDepth int) string {
var buf strings.Builder
caller := getCaller(callDepth)
if len(caller) > 0 {
buf.WriteString(caller)
buf.WriteByte(' ')
func errorTextSync(msg string) {
if shallLog(ErrorLevel) {
getWriter().Error(msg)
}
buf.WriteString(msg)
return buf.String()
}
func getCaller(callDepth int) string {
var buf strings.Builder
_, file, line, ok := runtime.Caller(callDepth)
if ok {
short := file
for i := len(file) - 1; i > 0; i-- {
if file[i] == '/' {
short = file[i+1:]
break
}
}
buf.WriteString(short)
buf.WriteByte(':')
buf.WriteString(strconv.Itoa(line))
func getWriter() Writer {
w := writer.Load()
if w == nil {
w = newConsoleWriter()
writer.Store(w)
}
return buf.String()
}
func getTimestamp() string {
return timex.Time().Format(timeFormat)
return w
}
func handleOptions(opts []LogOption) {
@@ -413,56 +311,19 @@ func handleOptions(opts []LogOption) {
func infoAnySync(val interface{}) {
if shallLog(InfoLevel) {
outputAny(infoLog, levelInfo, val)
getWriter().Info(val)
}
}
func infoFieldsSync(content string, fields ...LogField) {
if shallLog(InfoLevel) {
getWriter().Info(content, fields...)
}
}
func infoTextSync(msg string) {
if shallLog(InfoLevel) {
outputText(infoLog, levelInfo, msg)
}
}
func outputAny(writer io.Writer, level string, val interface{}) {
switch atomic.LoadUint32(&encoding) {
case plainEncodingType:
writePlainAny(writer, level, val)
default:
info := logEntry{
Timestamp: getTimestamp(),
Level: level,
Content: val,
}
outputJson(writer, info)
}
}
func outputText(writer io.Writer, level, msg string) {
switch atomic.LoadUint32(&encoding) {
case plainEncodingType:
writePlainText(writer, level, msg)
default:
info := logEntry{
Timestamp: getTimestamp(),
Level: level,
Content: msg,
}
outputJson(writer, info)
}
}
func outputError(writer io.Writer, msg string, callDepth int) {
content := formatWithCaller(msg, callDepth)
outputText(writer, levelError, content)
}
func outputJson(writer io.Writer, info interface{}) {
if content, err := json.Marshal(info); err != nil {
log.Println(err.Error())
} else if atomic.LoadUint32(&initialized) == 0 || writer == nil {
log.Println(string(content))
} else {
writer.Write(append(content, '\n'))
getWriter().Info(msg)
}
}
@@ -477,72 +338,18 @@ func setupLogLevel(c LogConf) {
}
}
func setupWithConsole(c LogConf) {
once.Do(func() {
atomic.StoreUint32(&initialized, 1)
writeConsole = true
setupLogLevel(c)
infoLog = newLogWriter(log.New(os.Stdout, "", flags))
errorLog = newLogWriter(log.New(os.Stderr, "", flags))
severeLog = newLogWriter(log.New(os.Stderr, "", flags))
slowLog = newLogWriter(log.New(os.Stderr, "", flags))
stackLog = newLessWriter(errorLog, options.logStackCooldownMills)
statLog = infoLog
})
func setupWithConsole() {
SetWriter(newConsoleWriter())
}
func setupWithFiles(c LogConf) error {
var opts []LogOption
var err error
if len(c.Path) == 0 {
return ErrLogPathNotSet
w, err := newFileWriter(c)
if err != nil {
return err
}
opts = append(opts, WithCooldownMillis(c.StackCooldownMillis))
if c.Compress {
opts = append(opts, WithGzip())
}
if c.KeepDays > 0 {
opts = append(opts, WithKeepDays(c.KeepDays))
}
accessFile := path.Join(c.Path, accessFilename)
errorFile := path.Join(c.Path, errorFilename)
severeFile := path.Join(c.Path, severeFilename)
slowFile := path.Join(c.Path, slowFilename)
statFile := path.Join(c.Path, statFilename)
once.Do(func() {
atomic.StoreUint32(&initialized, 1)
handleOptions(opts)
setupLogLevel(c)
if infoLog, err = createOutput(accessFile); err != nil {
return
}
if errorLog, err = createOutput(errorFile); err != nil {
return
}
if severeLog, err = createOutput(severeFile); err != nil {
return
}
if slowLog, err = createOutput(slowFile); err != nil {
return
}
if statLog, err = createOutput(statFile); err != nil {
return
}
stackLog = newLessWriter(errorLog, options.logStackCooldownMills)
})
return err
SetWriter(w)
return nil
}
func setupWithVolume(c LogConf) error {
@@ -556,7 +363,7 @@ func setupWithVolume(c LogConf) error {
func severeSync(msg string) {
if shallLog(SevereLevel) {
outputText(severeLog, levelSevere, fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
getWriter().Severe(fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
}
}
@@ -570,99 +377,30 @@ func shallLogStat() bool {
func slowAnySync(v interface{}) {
if shallLog(ErrorLevel) {
outputAny(slowLog, levelSlow, v)
getWriter().Slow(v)
}
}
func slowFieldsSync(content string, fields ...LogField) {
if shallLog(ErrorLevel) {
getWriter().Slow(content, fields...)
}
}
func slowTextSync(msg string) {
if shallLog(ErrorLevel) {
outputText(slowLog, levelSlow, msg)
getWriter().Slow(msg)
}
}
func stackSync(msg string) {
if shallLog(ErrorLevel) {
outputText(stackLog, levelError, fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
getWriter().Stack(fmt.Sprintf("%s\n%s", msg, string(debug.Stack())))
}
}
func statSync(msg string) {
if shallLogStat() && shallLog(InfoLevel) {
outputText(statLog, levelStat, msg)
getWriter().Stat(msg)
}
}
func writePlainAny(writer io.Writer, level string, val interface{}, fields ...string) {
switch v := val.(type) {
case string:
writePlainText(writer, level, v, fields...)
case error:
writePlainText(writer, level, v.Error(), fields...)
case fmt.Stringer:
writePlainText(writer, level, v.String(), fields...)
default:
var buf bytes.Buffer
buf.WriteString(getTimestamp())
buf.WriteByte(plainEncodingSep)
buf.WriteString(level)
for _, item := range fields {
buf.WriteByte(plainEncodingSep)
buf.WriteString(item)
}
buf.WriteByte(plainEncodingSep)
if err := json.NewEncoder(&buf).Encode(val); err != nil {
log.Println(err.Error())
return
}
buf.WriteByte('\n')
if atomic.LoadUint32(&initialized) == 0 || writer == nil {
log.Println(buf.String())
return
}
if _, err := writer.Write(buf.Bytes()); err != nil {
log.Println(err.Error())
}
}
}
func writePlainText(writer io.Writer, level, msg string, fields ...string) {
var buf bytes.Buffer
buf.WriteString(getTimestamp())
buf.WriteByte(plainEncodingSep)
buf.WriteString(level)
for _, item := range fields {
buf.WriteByte(plainEncodingSep)
buf.WriteString(item)
}
buf.WriteByte(plainEncodingSep)
buf.WriteString(msg)
buf.WriteByte('\n')
if atomic.LoadUint32(&initialized) == 0 || writer == nil {
log.Println(buf.String())
return
}
if _, err := writer.Write(buf.Bytes()); err != nil {
log.Println(err.Error())
}
}
type logWriter struct {
logger *log.Logger
}
func newLogWriter(logger *log.Logger) logWriter {
return logWriter{
logger: logger,
}
}
func (lw logWriter) Close() error {
return nil
}
func (lw logWriter) Write(data []byte) (int, error) {
lw.logger.Print(string(data))
return len(data), nil
}