From d4c19f790de568c96406efec11f6370a4344141b Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Mon, 6 Mar 2023 22:25:43 +0100 Subject: [PATCH 1/3] add an ability to set levels separately for named loggers with glob support --- app/logger/config.go | 40 ++++++++++++++----- app/logger/log.go | 93 ++++++++++++++++++++++++++++++++++++-------- go.mod | 3 +- go.sum | 2 + 4 files changed, 110 insertions(+), 28 deletions(-) diff --git a/app/logger/config.go b/app/logger/config.go index 2f3547ce..7906312c 100644 --- a/app/logger/config.go +++ b/app/logger/config.go @@ -1,11 +1,17 @@ package logger -import "go.uber.org/zap" +import ( + "go.uber.org/zap" + + "github.com/anytypeio/any-sync/util/slice" +) type Config struct { - Production bool `yaml:"production"` - DefaultLevel string `yaml:"defaultLevel"` - NamedLevels map[string]string `yaml:"namedLevels"` + Production bool `yaml:"production"` + DefaultLevel string `yaml:"defaultLevel"` + NamedLevels map[string]string `yaml:"namedLevels"` + AddOutputPaths []string + DisableStdErr bool } func (l Config) ApplyGlobal() { @@ -15,19 +21,33 @@ func (l Config) ApplyGlobal() { } else { conf = zap.NewDevelopmentConfig() } + if len(l.AddOutputPaths) > 0 { + conf.OutputPaths = append(conf.OutputPaths, l.AddOutputPaths...) + } + if l.DisableStdErr { + conf.OutputPaths = slice.Filter(conf.OutputPaths, func(path string) bool { + return path != "stderr" + }) + } + + var err error if defaultLevel, err := zap.ParseAtomicLevel(l.DefaultLevel); err == nil { conf.Level = defaultLevel } - var levels = make(map[string]zap.AtomicLevel) + var lvl = make(map[string]zap.AtomicLevel) for k, v := range l.NamedLevels { - if lev, err := zap.ParseAtomicLevel(v); err != nil { - levels[k] = lev + if lev, err := zap.ParseAtomicLevel(v); err == nil { + lvl[k] = lev + // we need to have a minimum level of all named loggers for the main logger + if lev.Level() < conf.Level.Level() { + conf.Level.SetLevel(lev.Level()) + } } } - defaultLogger, err := conf.Build() + lg, err := conf.Build() if err != nil { Default().Fatal("can't build logger", zap.Error(err)) } - SetDefault(defaultLogger) - SetNamedLevels(levels) + SetDefault(lg) + SetNamedLevels(lvl) } diff --git a/app/logger/log.go b/app/logger/log.go index 028c9988..ce09c3cd 100644 --- a/app/logger/log.go +++ b/app/logger/log.go @@ -1,51 +1,110 @@ package logger import ( - "go.uber.org/zap" "sync" + + "github.com/gobwas/glob" + "go.uber.org/zap" ) var ( - mu sync.Mutex - defaultLogger *zap.Logger - levels = make(map[string]zap.AtomicLevel) - loggers = make(map[string]CtxLogger) + mu sync.Mutex + logger *zap.Logger + loggerConfig zap.Config + namedLevels = make(map[string]zap.AtomicLevel) + namedGlobs = make(map[string]glob.Glob) + namedLoggers = make(map[string]CtxLogger) ) func init() { - defaultLogger, _ = zap.NewDevelopment() - zap.NewProduction() + loggerConfig = zap.NewDevelopmentConfig() + logger, _ = loggerConfig.Build() } +// SetDefault replaces the default logger +// you need to call SetNamedLevels after in case you have named loggers, +// otherwise they will use the old logger func SetDefault(l *zap.Logger) { mu.Lock() defer mu.Unlock() - *defaultLogger = *l - for name, l := range loggers { - *l.Logger = *defaultLogger.Named(name) - } + *logger = *l } +// SetNamedLevels sets the namedLevels for named loggers +// it also supports glob patterns for names, like "app*" +// can be racy in case there are existing named loggers +// so consider to call only once at the beginning func SetNamedLevels(l map[string]zap.AtomicLevel) { mu.Lock() defer mu.Unlock() - levels = l + namedLevels = l + + var minLevel = logger.Level() + for k, l := range namedLevels { + g, err := glob.Compile(k) + if err == nil { + namedGlobs[k] = g + } + namedLevels[k] = l + if l.Level() < minLevel { + minLevel = l.Level() + } + } + + if minLevel < logger.Level() { + // recreate logger if the min level is lower than the current min one + loggerConfig.Level = zap.NewAtomicLevelAt(minLevel) + logger, _ = loggerConfig.Build() + } + + for name, lg := range namedLoggers { + level := getLevel(name) + // this can be racy, but + lg.Logger = zap.New(logger.Core()).WithOptions( + zap.IncreaseLevel(level), + ).Named(name) + } } func Default() *zap.Logger { mu.Lock() defer mu.Unlock() - return defaultLogger + return logger +} + +func getLevel(name string) zap.AtomicLevel { + level, ok := namedLevels[name] + if !ok { + var found bool + for globName, glob := range namedGlobs { + if glob.Match(name) { + found = true + level, _ = namedLevels[globName] + // no need to check ok, because we know that globName exists + break + } + } + if !found { + level = loggerConfig.Level + } + } + return level } func NewNamed(name string, fields ...zap.Field) CtxLogger { mu.Lock() defer mu.Unlock() - l := defaultLogger.Named(name) - if len(fields) > 0 { - l = l.With(fields...) + + if l, nameExists := namedLoggers[name]; nameExists { + return l } + + level := getLevel(name) + l := zap.New(logger.Core()).WithOptions( + zap.IncreaseLevel(level), + ).Named(name) + ctxL := CtxLogger{l} - loggers[name] = ctxL + namedLoggers[name] = ctxL return ctxL } diff --git a/go.mod b/go.mod index a4c42fac..1c763027 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/awalterschulze/gographviz v2.0.3+incompatible github.com/cespare/xxhash v1.1.0 github.com/cheggaaa/mb/v3 v3.0.1 + github.com/gobwas/glob v0.2.3 github.com/goccy/go-graphviz v0.1.0 github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 @@ -22,6 +23,7 @@ require ( github.com/ipfs/go-unixfs v0.4.4 github.com/libp2p/go-libp2p v0.24.1 github.com/minio/sha256-simd v1.0.0 + github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multibase v0.1.1 github.com/multiformats/go-multihash v0.2.1 github.com/prometheus/client_golang v1.14.0 @@ -74,7 +76,6 @@ require ( github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-pointer v0.0.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr v0.8.0 // indirect diff --git a/go.sum b/go.sum index d8708cf2..1cc3b14b 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-graphviz v0.1.0 h1:6OqQoQ5PeAiHYe/YcusyeulqBrOkUb16HQ4ctRdyVUU= github.com/goccy/go-graphviz v0.1.0/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= From 3693218fc53e319f2f8c9c212cd47ecd7edf7de6 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Mon, 6 Mar 2023 22:26:32 +0100 Subject: [PATCH 2/3] rename logger common.commonspace.settings -> commonspace.settings --- commonspace/settings/settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commonspace/settings/settings.go b/commonspace/settings/settings.go index 920b385b..1b75e42d 100644 --- a/commonspace/settings/settings.go +++ b/commonspace/settings/settings.go @@ -22,7 +22,7 @@ import ( "golang.org/x/exp/slices" ) -var log = logger.NewNamed("common.commonspace.settings") +var log = logger.NewNamed("commonspace.settings") type SettingsObject interface { synctree.SyncTree From 0d4083819612745e243e6b86e07b09556359ec59 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Tue, 7 Mar 2023 10:42:36 +0100 Subject: [PATCH 3/3] rename some loggers to the same convention --- app/logger/log.go | 4 ++-- commonspace/object/tree/synctree/synctree.go | 5 +++-- commonspace/objectsync/objectsync.go | 7 ++++--- commonspace/settings/settings.go | 5 +++-- commonspace/syncstatus/syncstatus.go | 9 +++++---- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/app/logger/log.go b/app/logger/log.go index ce09c3cd..264cdaf1 100644 --- a/app/logger/log.go +++ b/app/logger/log.go @@ -57,10 +57,10 @@ func SetNamedLevels(l map[string]zap.AtomicLevel) { logger, _ = loggerConfig.Build() } - for name, lg := range namedLoggers { + for name, nl := range namedLoggers { level := getLevel(name) // this can be racy, but - lg.Logger = zap.New(logger.Core()).WithOptions( + nl.Logger = zap.New(logger.Core()).WithOptions( zap.IncreaseLevel(level), ).Named(name) } diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index d9084017..f25e14d0 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -3,6 +3,7 @@ package synctree import ( "context" "errors" + "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" @@ -10,7 +11,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/objectsync" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" - spacestorage "github.com/anytypeio/any-sync/commonspace/spacestorage" + "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/anytypeio/any-sync/net/peer" "github.com/anytypeio/any-sync/nodeconf" @@ -50,7 +51,7 @@ type syncTree struct { isDeleted bool } -var log = logger.NewNamed("commonspace.synctree") +var log = logger.NewNamed("common.commonspace.synctree") var buildObjectTree = objecttree.BuildObjectTree var createSyncClient = newSyncClient diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index 7c72e635..85c587d9 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -2,6 +2,9 @@ package objectsync import ( "context" + "sync/atomic" + "time" + "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/app/ocache" "github.com/anytypeio/any-sync/commonspace/object/syncobjectgetter" @@ -12,11 +15,9 @@ import ( "github.com/anytypeio/any-sync/nodeconf" "go.uber.org/zap" "golang.org/x/exp/slices" - "sync/atomic" - "time" ) -var log = logger.NewNamed("commonspace.objectsync") +var log = logger.NewNamed("common.commonspace.objectsync") type ObjectSync interface { ocache.ObjectLastUsage diff --git a/commonspace/settings/settings.go b/commonspace/settings/settings.go index 1b75e42d..453df055 100644 --- a/commonspace/settings/settings.go +++ b/commonspace/settings/settings.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "github.com/anytypeio/any-sync/accountservice" "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/commonspace/object/keychain" @@ -14,7 +15,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/treegetter" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" - spacestorage "github.com/anytypeio/any-sync/commonspace/spacestorage" + "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/nodeconf" "github.com/gogo/protobuf/proto" @@ -22,7 +23,7 @@ import ( "golang.org/x/exp/slices" ) -var log = logger.NewNamed("commonspace.settings") +var log = logger.NewNamed("common.commonspace.settings") type SettingsObject interface { synctree.SyncTree diff --git a/commonspace/syncstatus/syncstatus.go b/commonspace/syncstatus/syncstatus.go index b5a489a5..091988f5 100644 --- a/commonspace/syncstatus/syncstatus.go +++ b/commonspace/syncstatus/syncstatus.go @@ -3,15 +3,16 @@ package syncstatus import ( "context" "fmt" + "sync" + "time" + "github.com/anytypeio/any-sync/app/logger" - treestorage "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/nodeconf" "github.com/anytypeio/any-sync/util/periodicsync" "github.com/anytypeio/any-sync/util/slice" "golang.org/x/exp/slices" - "sync" - "time" ) const ( @@ -19,7 +20,7 @@ const ( syncTimeout = time.Second ) -var log = logger.NewNamed("commonspace.syncstatus") +var log = logger.NewNamed("common.commonspace.syncstatus") type UpdateReceiver interface { UpdateTree(ctx context.Context, treeId string, status SyncStatus) (err error)