Remove from config

Add sync logger to reduce number of logs
Merge pull request #48 from anyproto/fix-open-requests
2023-07-17 14:48:05 +02:00 · 2023-07-17 13:50:49 +02:00 · 2023-07-14 16:25:31 +02:00 · 2023-07-14 16:19:38 +02:00 · 2023-07-10 19:13:21 +02:00 · 2023-07-10 19:13:11 +02:00
6 changed files with 61 additions and 23 deletions
@@ -26,6 +26,8 @@ type DiffSyncer interface {
 	Close() error
 }

+const logPeriodSecs = 200
+
 func newDiffSyncer(hs *headSync) DiffSyncer {
 	return &diffSyncer{
 		diff:               hs.diff,
@@ -35,7 +37,7 @@ func newDiffSyncer(hs *headSync) DiffSyncer {
 		peerManager:        hs.peerManager,
 		clientFactory:      spacesyncproto.ClientFactoryFunc(spacesyncproto.NewDRPCSpaceSyncClient),
 		credentialProvider: hs.credentialProvider,
-		log:                log,
+		log:                newSyncLogger(hs.log, logPeriodSecs),
 		syncStatus:         hs.syncStatus,
 		deletionState:      hs.deletionState,
 	}
@@ -48,7 +50,7 @@ type diffSyncer struct {
 	treeManager        treemanager.TreeManager
 	storage            spacestorage.SpaceStorage
 	clientFactory      spacesyncproto.ClientFactory
-	log                logger.CtxLogger
+	log                syncLogger
 	deletionState      deletionstate.ObjectDeletionState
 	credentialProvider credentialprovider.CredentialProvider
 	syncStatus         syncstatus.StatusUpdater
@@ -100,7 +102,7 @@ func (d *diffSyncer) Sync(ctx context.Context) error {
 			d.log.ErrorCtx(ctx, "can't sync with peer", zap.String("peer", p.Id()), zap.Error(err))
 		}
 	}
-	d.log.InfoCtx(ctx, "diff done", zap.String("spaceId", d.spaceId), zap.Duration("dur", time.Since(st)))
+	d.log.DebugCtx(ctx, "diff done", zap.String("spaceId", d.spaceId), zap.Duration("dur", time.Since(st)))
 	return nil
 }

@@ -145,12 +147,7 @@ func (d *diffSyncer) syncWithPeer(ctx context.Context, p peer.Peer) (err error)
 	if err != nil {
 		return err
 	}
-	d.log.Info("sync done:", zap.Int("newIds", len(newIds)),
-		zap.Int("changedIds", len(changedIds)),
-		zap.Int("removedIds", len(removedIds)),
-		zap.Int("already deleted ids", totalLen-len(existingIds)-len(missingIds)),
-		zap.String("peerId", p.Id()),
-	)
+	d.log.logSyncDone(p.Id(), len(newIds), len(changedIds), len(removedIds), totalLen-len(existingIds)-len(missingIds))
 	return
 }

@@ -6,7 +6,7 @@ import (
 	"github.com/anyproto/any-sync/app"
 	"github.com/anyproto/any-sync/app/ldiff"
 	"github.com/anyproto/any-sync/app/logger"
-	config2 "github.com/anyproto/any-sync/commonspace/config"
+	"github.com/anyproto/any-sync/commonspace/config"
 	"github.com/anyproto/any-sync/commonspace/credentialprovider"
 	"github.com/anyproto/any-sync/commonspace/deletionstate"
 	"github.com/anyproto/any-sync/commonspace/object/treemanager"
@@ -70,7 +70,7 @@ var createDiffSyncer = newDiffSyncer

 func (h *headSync) Init(a *app.App) (err error) {
 	shared := a.MustComponent(spacestate.CName).(*spacestate.SpaceState)
-	cfg := a.MustComponent("config").(config2.ConfigGetter)
+	cfg := a.MustComponent("config").(config.ConfigGetter)
 	h.spaceId = shared.SpaceId
 	h.spaceIsDeleted = shared.SpaceIsDeleted
 	h.syncPeriod = cfg.GetSpace().SyncPeriod
@@ -0,0 +1,41 @@
+package headsync
+
+import (
+	"time"
+
+	"github.com/anyproto/any-sync/app/logger"
+	"go.uber.org/zap"
+)
+
+type syncLogger struct {
+	lastLogged  map[string]time.Time
+	logInterval time.Duration
+	logger.CtxLogger
+}
+
+func newSyncLogger(log logger.CtxLogger, syncLogPeriodSecs int) syncLogger {
+	return syncLogger{
+		lastLogged:  map[string]time.Time{},
+		logInterval: time.Duration(syncLogPeriodSecs) * time.Second,
+		CtxLogger:   log,
+	}
+}
+
+func (s syncLogger) logSyncDone(peerId string, newIds, changedIds, removedIds, deltedIds int) {
+	now := time.Now()
+	differentIds := newIds + changedIds + removedIds + deltedIds
+	// always logging if some ids are different or there are no log interval
+	if differentIds == 0 && s.logInterval > 0 {
+		lastLogged := s.lastLogged[peerId]
+		if now.Before(lastLogged.Add(s.logInterval)) {
+			return
+		}
+	}
+	s.lastLogged[peerId] = now
+	s.Info("sync done:", zap.Int("newIds", newIds),
+		zap.Int("changedIds", changedIds),
+		zap.Int("removedIds", removedIds),
+		zap.Int("already deleted ids", deltedIds),
+		zap.String("peerId", peerId),
+	)
+}
@@ -35,9 +35,9 @@ require (
 	github.com/zeebo/blake3 v0.2.3
 	go.uber.org/atomic v1.11.0
 	go.uber.org/zap v1.24.0
-	golang.org/x/crypto v0.10.0
+	golang.org/x/crypto v0.11.0
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
-	golang.org/x/net v0.11.0
+	golang.org/x/net v0.12.0
 	gopkg.in/yaml.v3 v3.0.1
 	storj.io/drpc v0.0.33
 )
@@ -99,7 +99,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/image v0.6.0 // indirect
 	golang.org/x/sync v0.2.0 // indirect
-	golang.org/x/sys v0.9.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
 	golang.org/x/tools v0.9.3 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	lukechampine.com/blake3 v1.2.1 // indirect
@@ -298,8 +298,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
-golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
+golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
+golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/image v0.6.0 h1:bR8b5okrPI3g/gyZakLZHeWxAR8Dn5CyxXv1hLH5g/4=
@@ -321,8 +321,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
-golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
+golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
+golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -346,8 +346,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
-golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -356,7 +356,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
+golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
@@ -106,10 +106,10 @@ func (p *peer) AcquireDrpcConn(ctx context.Context) (drpc.Conn, error) {
 	p.mu.Lock()
 	if len(p.inactive) == 0 {
 		wait := p.limiter.wait(len(p.active) + int(p.openingWaitCount.Load()))
+		p.openingWaitCount.Add(1)
+		defer p.openingWaitCount.Add(-1)
 		p.mu.Unlock()
 		if wait != nil {
-			p.openingWaitCount.Add(1)
-			defer p.openingWaitCount.Add(-1)
 			// throttle new connection opening
 			select {
 			case <-ctx.Done():
Author	SHA1	Message	Date
mcrakhman	959b1307c6	Remove from config	2023-07-17 14:48:05 +02:00
mcrakhman	a35d94a20a	Add sync logger to reduce number of logs	2023-07-17 13:50:49 +02:00
Mikhail Rakhmanov	7049a884a7	Merge pull request #48 from anyproto/fix-open-requests Fix openingWaitCount logic	2023-07-14 16:25:31 +02:00
mcrakhman	21a9ce6035	Fix openingWaitCount logic	2023-07-14 16:19:38 +02:00
Sergey Cherepanov	4bccbf1faf	Merge pull request #45 from anyproto/dependabot/go_modules/golang.org/x/net-0.12.0 Bump golang.org/x/net from 0.11.0 to 0.12.0	2023-07-10 19:13:21 +02:00
Sergey Cherepanov	26b18fba87	Merge pull request #44 from anyproto/dependabot/go_modules/golang.org/x/crypto-0.11.0 Bump golang.org/x/crypto from 0.10.0 to 0.11.0	2023-07-10 19:13:11 +02:00
dependabot[bot]	fc0c3d54f1	Bump golang.org/x/net from 0.11.0 to 0.12.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.11.0 to 0.12.0. - [Commits](https://github.com/golang/net/compare/v0.11.0...v0.12.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>	2023-07-10 15:26:07 +00:00
dependabot[bot]	9dde29a280	Bump golang.org/x/crypto from 0.10.0 to 0.11.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.10.0 to 0.11.0. - [Commits](https://github.com/golang/crypto/compare/v0.10.0...v0.11.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>	2023-07-10 15:25:50 +00:00