mod tidy

Bump github.com/libp2p/go-libp2p from 0.27.3 to 0.27.5

app/ocache/metrics.go

@@ -6,6 +6,9 @@ import (
 )
 
 func WithPrometheus(reg *prometheus.Registry, namespace, subsystem string) Option {
+	if reg == nil {
+		return nil
+	}
 	if subsystem == "" {
 		subsystem = "cache"
 	}
@@ -13,9 +16,7 @@ func WithPrometheus(reg *prometheus.Registry, namespace, subsystem string) Optio
 	subSplit := strings.Split(subsystem, ".")
 	namespace = strings.Join(nameSplit, "_")
 	subsystem = strings.Join(subSplit, "_")
-	if reg == nil {
-		return nil
-	}
+
 	return func(cache *oCache) {
 		cache.metrics = &metrics{
 			hit: prometheus.NewCounter(prometheus.CounterOpts{

commonfile/fileproto/file_drpc.pb.go

@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go-drpc. DO NOT EDIT.
-// protoc-gen-go-drpc version: v0.0.32
+// protoc-gen-go-drpc version: v0.0.33
 // source: commonfile/fileproto/protos/file.proto
 
 package fileproto

commonspace/spacesyncproto/spacesync_drpc.pb.go

@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go-drpc. DO NOT EDIT.
-// protoc-gen-go-drpc version: v0.0.32
+// protoc-gen-go-drpc version: v0.0.33
 // source: commonspace/spacesyncproto/protos/spacesync.proto
 
 package spacesyncproto
@@ -102,6 +102,10 @@ type drpcSpaceSync_ObjectSyncStreamClient struct {
 	drpc.Stream
 }
 
+func (x *drpcSpaceSync_ObjectSyncStreamClient) GetStream() drpc.Stream {
+	return x.Stream
+}
+
 func (x *drpcSpaceSync_ObjectSyncStreamClient) Send(m *ObjectSyncMessage) error {
 	return x.MsgSend(m, drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{})
 }

coordinator/coordinatorclient/coordinatorclient.go

@@ -10,6 +10,7 @@ import (
 	"github.com/anyproto/any-sync/net/rpc/rpcerr"
 	"github.com/anyproto/any-sync/nodeconf"
 	"github.com/anyproto/any-sync/util/crypto"
+	"storj.io/drpc"
 )
 
 const CName = "common.coordinator.coordinatorclient"
@@ -39,42 +40,8 @@ type coordinatorClient struct {
 	nodeConf nodeconf.Service
 }
 
-func (c *coordinatorClient) ChangeStatus(ctx context.Context, spaceId string, deleteRaw *treechangeproto.RawTreeChangeWithId) (status *coordinatorproto.SpaceStatusPayload, err error) {
-	cl, err := c.client(ctx)
-	if err != nil {
-		return
-	}
-	resp, err := cl.SpaceStatusChange(ctx, &coordinatorproto.SpaceStatusChangeRequest{
-		SpaceId:               spaceId,
-		DeletionChangeId:      deleteRaw.GetId(),
-		DeletionChangePayload: deleteRaw.GetRawChange(),
-	})
-	if err != nil {
-		err = rpcerr.Unwrap(err)
-		return
-	}
-	status = resp.Payload
-	return
-}
-
-func (c *coordinatorClient) StatusCheck(ctx context.Context, spaceId string) (status *coordinatorproto.SpaceStatusPayload, err error) {
-	cl, err := c.client(ctx)
-	if err != nil {
-		return
-	}
-	resp, err := cl.SpaceStatusCheck(ctx, &coordinatorproto.SpaceStatusCheckRequest{
-		SpaceId: spaceId,
-	})
-	if err != nil {
-		err = rpcerr.Unwrap(err)
-		return
-	}
-	status = resp.Payload
-	return
-}
-
 func (c *coordinatorClient) Init(a *app.App) (err error) {
-	c.pool = a.MustComponent(pool.CName).(pool.Service).NewPool(CName)
+	c.pool = a.MustComponent(pool.CName).(pool.Service)
 	c.nodeConf = a.MustComponent(nodeconf.CName).(nodeconf.Service)
 	return
 }
@@ -83,8 +50,37 @@ func (c *coordinatorClient) Name() (name string) {
 	return CName
 }
 
+func (c *coordinatorClient) ChangeStatus(ctx context.Context, spaceId string, deleteRaw *treechangeproto.RawTreeChangeWithId) (status *coordinatorproto.SpaceStatusPayload, err error) {
+	err = c.doClient(ctx, func(cl coordinatorproto.DRPCCoordinatorClient) error {
+		resp, err := cl.SpaceStatusChange(ctx, &coordinatorproto.SpaceStatusChangeRequest{
+			SpaceId:               spaceId,
+			DeletionChangeId:      deleteRaw.GetId(),
+			DeletionChangePayload: deleteRaw.GetRawChange(),
+		})
+		if err != nil {
+			return rpcerr.Unwrap(err)
+		}
+		status = resp.Payload
+		return nil
+	})
+	return
+}
+
+func (c *coordinatorClient) StatusCheck(ctx context.Context, spaceId string) (status *coordinatorproto.SpaceStatusPayload, err error) {
+	err = c.doClient(ctx, func(cl coordinatorproto.DRPCCoordinatorClient) error {
+		resp, err := cl.SpaceStatusCheck(ctx, &coordinatorproto.SpaceStatusCheckRequest{
+			SpaceId: spaceId,
+		})
+		if err != nil {
+			return rpcerr.Unwrap(err)
+		}
+		status = resp.Payload
+		return nil
+	})
+	return
+}
+
 func (c *coordinatorClient) SpaceSign(ctx context.Context, payload SpaceSignPayload) (receipt *coordinatorproto.SpaceReceiptWithSignature, err error) {
-	cl, err := c.client(ctx)
 	if err != nil {
 		return
 	}
@@ -100,54 +96,56 @@ func (c *coordinatorClient) SpaceSign(ctx context.Context, payload SpaceSignPayl
 	if err != nil {
 		return
 	}
-	resp, err := cl.SpaceSign(ctx, &coordinatorproto.SpaceSignRequest{
-		SpaceId:              payload.SpaceId,
-		Header:               payload.SpaceHeader,
-		OldIdentity:          oldIdentity,
-		NewIdentitySignature: newSignature,
+	err = c.doClient(ctx, func(cl coordinatorproto.DRPCCoordinatorClient) error {
+		resp, err := cl.SpaceSign(ctx, &coordinatorproto.SpaceSignRequest{
+			SpaceId:              payload.SpaceId,
+			Header:               payload.SpaceHeader,
+			OldIdentity:          oldIdentity,
+			NewIdentitySignature: newSignature,
+		})
+		if err != nil {
+			return rpcerr.Unwrap(err)
+		}
+		receipt = resp.Receipt
+		return nil
 	})
-	if err != nil {
-		err = rpcerr.Unwrap(err)
-		return
-	}
-	return resp.Receipt, nil
-}
-
-func (c *coordinatorClient) FileLimitCheck(ctx context.Context, spaceId string, identity []byte) (limit uint64, err error) {
-	cl, err := c.client(ctx)
-	if err != nil {
-		return
-	}
-	resp, err := cl.FileLimitCheck(ctx, &coordinatorproto.FileLimitCheckRequest{
-		AccountIdentity: identity,
-		SpaceId:         spaceId,
-	})
-	if err != nil {
-		err = rpcerr.Unwrap(err)
-		return
-	}
-	return resp.Limit, nil
-}
-
-func (c *coordinatorClient) NetworkConfiguration(ctx context.Context, currentId string) (resp *coordinatorproto.NetworkConfigurationResponse, err error) {
-	cl, err := c.client(ctx)
-	if err != nil {
-		return
-	}
-	resp, err = cl.NetworkConfiguration(ctx, &coordinatorproto.NetworkConfigurationRequest{
-		CurrentId: currentId,
-	})
-	if err != nil {
-		err = rpcerr.Unwrap(err)
-		return
-	}
 	return
 }
 
-func (c *coordinatorClient) client(ctx context.Context) (coordinatorproto.DRPCCoordinatorClient, error) {
+func (c *coordinatorClient) FileLimitCheck(ctx context.Context, spaceId string, identity []byte) (limit uint64, err error) {
+	err = c.doClient(ctx, func(cl coordinatorproto.DRPCCoordinatorClient) error {
+		resp, err := cl.FileLimitCheck(ctx, &coordinatorproto.FileLimitCheckRequest{
+			AccountIdentity: identity,
+			SpaceId:         spaceId,
+		})
+		if err != nil {
+			return rpcerr.Unwrap(err)
+		}
+		limit = resp.Limit
+		return nil
+	})
+	return
+}
+
+func (c *coordinatorClient) NetworkConfiguration(ctx context.Context, currentId string) (resp *coordinatorproto.NetworkConfigurationResponse, err error) {
+	err = c.doClient(ctx, func(cl coordinatorproto.DRPCCoordinatorClient) error {
+		resp, err = cl.NetworkConfiguration(ctx, &coordinatorproto.NetworkConfigurationRequest{
+			CurrentId: currentId,
+		})
+		if err != nil {
+			return rpcerr.Unwrap(err)
+		}
+		return nil
+	})
+	return
+}
+
+func (c *coordinatorClient) doClient(ctx context.Context, f func(cl coordinatorproto.DRPCCoordinatorClient) error) error {
 	p, err := c.pool.GetOneOf(ctx, c.nodeConf.CoordinatorPeers())
 	if err != nil {
-		return nil, err
+		return err
 	}
-	return coordinatorproto.NewDRPCCoordinatorClient(p), nil
+	return p.DoDrpc(ctx, func(conn drpc.Conn) error {
+		return f(coordinatorproto.NewDRPCCoordinatorClient(conn))
+	})
 }

coordinator/coordinatorproto/coordinator_drpc.pb.go

@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go-drpc. DO NOT EDIT.
-// protoc-gen-go-drpc version: v0.0.32
+// protoc-gen-go-drpc version: v0.0.33
 // source: coordinator/coordinatorproto/protos/coordinator.proto
 
 package coordinatorproto

go.mod

@@ -14,25 +14,27 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.3.0
+	github.com/hashicorp/yamux v0.1.1
 	github.com/huandu/skiplist v1.2.0
 	github.com/ipfs/go-block-format v0.1.2
-	github.com/ipfs/go-blockservice v0.5.1
+	github.com/ipfs/go-blockservice v0.5.2
 	github.com/ipfs/go-cid v0.4.1
-	github.com/ipfs/go-ipfs-blockstore v1.3.0
-	github.com/ipfs/go-ipfs-chunker v0.0.5
+	github.com/ipfs/go-ipfs-blockstore v1.3.1
+	github.com/ipfs/go-ipfs-chunker v0.0.6
 	github.com/ipfs/go-ipfs-exchange-interface v0.2.1
 	github.com/ipfs/go-ipld-format v0.4.0
 	github.com/ipfs/go-merkledag v0.10.0
-	github.com/ipfs/go-unixfs v0.4.5
-	github.com/libp2p/go-libp2p v0.27.3
+	github.com/ipfs/go-unixfs v0.4.6
+	github.com/libp2p/go-libp2p v0.27.5
 	github.com/mr-tron/base58 v1.2.0
 	github.com/multiformats/go-multibase v0.2.0
-	github.com/multiformats/go-multihash v0.2.1
+	github.com/multiformats/go-multihash v0.2.2
 	github.com/prometheus/client_golang v1.15.1
-	github.com/stretchr/testify v1.8.3
+	github.com/stretchr/testify v1.8.4
 	github.com/tyler-smith/go-bip39 v1.1.0
 	github.com/zeebo/blake3 v0.2.3
 	github.com/zeebo/errs v1.3.0
+	go.uber.org/atomic v1.11.0
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.9.0
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
@@ -55,7 +57,7 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3 // indirect
+	github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/huin/goupnp v1.2.0 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
@@ -88,7 +90,7 @@ require (
 	github.com/multiformats/go-multicodec v0.9.0 // indirect
 	github.com/multiformats/go-multistream v0.4.1 // indirect
 	github.com/multiformats/go-varint v0.0.7 // indirect
-	github.com/onsi/ginkgo/v2 v2.9.5 // indirect
+	github.com/onsi/ginkgo/v2 v2.9.7 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -96,19 +98,18 @@ require (
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.10.0 // indirect
-	github.com/quic-go/quic-go v0.34.0 // indirect
+	github.com/quic-go/quic-go v0.35.1 // indirect
 	github.com/quic-go/webtransport-go v0.5.3 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 // indirect
 	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
 	go.opentelemetry.io/otel v1.7.0 // indirect
 	go.opentelemetry.io/otel/trace v1.7.0 // indirect
-	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/image v0.6.0 // indirect
 	golang.org/x/sync v0.2.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/tools v0.9.1 // indirect
+	golang.org/x/tools v0.9.3 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	lukechampine.com/blake3 v1.2.1 // indirect

go.sum

@@ -51,7 +51,6 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/goccy/go-graphviz v0.1.1 h1:MGrsnzBxTyt7KG8FhHsFPDTGvF7UaQMmSa6A610DqPg=
 github.com/goccy/go-graphviz v0.1.1/go.mod h1:lpnwvVDjskayq84ZxG8tGCPeZX/WxP88W+OJajh+gFk=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
@@ -68,8 +67,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
-github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3 h1:2XF1Vzq06X+inNqgJ9tRnGuw+ZVCB3FazXODD6JE1R8=
-github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk=
+github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs=
+github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
@@ -80,6 +79,8 @@ github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfm
 github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE=
+github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ=
 github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c=
 github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U=
 github.com/huandu/skiplist v1.2.0 h1:gox56QD77HzSC0w+Ws3MH3iie755GBJU1OER3h5VsYw=
@@ -95,8 +96,8 @@ github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJ
 github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk=
 github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo=
 github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE=
-github.com/ipfs/go-blockservice v0.5.1 h1:9pAtkyKAz/skdHTh0kH8VulzWp+qmSDD0aI17TYP/s0=
-github.com/ipfs/go-blockservice v0.5.1/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk=
+github.com/ipfs/go-blockservice v0.5.2 h1:in9Bc+QcXwd1apOVM7Un9t8tixPKdaHQFdLSUM1Xgk8=
+github.com/ipfs/go-blockservice v0.5.2/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk=
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
@@ -110,11 +111,11 @@ github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0M
 github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8=
 github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
 github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
-github.com/ipfs/go-ipfs-blockstore v1.3.0 h1:m2EXaWgwTzAfsmt5UdJ7Is6l4gJcaM/A12XwJyvYvMM=
-github.com/ipfs/go-ipfs-blockstore v1.3.0/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE=
+github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ=
+github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE=
 github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ=
-github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8=
-github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8=
+github.com/ipfs/go-ipfs-chunker v0.0.6 h1:+EBescK+ekHPMfmX3VYXRyOn/f80RXv7V8tGgLEYvo8=
+github.com/ipfs/go-ipfs-chunker v0.0.6/go.mod h1:whszqTIBqWNUvYvjkKvBSoR1akrsSpZbZCYMFbekMjE=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ=
 github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q=
@@ -139,7 +140,6 @@ github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSg
 github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM=
 github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc=
 github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg=
-github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
 github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8=
 github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo=
 github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g=
@@ -150,8 +150,8 @@ github.com/ipfs/go-merkledag v0.10.0/go.mod h1:zkVav8KiYlmbzUzNM6kENzkdP5+qR7+2m
 github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
 github.com/ipfs/go-peertaskqueue v0.8.0 h1:JyNO144tfu9bx6Hpo119zvbEL9iQ760FHOiJYsUjqaU=
-github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU=
-github.com/ipfs/go-unixfs v0.4.5/go.mod h1:BIznJNvt/gEx/ooRMI4Us9K8+qeGO7vx1ohnbk8gjFg=
+github.com/ipfs/go-unixfs v0.4.6 h1:4PCH8+ptflEqmD1ifrdjGu0hA/MfM1s4QlrsQb4BvJM=
+github.com/ipfs/go-unixfs v0.4.6/go.mod h1:BIznJNvt/gEx/ooRMI4Us9K8+qeGO7vx1ohnbk8gjFg=
 github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
 github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
@@ -168,7 +168,6 @@ github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZl
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
@@ -189,8 +188,8 @@ github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoR
 github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8=
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
 github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c=
-github.com/libp2p/go-libp2p v0.27.3 h1:tkV/zm3KCZ4R5er9Xcs2pt0YNB4JH0iBfGAtHJdLHRs=
-github.com/libp2p/go-libp2p v0.27.3/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE=
+github.com/libp2p/go-libp2p v0.27.5 h1:KwA7pXKXpz8hG6Cr1fMA7UkgleogcwQj0sxl5qquWRg=
+github.com/libp2p/go-libp2p v0.27.5/go.mod h1:oMfQGTb9CHnrOuSM6yMmyK2lXz3qIhnkn2+oK3B1Y2g=
 github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s=
 github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
 github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA=
@@ -198,8 +197,6 @@ github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0
 github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg=
 github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU=
 github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rBfZQ=
-github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
-github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
@@ -238,8 +235,8 @@ github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpK
 github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
 github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
 github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
-github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108=
-github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc=
+github.com/multiformats/go-multihash v0.2.2 h1:Uu7LWs/PmWby1gkj1S1DXx3zyd3aVabA4FiMKn/2tAc=
+github.com/multiformats/go-multihash v0.2.2/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM=
 github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo=
 github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q=
 github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
@@ -247,9 +244,8 @@ github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS
 github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
 github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY=
-github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
-github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
-github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss=
+github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0=
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -272,8 +268,8 @@ github.com/prometheus/procfs v0.10.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPH
 github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
 github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U=
 github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E=
-github.com/quic-go/quic-go v0.34.0 h1:OvOJ9LFjTySgwOTYUZmNoq0FzVicP8YujpV0kB7m2lU=
-github.com/quic-go/quic-go v0.34.0/go.mod h1:+4CVgVppm0FNjpG3UcX8Joi/frKOH7/ciD5yGcwOO1g=
+github.com/quic-go/quic-go v0.35.1 h1:b0kzj6b/cQAf05cT0CkQubHM31wiA+xH3IBkxP62poo=
+github.com/quic-go/quic-go v0.35.1/go.mod h1:+4CVgVppm0FNjpG3UcX8Joi/frKOH7/ciD5yGcwOO1g=
 github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU=
 github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -291,8 +287,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
-github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8=
 github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U=
 github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U=
@@ -303,7 +299,6 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 h1:WXhVOwj2
 github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
-github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
@@ -358,7 +353,6 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
-golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -381,7 +375,6 @@ golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
 golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -409,7 +402,6 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -423,8 +415,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
-golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
+golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM=
+golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

net/connutil/timeout.go

@@ -1,4 +1,4 @@
-package timeoutconn
+package connutil
 
 import (
 	"errors"
@@ -10,18 +10,18 @@ import (
 	"go.uber.org/zap"
 )
 
-var log = logger.NewNamed("common.net.timeoutconn")
+var log = logger.NewNamed("common.net.connutil")
 
-type Conn struct {
+type TimeoutConn struct {
 	net.Conn
 	timeout time.Duration
 }
 
-func NewConn(conn net.Conn, timeout time.Duration) *Conn {
-	return &Conn{conn, timeout}
+func NewTimeout(conn net.Conn, timeout time.Duration) *TimeoutConn {
+	return &TimeoutConn{conn, timeout}
 }
 
-func (c *Conn) Write(p []byte) (n int, err error) {
+func (c *TimeoutConn) Write(p []byte) (n int, err error) {
 	for {
 		if c.timeout != 0 {
 			if e := c.Conn.SetWriteDeadline(time.Now().Add(c.timeout)); e != nil {

net/connutil/usage.go

@@ -0,0 +1,30 @@
+package connutil
+
+import (
+	"go.uber.org/atomic"
+	"net"
+	"time"
+)
+
+func NewLastUsageConn(conn net.Conn) *LastUsageConn {
+	return &LastUsageConn{Conn: conn}
+}
+
+type LastUsageConn struct {
+	net.Conn
+	lastUsage atomic.Time
+}
+
+func (c *LastUsageConn) Write(p []byte) (n int, err error) {
+	c.lastUsage.Store(time.Now())
+	return c.Conn.Write(p)
+}
+
+func (c *LastUsageConn) Read(p []byte) (n int, err error) {
+	c.lastUsage.Store(time.Now())
+	return c.Conn.Read(p)
+}
+
+func (c *LastUsageConn) LastUsage() time.Time {
+	return c.lastUsage.Load()
+}

net/dialer/dialer.go

@@ -1,137 +0,0 @@
-package dialer
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"github.com/anyproto/any-sync/app"
-	"github.com/anyproto/any-sync/app/logger"
-	net2 "github.com/anyproto/any-sync/net"
-	"github.com/anyproto/any-sync/net/peer"
-	"github.com/anyproto/any-sync/net/secureservice"
-	"github.com/anyproto/any-sync/net/secureservice/handshake"
-	"github.com/anyproto/any-sync/net/timeoutconn"
-	"github.com/anyproto/any-sync/nodeconf"
-	"github.com/libp2p/go-libp2p/core/sec"
-	"go.uber.org/zap"
-	"net"
-	"storj.io/drpc"
-	"storj.io/drpc/drpcconn"
-	"storj.io/drpc/drpcmanager"
-	"storj.io/drpc/drpcwire"
-	"sync"
-	"time"
-)
-
-const CName = "common.net.dialer"
-
-var (
-	ErrAddrsNotFound      = errors.New("addrs for peer not found")
-	ErrPeerIdIsUnexpected = errors.New("expected to connect with other peer id")
-)
-
-var log = logger.NewNamed(CName)
-
-func New() Dialer {
-	return &dialer{peerAddrs: map[string][]string{}}
-}
-
-type Dialer interface {
-	Dial(ctx context.Context, peerId string) (peer peer.Peer, err error)
-	SetPeerAddrs(peerId string, addrs []string)
-	app.Component
-}
-
-type dialer struct {
-	transport secureservice.SecureService
-	config    net2.Config
-	nodeConf  nodeconf.NodeConf
-	peerAddrs map[string][]string
-
-	mu sync.RWMutex
-}
-
-func (d *dialer) Init(a *app.App) (err error) {
-	d.transport = a.MustComponent(secureservice.CName).(secureservice.SecureService)
-	d.nodeConf = a.MustComponent(nodeconf.CName).(nodeconf.NodeConf)
-	d.config = a.MustComponent("config").(net2.ConfigGetter).GetNet()
-	return
-}
-
-func (d *dialer) Name() (name string) {
-	return CName
-}
-
-func (d *dialer) SetPeerAddrs(peerId string, addrs []string) {
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	d.peerAddrs[peerId] = addrs
-}
-
-func (d *dialer) getPeerAddrs(peerId string) ([]string, error) {
-	if addrs, ok := d.nodeConf.PeerAddresses(peerId); ok {
-		return addrs, nil
-	}
-	addrs, ok := d.peerAddrs[peerId]
-	if !ok || len(addrs) == 0 {
-		return nil, ErrAddrsNotFound
-	}
-	return addrs, nil
-}
-
-func (d *dialer) Dial(ctx context.Context, peerId string) (p peer.Peer, err error) {
-	var ctxCancel context.CancelFunc
-	ctx, ctxCancel = context.WithTimeout(ctx, time.Second*10)
-	defer ctxCancel()
-	d.mu.RLock()
-	defer d.mu.RUnlock()
-
-	addrs, err := d.getPeerAddrs(peerId)
-	if err != nil {
-		return
-	}
-
-	var (
-		conn drpc.Conn
-		sc   sec.SecureConn
-	)
-	log.InfoCtx(ctx, "dial", zap.String("peerId", peerId), zap.Strings("addrs", addrs))
-	for _, addr := range addrs {
-		conn, sc, err = d.handshake(ctx, addr, peerId)
-		if err != nil {
-			log.InfoCtx(ctx, "can't connect to host", zap.String("addr", addr), zap.Error(err))
-		} else {
-			break
-		}
-	}
-	if err != nil {
-		return
-	}
-	return peer.NewPeer(sc, conn), nil
-}
-
-func (d *dialer) handshake(ctx context.Context, addr, peerId string) (conn drpc.Conn, sc sec.SecureConn, err error) {
-	st := time.Now()
-	// TODO: move dial timeout to config
-	tcpConn, err := net.DialTimeout("tcp", addr, time.Second*15)
-	if err != nil {
-		return nil, nil, fmt.Errorf("dialTimeout error: %v; since start: %v", err, time.Since(st))
-	}
-
-	timeoutConn := timeoutconn.NewConn(tcpConn, time.Millisecond*time.Duration(d.config.Stream.TimeoutMilliseconds))
-	sc, err = d.transport.SecureOutbound(ctx, timeoutConn)
-	if err != nil {
-		if he, ok := err.(handshake.HandshakeError); ok {
-			return nil, nil, he
-		}
-		return nil, nil, fmt.Errorf("tls handshaeke error: %v; since start: %v", err, time.Since(st))
-	}
-	if peerId != sc.RemotePeer().String() {
-		return nil, nil, ErrPeerIdIsUnexpected
-	}
-	log.Info("connected with remote host", zap.String("serverPeer", sc.RemotePeer().String()), zap.String("addr", addr))
-	conn = drpcconn.NewWithOptions(sc, drpcconn.Options{Manager: drpcmanager.Options{
-		Reader: drpcwire.ReaderOptions{MaximumBufferSize: d.config.Stream.MaxMsgSizeMb * (1 << 20)},
-	}})
-	return conn, sc, err
-}

net/peer/peer.go

@@ -2,99 +2,220 @@ package peer
 
 import (
 	"context"
-	"sync/atomic"
-	"time"
-
 	"github.com/anyproto/any-sync/app/logger"
-	"github.com/libp2p/go-libp2p/core/sec"
+	"github.com/anyproto/any-sync/app/ocache"
+	"github.com/anyproto/any-sync/net/connutil"
+	"github.com/anyproto/any-sync/net/secureservice/handshake"
+	"github.com/anyproto/any-sync/net/secureservice/handshake/handshakeproto"
+	"github.com/anyproto/any-sync/net/transport"
 	"go.uber.org/zap"
+	"io"
+	"net"
 	"storj.io/drpc"
+	"storj.io/drpc/drpcconn"
+	"sync"
+	"time"
 )
 
 var log = logger.NewNamed("common.net.peer")
 
-func NewPeer(sc sec.SecureConn, conn drpc.Conn) Peer {
-	return &peer{
-		id:        sc.RemotePeer().String(),
-		lastUsage: time.Now().Unix(),
-		sc:        sc,
-		Conn:      conn,
+type connCtrl interface {
+	ServeConn(ctx context.Context, conn net.Conn) (err error)
+}
+
+func NewPeer(mc transport.MultiConn, ctrl connCtrl) (p Peer, err error) {
+	ctx := mc.Context()
+	pr := &peer{
+		active:    map[*subConn]struct{}{},
+		MultiConn: mc,
+		ctrl:      ctrl,
 	}
+	if pr.id, err = CtxPeerId(ctx); err != nil {
+		return
+	}
+	go pr.acceptLoop()
+	return pr, nil
 }
 
 type Peer interface {
 	Id() string
-	LastUsage() time.Time
-	UpdateLastUsage()
-	Addr() string
+	Context() context.Context
+
+	AcquireDrpcConn(ctx context.Context) (drpc.Conn, error)
+	ReleaseDrpcConn(conn drpc.Conn)
+	DoDrpc(ctx context.Context, do func(conn drpc.Conn) error) error
+
+	IsClosed() bool
+
 	TryClose(objectTTL time.Duration) (res bool, err error)
+
+	ocache.Object
+}
+
+type subConn struct {
 	drpc.Conn
+	*connutil.LastUsageConn
 }
 
 type peer struct {
-	id        string
-	ttl       time.Duration
-	lastUsage int64
-	sc        sec.SecureConn
-	drpc.Conn
+	id string
+
+	ctrl connCtrl
+
+	// drpc conn pool
+	inactive []*subConn
+	active   map[*subConn]struct{}
+
+	mu sync.Mutex
+
+	transport.MultiConn
 }
 
 func (p *peer) Id() string {
 	return p.id
 }
 
-func (p *peer) LastUsage() time.Time {
-	select {
-	case <-p.Closed():
-		return time.Unix(0, 0)
-	default:
+func (p *peer) AcquireDrpcConn(ctx context.Context) (drpc.Conn, error) {
+	p.mu.Lock()
+	if len(p.inactive) == 0 {
+		p.mu.Unlock()
+		dconn, err := p.openDrpcConn(ctx)
+		if err != nil {
+			return nil, err
+		}
+		p.mu.Lock()
+		p.inactive = append(p.inactive, dconn)
 	}
-	return time.Unix(atomic.LoadInt64(&p.lastUsage), 0)
+	idx := len(p.inactive) - 1
+	res := p.inactive[idx]
+	p.inactive = p.inactive[:idx]
+	p.active[res] = struct{}{}
+	p.mu.Unlock()
+	return res, nil
 }
 
-func (p *peer) Invoke(ctx context.Context, rpc string, enc drpc.Encoding, in, out drpc.Message) error {
-	defer p.UpdateLastUsage()
-	return p.Conn.Invoke(ctx, rpc, enc, in, out)
-}
-
-func (p *peer) NewStream(ctx context.Context, rpc string, enc drpc.Encoding) (drpc.Stream, error) {
-	defer p.UpdateLastUsage()
-	return p.Conn.NewStream(ctx, rpc, enc)
-}
-
-func (p *peer) Read(b []byte) (n int, err error) {
-	if n, err = p.sc.Read(b); err == nil {
-		p.UpdateLastUsage()
+func (p *peer) ReleaseDrpcConn(conn drpc.Conn) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	sc, ok := conn.(*subConn)
+	if !ok {
+		return
 	}
+	if _, ok = p.active[sc]; ok {
+		delete(p.active, sc)
+	}
+	p.inactive = append(p.inactive, sc)
 	return
 }
 
-func (p *peer) Write(b []byte) (n int, err error) {
-	if n, err = p.sc.Write(b); err == nil {
-		p.UpdateLastUsage()
+func (p *peer) DoDrpc(ctx context.Context, do func(conn drpc.Conn) error) error {
+	conn, err := p.AcquireDrpcConn(ctx)
+	if err != nil {
+		return err
 	}
-	return
+	defer p.ReleaseDrpcConn(conn)
+	return do(conn)
 }
 
-func (p *peer) UpdateLastUsage() {
-	atomic.StoreInt64(&p.lastUsage, time.Now().Unix())
+func (p *peer) openDrpcConn(ctx context.Context) (dconn *subConn, err error) {
+	conn, err := p.Open(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if err = handshake.OutgoingProtoHandshake(ctx, conn, handshakeproto.ProtoType_DRPC); err != nil {
+		return nil, err
+	}
+	tconn := connutil.NewLastUsageConn(conn)
+	return &subConn{
+		Conn:          drpcconn.New(tconn),
+		LastUsageConn: tconn,
+	}, nil
+}
+
+func (p *peer) acceptLoop() {
+	var exitErr error
+	defer func() {
+		if exitErr != transport.ErrConnClosed {
+			log.Warn("accept error: close connection", zap.Error(exitErr))
+			_ = p.MultiConn.Close()
+		}
+	}()
+	for {
+		conn, err := p.Accept()
+		if err != nil {
+			exitErr = err
+			return
+		}
+		go func() {
+			serveErr := p.serve(conn)
+			if serveErr != io.EOF && serveErr != transport.ErrConnClosed {
+				log.InfoCtx(p.Context(), "serve connection error", zap.Error(serveErr))
+			}
+		}()
+	}
+}
+
+var defaultProtoChecker = handshake.ProtoChecker{
+	AllowedProtoTypes: []handshakeproto.ProtoType{
+		handshakeproto.ProtoType_DRPC,
+	},
+}
+
+func (p *peer) serve(conn net.Conn) (err error) {
+	hsCtx, cancel := context.WithTimeout(p.Context(), time.Second*20)
+	if _, err = handshake.IncomingProtoHandshake(hsCtx, conn, defaultProtoChecker); err != nil {
+		cancel()
+		return
+	}
+	cancel()
+	return p.ctrl.ServeConn(p.Context(), conn)
 }
 
 func (p *peer) TryClose(objectTTL time.Duration) (res bool, err error) {
+	p.gc(objectTTL)
 	if time.Now().Sub(p.LastUsage()) < objectTTL {
 		return false, nil
 	}
 	return true, p.Close()
 }
 
-func (p *peer) Addr() string {
-	if p.sc != nil {
-		return p.sc.RemoteAddr().String()
+func (p *peer) gc(ttl time.Duration) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	minLastUsage := time.Now().Add(-ttl)
+	var hasClosed bool
+	for i, in := range p.inactive {
+		select {
+		case <-in.Closed():
+			p.inactive[i] = nil
+			hasClosed = true
+		default:
+		}
+		if in.LastUsage().Before(minLastUsage) {
+			_ = in.Close()
+			p.inactive[i] = nil
+			hasClosed = true
+		}
+	}
+	if hasClosed {
+		inactive := p.inactive
+		p.inactive = p.inactive[:0]
+		for _, in := range inactive {
+			if in != nil {
+				p.inactive = append(p.inactive, in)
+			}
+		}
+	}
+	for act := range p.active {
+		select {
+		case <-act.Closed():
+			delete(p.active, act)
+		default:
+		}
 	}
-	return ""
 }
 
 func (p *peer) Close() (err error) {
 	log.Debug("peer close", zap.String("peerId", p.id))
-	return p.Conn.Close()
+	return p.MultiConn.Close()
 }

net/peer/peer_test.go

@@ -0,0 +1,187 @@
+package peer
+
+import (
+	"context"
+	"github.com/anyproto/any-sync/net/secureservice/handshake"
+	"github.com/anyproto/any-sync/net/secureservice/handshake/handshakeproto"
+	"github.com/anyproto/any-sync/net/transport/mock_transport"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"io"
+	"net"
+	_ "net/http/pprof"
+	"testing"
+	"time"
+)
+
+var ctx = context.Background()
+
+func TestPeer_AcquireDrpcConn(t *testing.T) {
+	fx := newFixture(t, "p1")
+	defer fx.finish()
+	in, out := net.Pipe()
+	go func() {
+		handshake.IncomingProtoHandshake(ctx, out, defaultProtoChecker)
+	}()
+	defer out.Close()
+	fx.mc.EXPECT().Open(gomock.Any()).Return(in, nil)
+	dc, err := fx.AcquireDrpcConn(ctx)
+	require.NoError(t, err)
+	assert.NotEmpty(t, dc)
+	defer dc.Close()
+
+	assert.Len(t, fx.active, 1)
+	assert.Len(t, fx.inactive, 0)
+
+	fx.ReleaseDrpcConn(dc)
+
+	assert.Len(t, fx.active, 0)
+	assert.Len(t, fx.inactive, 1)
+
+	dc, err = fx.AcquireDrpcConn(ctx)
+	require.NoError(t, err)
+	assert.NotEmpty(t, dc)
+	assert.Len(t, fx.active, 1)
+	assert.Len(t, fx.inactive, 0)
+}
+
+func TestPeerAccept(t *testing.T) {
+	fx := newFixture(t, "p1")
+	defer fx.finish()
+	in, out := net.Pipe()
+	defer out.Close()
+
+	var outHandshakeCh = make(chan error)
+	go func() {
+		outHandshakeCh <- handshake.OutgoingProtoHandshake(ctx, out, handshakeproto.ProtoType_DRPC)
+	}()
+	fx.acceptCh <- acceptedConn{conn: in}
+	cn := <-fx.testCtrl.serveConn
+	assert.Equal(t, in, cn)
+	assert.NoError(t, <-outHandshakeCh)
+}
+
+func TestPeer_TryClose(t *testing.T) {
+	t.Run("ttl", func(t *testing.T) {
+		fx := newFixture(t, "p1")
+		defer fx.finish()
+		lu := time.Now()
+		fx.mc.EXPECT().LastUsage().Return(lu)
+		res, err := fx.TryClose(time.Second)
+		require.NoError(t, err)
+		assert.False(t, res)
+	})
+	t.Run("close", func(t *testing.T) {
+		fx := newFixture(t, "p1")
+		defer fx.finish()
+		lu := time.Now().Add(-time.Second * 2)
+		fx.mc.EXPECT().LastUsage().Return(lu)
+		res, err := fx.TryClose(time.Second)
+		require.NoError(t, err)
+		assert.True(t, res)
+	})
+	t.Run("gc", func(t *testing.T) {
+		fx := newFixture(t, "p1")
+		defer fx.finish()
+		now := time.Now()
+		fx.mc.EXPECT().LastUsage().Return(now.Add(time.Millisecond * 100))
+
+		// make one inactive
+		in, out := net.Pipe()
+		go func() {
+			handshake.IncomingProtoHandshake(ctx, out, defaultProtoChecker)
+		}()
+		defer out.Close()
+		fx.mc.EXPECT().Open(gomock.Any()).Return(in, nil)
+		dc, err := fx.AcquireDrpcConn(ctx)
+		require.NoError(t, err)
+
+		// make one active but closed
+		in2, out2 := net.Pipe()
+		go func() {
+			handshake.IncomingProtoHandshake(ctx, out2, defaultProtoChecker)
+		}()
+		defer out2.Close()
+		fx.mc.EXPECT().Open(gomock.Any()).Return(in2, nil)
+		dc2, err := fx.AcquireDrpcConn(ctx)
+		require.NoError(t, err)
+		_ = dc2.Close()
+
+		// make one inactive and closed
+		in3, out3 := net.Pipe()
+		go func() {
+			handshake.IncomingProtoHandshake(ctx, out3, defaultProtoChecker)
+		}()
+		defer out3.Close()
+		fx.mc.EXPECT().Open(gomock.Any()).Return(in3, nil)
+		dc3, err := fx.AcquireDrpcConn(ctx)
+		require.NoError(t, err)
+		fx.ReleaseDrpcConn(dc3)
+		_ = dc3.Close()
+		fx.ReleaseDrpcConn(dc)
+
+		time.Sleep(time.Millisecond * 100)
+
+		res, err := fx.TryClose(time.Millisecond * 50)
+		require.NoError(t, err)
+		assert.False(t, res)
+	})
+}
+
+type acceptedConn struct {
+	conn net.Conn
+	err  error
+}
+
+func newFixture(t *testing.T, peerId string) *fixture {
+	fx := &fixture{
+		ctrl:     gomock.NewController(t),
+		acceptCh: make(chan acceptedConn),
+		testCtrl: newTesCtrl(),
+	}
+	fx.mc = mock_transport.NewMockMultiConn(fx.ctrl)
+	ctx := CtxWithPeerId(context.Background(), peerId)
+	fx.mc.EXPECT().Context().Return(ctx).AnyTimes()
+	fx.mc.EXPECT().Accept().DoAndReturn(func() (net.Conn, error) {
+		ac := <-fx.acceptCh
+		return ac.conn, ac.err
+	}).AnyTimes()
+	fx.mc.EXPECT().Close().AnyTimes()
+	p, err := NewPeer(fx.mc, fx.testCtrl)
+	require.NoError(t, err)
+	fx.peer = p.(*peer)
+	return fx
+}
+
+type fixture struct {
+	*peer
+	ctrl     *gomock.Controller
+	mc       *mock_transport.MockMultiConn
+	acceptCh chan acceptedConn
+	testCtrl *testCtrl
+}
+
+func (fx *fixture) finish() {
+	fx.testCtrl.close()
+	fx.ctrl.Finish()
+}
+
+func newTesCtrl() *testCtrl {
+	return &testCtrl{closeCh: make(chan struct{}), serveConn: make(chan net.Conn, 10)}
+}
+
+type testCtrl struct {
+	serveConn chan net.Conn
+	closeCh   chan struct{}
+}
+
+func (t *testCtrl) ServeConn(ctx context.Context, conn net.Conn) (err error) {
+	t.serveConn <- conn
+	<-t.closeCh
+	return io.EOF
+}
+
+func (t *testCtrl) close() {
+	close(t.closeCh)
+}

net/peerservice/peerservice.go

@@ -0,0 +1,110 @@
+package peerservice
+
+import (
+	"context"
+	"errors"
+	"github.com/anyproto/any-sync/app"
+	"github.com/anyproto/any-sync/app/logger"
+	"github.com/anyproto/any-sync/net/peer"
+	"github.com/anyproto/any-sync/net/pool"
+	"github.com/anyproto/any-sync/net/rpc/server"
+	"github.com/anyproto/any-sync/net/transport"
+	"github.com/anyproto/any-sync/net/transport/yamux"
+	"github.com/anyproto/any-sync/nodeconf"
+	"go.uber.org/zap"
+	"sync"
+)
+
+const CName = "net.peerservice"
+
+var log = logger.NewNamed(CName)
+
+var (
+	ErrAddrsNotFound = errors.New("addrs for peer not found")
+)
+
+func New() PeerService {
+	return new(peerService)
+}
+
+type PeerService interface {
+	Dial(ctx context.Context, peerId string) (pr peer.Peer, err error)
+	SetPeerAddrs(peerId string, addrs []string)
+	transport.Accepter
+	app.Component
+}
+
+type peerService struct {
+	yamux     transport.Transport
+	nodeConf  nodeconf.NodeConf
+	peerAddrs map[string][]string
+	pool      pool.Pool
+	server    server.DRPCServer
+	mu        sync.RWMutex
+}
+
+func (p *peerService) Init(a *app.App) (err error) {
+	p.yamux = a.MustComponent(yamux.CName).(transport.Transport)
+	p.nodeConf = a.MustComponent(nodeconf.CName).(nodeconf.NodeConf)
+	p.pool = a.MustComponent(pool.CName).(pool.Pool)
+	p.server = a.MustComponent(server.CName).(server.DRPCServer)
+	p.peerAddrs = map[string][]string{}
+	return nil
+}
+
+func (p *peerService) Name() (name string) {
+	return CName
+}
+
+func (p *peerService) Dial(ctx context.Context, peerId string) (pr peer.Peer, err error) {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+
+	addrs, err := p.getPeerAddrs(peerId)
+	if err != nil {
+		return
+	}
+
+	var mc transport.MultiConn
+	log.InfoCtx(ctx, "dial", zap.String("peerId", peerId), zap.Strings("addrs", addrs))
+	for _, addr := range addrs {
+		mc, err = p.yamux.Dial(ctx, addr)
+		if err != nil {
+			log.InfoCtx(ctx, "can't connect to host", zap.String("addr", addr), zap.Error(err))
+		} else {
+			break
+		}
+	}
+	if err != nil {
+		return
+	}
+	return peer.NewPeer(mc, p.server)
+}
+
+func (p *peerService) Accept(mc transport.MultiConn) (err error) {
+	pr, err := peer.NewPeer(mc, p.server)
+	if err != nil {
+		return err
+	}
+	if err = p.pool.AddPeer(context.Background(), pr); err != nil {
+		_ = pr.Close()
+	}
+	return
+}
+
+func (p *peerService) SetPeerAddrs(peerId string, addrs []string) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.peerAddrs[peerId] = addrs
+}
+
+func (p *peerService) getPeerAddrs(peerId string) ([]string, error) {
+	if addrs, ok := p.nodeConf.PeerAddresses(peerId); ok {
+		return addrs, nil
+	}
+	addrs, ok := p.peerAddrs[peerId]
+	if !ok || len(addrs) == 0 {
+		return nil, ErrAddrsNotFound
+	}
+	return addrs, nil
+}

net/pool/pool.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"github.com/anyproto/any-sync/app/ocache"
 	"github.com/anyproto/any-sync/net"
-	"github.com/anyproto/any-sync/net/dialer"
 	"github.com/anyproto/any-sync/net/peer"
 	"github.com/anyproto/any-sync/net/secureservice/handshake"
 	"go.uber.org/zap"
@@ -13,59 +12,61 @@ import (
 
 // Pool creates and caches outgoing connection
 type Pool interface {
-	// Get lookups to peer in existing connections or creates and cache new one
+	// Get lookups to peer in existing connections or creates and outgoing new one
 	Get(ctx context.Context, id string) (peer.Peer, error)
-	// Dial creates new connection to peer and not use cache
-	Dial(ctx context.Context, id string) (peer.Peer, error)
-	// GetOneOf searches at least one existing connection in cache or creates a new one from a randomly selected id from given list
+	// GetOneOf searches at least one existing connection in outgoing or creates a new one from a randomly selected id from given list
 	GetOneOf(ctx context.Context, peerIds []string) (peer.Peer, error)
-
-	DialOneOf(ctx context.Context, peerIds []string) (peer.Peer, error)
+	// AddPeer adds incoming peer to the pool
+	AddPeer(ctx context.Context, p peer.Peer) (err error)
 }
 
 type pool struct {
-	cache  ocache.OCache
-	dialer dialer.Dialer
+	outgoing ocache.OCache
+	incoming ocache.OCache
 }
 
 func (p *pool) Name() (name string) {
 	return CName
 }
 
-func (p *pool) Run(ctx context.Context) (err error) {
-	return nil
+func (p *pool) Get(ctx context.Context, id string) (pr peer.Peer, err error) {
+	// if we have incoming connection - try to reuse it
+	if pr, err = p.get(ctx, p.incoming, id); err != nil {
+		// or try to get or create outgoing
+		return p.get(ctx, p.outgoing, id)
+	}
+	return
 }
 
-func (p *pool) Get(ctx context.Context, id string) (peer.Peer, error) {
-	v, err := p.cache.Get(ctx, id)
+func (p *pool) get(ctx context.Context, source ocache.OCache, id string) (peer.Peer, error) {
+	v, err := source.Get(ctx, id)
 	if err != nil {
 		return nil, err
 	}
 	pr := v.(peer.Peer)
-	select {
-	case <-pr.Closed():
-	default:
+	if !pr.IsClosed() {
 		return pr, nil
 	}
-	_, _ = p.cache.Remove(ctx, id)
+	_, _ = source.Remove(ctx, id)
 	return p.Get(ctx, id)
 }
 
-func (p *pool) Dial(ctx context.Context, id string) (peer.Peer, error) {
-	return p.dialer.Dial(ctx, id)
-}
-
 func (p *pool) GetOneOf(ctx context.Context, peerIds []string) (peer.Peer, error) {
 	// finding existing connection
 	for _, peerId := range peerIds {
-		if v, err := p.cache.Pick(ctx, peerId); err == nil {
+		if v, err := p.incoming.Pick(ctx, peerId); err == nil {
 			pr := v.(peer.Peer)
-			select {
-			case <-pr.Closed():
-			default:
+			if !pr.IsClosed() {
 				return pr, nil
 			}
-			_, _ = p.cache.Remove(ctx, peerId)
+			_, _ = p.incoming.Remove(ctx, peerId)
+		}
+		if v, err := p.outgoing.Pick(ctx, peerId); err == nil {
+			pr := v.(peer.Peer)
+			if !pr.IsClosed() {
+				return pr, nil
+			}
+			_, _ = p.outgoing.Remove(ctx, peerId)
 		}
 	}
 	// shuffle ids for better consistency
@@ -75,8 +76,8 @@ func (p *pool) GetOneOf(ctx context.Context, peerIds []string) (peer.Peer, error
 	// connecting
 	var lastErr error
 	for _, peerId := range peerIds {
-		if v, err := p.cache.Get(ctx, peerId); err == nil {
-			return v.(peer.Peer), nil
+		if v, err := p.Get(ctx, peerId); err == nil {
+			return v, nil
 		} else {
 			log.Debug("unable to connect", zap.String("peerId", peerId), zap.Error(err))
 			lastErr = err
@@ -88,27 +89,18 @@ func (p *pool) GetOneOf(ctx context.Context, peerIds []string) (peer.Peer, error
 	return nil, lastErr
 }
 
-func (p *pool) DialOneOf(ctx context.Context, peerIds []string) (peer.Peer, error) {
-	// shuffle ids for better consistency
-	rand.Shuffle(len(peerIds), func(i, j int) {
-		peerIds[i], peerIds[j] = peerIds[j], peerIds[i]
-	})
-	// connecting
-	var lastErr error
-	for _, peerId := range peerIds {
-		if v, err := p.dialer.Dial(ctx, peerId); err == nil {
-			return v.(peer.Peer), nil
+func (p *pool) AddPeer(ctx context.Context, pr peer.Peer) (err error) {
+	if err = p.incoming.Add(pr.Id(), pr); err != nil {
+		if err == ocache.ErrExists {
+			// in case when an incoming connection with a peer already exists, we close and remove an existing connection
+			if v, e := p.incoming.Pick(ctx, pr.Id()); e == nil {
+				_ = v.Close()
+				_, _ = p.incoming.Remove(ctx, pr.Id())
+				return p.incoming.Add(pr.Id(), pr)
+			}
 		} else {
-			log.Debug("unable to connect", zap.String("peerId", peerId), zap.Error(err))
-			lastErr = err
+			return err
 		}
 	}
-	if _, ok := lastErr.(handshake.HandshakeError); !ok {
-		lastErr = net.ErrUnableToConnect
-	}
-	return nil, lastErr
-}
-
-func (p *pool) Close(ctx context.Context) (err error) {
-	return p.cache.Close()
+	return
 }

net/pool/pool_test.go

@@ -6,11 +6,11 @@ import (
 	"fmt"
 	"github.com/anyproto/any-sync/app"
 	"github.com/anyproto/any-sync/net"
-	"github.com/anyproto/any-sync/net/dialer"
 	"github.com/anyproto/any-sync/net/peer"
 	"github.com/anyproto/any-sync/net/secureservice/handshake"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	net2 "net"
 	"storj.io/drpc"
 	"testing"
 	"time"
@@ -133,6 +133,27 @@ func TestPool_GetOneOf(t *testing.T) {
 	})
 }
 
+func TestPool_AddPeer(t *testing.T) {
+	t.Run("success", func(t *testing.T) {
+		fx := newFixture(t)
+		defer fx.Finish()
+		require.NoError(t, fx.AddPeer(ctx, newTestPeer("p1")))
+	})
+	t.Run("two peers", func(t *testing.T) {
+		fx := newFixture(t)
+		defer fx.Finish()
+		p1, p2 := newTestPeer("p1"), newTestPeer("p1")
+		require.NoError(t, fx.AddPeer(ctx, p1))
+		require.NoError(t, fx.AddPeer(ctx, p2))
+		select {
+		case <-p1.closed:
+		default:
+			assert.Truef(t, false, "peer not closed")
+		}
+	})
+
+}
+
 func newFixture(t *testing.T) *fixture {
 	fx := &fixture{
 		Service: New(),
@@ -158,7 +179,7 @@ type fixture struct {
 	t      *testing.T
 }
 
-var _ dialer.Dialer = (*dialerMock)(nil)
+var _ dialer = (*dialerMock)(nil)
 
 type dialerMock struct {
 	dial func(ctx context.Context, peerId string) (peer peer.Peer, err error)
@@ -181,7 +202,7 @@ func (d *dialerMock) Init(a *app.App) (err error) {
 }
 
 func (d *dialerMock) Name() (name string) {
-	return dialer.CName
+	return "net.peerservice"
 }
 
 func newTestPeer(id string) *testPeer {
@@ -196,6 +217,31 @@ type testPeer struct {
 	closed chan struct{}
 }
 
+func (t *testPeer) DoDrpc(ctx context.Context, do func(conn drpc.Conn) error) error {
+	return fmt.Errorf("not implemented")
+}
+
+func (t *testPeer) AcquireDrpcConn(ctx context.Context) (drpc.Conn, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (t *testPeer) ReleaseDrpcConn(conn drpc.Conn) {}
+
+func (t *testPeer) Context() context.Context {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (t *testPeer) Accept() (conn net2.Conn, err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (t *testPeer) Open(ctx context.Context) (conn net2.Conn, err error) {
+	//TODO implement me
+	panic("implement me")
+}
+
 func (t *testPeer) Addr() string {
 	return ""
 }
@@ -204,12 +250,6 @@ func (t *testPeer) Id() string {
 	return t.id
 }
 
-func (t *testPeer) LastUsage() time.Time {
-	return time.Now()
-}
-
-func (t *testPeer) UpdateLastUsage() {}
-
 func (t *testPeer) TryClose(objectTTL time.Duration) (res bool, err error) {
 	return true, t.Close()
 }
@@ -224,14 +264,11 @@ func (t *testPeer) Close() error {
 	return nil
 }
 
-func (t *testPeer) Closed() <-chan struct{} {
-	return t.closed
-}
-
-func (t *testPeer) Invoke(ctx context.Context, rpc string, enc drpc.Encoding, in, out drpc.Message) error {
-	return fmt.Errorf("call Invoke on test peer")
-}
-
-func (t *testPeer) NewStream(ctx context.Context, rpc string, enc drpc.Encoding) (drpc.Stream, error) {
-	return nil, fmt.Errorf("call NewStream on test peer")
+func (t *testPeer) IsClosed() bool {
+	select {
+	case <-t.closed:
+		return true
+	default:
+		return false
+	}
 }

net/pool/poolservice.go

@@ -6,8 +6,9 @@ import (
 	"github.com/anyproto/any-sync/app/logger"
 	"github.com/anyproto/any-sync/app/ocache"
 	"github.com/anyproto/any-sync/metric"
-	"github.com/anyproto/any-sync/net/dialer"
+	"github.com/anyproto/any-sync/net/peer"
 	"github.com/prometheus/client_golang/prometheus"
+	"go.uber.org/zap"
 	"time"
 )
 
@@ -23,46 +24,54 @@ func New() Service {
 
 type Service interface {
 	Pool
-	NewPool(name string) Pool
 	app.ComponentRunnable
 }
 
+type dialer interface {
+	Dial(ctx context.Context, peerId string) (pr peer.Peer, err error)
+}
+
 type poolService struct {
 	// default pool
 	*pool
-	dialer    dialer.Dialer
+	dialer    dialer
 	metricReg *prometheus.Registry
 }
 
 func (p *poolService) Init(a *app.App) (err error) {
-	p.dialer = a.MustComponent(dialer.CName).(dialer.Dialer)
-	p.pool = &pool{dialer: p.dialer}
+	p.dialer = a.MustComponent("net.peerservice").(dialer)
+	p.pool = &pool{}
 	if m := a.Component(metric.CName); m != nil {
 		p.metricReg = m.(metric.Metric).Registry()
 	}
-	p.pool.cache = ocache.New(
+	p.pool.outgoing = ocache.New(
 		func(ctx context.Context, id string) (value ocache.Object, err error) {
 			return p.dialer.Dial(ctx, id)
 		},
 		ocache.WithLogger(log.Sugar()),
 		ocache.WithGCPeriod(time.Minute),
 		ocache.WithTTL(time.Minute*5),
-		ocache.WithPrometheus(p.metricReg, "netpool", "default"),
+		ocache.WithPrometheus(p.metricReg, "netpool", "outgoing"),
+	)
+	p.pool.incoming = ocache.New(
+		func(ctx context.Context, id string) (value ocache.Object, err error) {
+			return nil, ocache.ErrNotExists
+		},
+		ocache.WithLogger(log.Sugar()),
+		ocache.WithGCPeriod(time.Minute),
+		ocache.WithTTL(time.Minute*5),
+		ocache.WithPrometheus(p.metricReg, "netpool", "incoming"),
 	)
 	return nil
 }
 
-func (p *poolService) NewPool(name string) Pool {
-	return &pool{
-		dialer: p.dialer,
-		cache: ocache.New(
-			func(ctx context.Context, id string) (value ocache.Object, err error) {
-				return p.dialer.Dial(ctx, id)
-			},
-			ocache.WithLogger(log.Sugar()),
-			ocache.WithGCPeriod(time.Minute),
-			ocache.WithTTL(time.Minute*5),
-			ocache.WithPrometheus(p.metricReg, "netpool", name),
-		),
-	}
+func (p *pool) Run(ctx context.Context) (err error) {
+	return nil
+}
+
+func (p *pool) Close(ctx context.Context) (err error) {
+	if e := p.incoming.Close(); e != nil {
+		log.Warn("close incoming cache error", zap.Error(e))
+	}
+	return p.outgoing.Close()
 }

net/rpc/rpctest/peer.go

@@ -0,0 +1,30 @@
+package rpctest
+
+import (
+	"context"
+	"github.com/anyproto/any-sync/net/connutil"
+	"github.com/anyproto/any-sync/net/peer"
+	"github.com/anyproto/any-sync/net/transport"
+	yamux2 "github.com/anyproto/any-sync/net/transport/yamux"
+	"github.com/hashicorp/yamux"
+	"net"
+)
+
+func MultiConnPair(peerIdServ, peerIdClient string) (serv, client transport.MultiConn) {
+	sc, cc := net.Pipe()
+	var servConn = make(chan transport.MultiConn, 1)
+	go func() {
+		sess, err := yamux.Server(sc, yamux.DefaultConfig())
+		if err != nil {
+			panic(err)
+		}
+		servConn <- yamux2.NewMultiConn(peer.CtxWithPeerId(context.Background(), peerIdServ), connutil.NewLastUsageConn(sc), "", sess)
+	}()
+	sess, err := yamux.Client(cc, yamux.DefaultConfig())
+	if err != nil {
+		panic(err)
+	}
+	client = yamux2.NewMultiConn(peer.CtxWithPeerId(context.Background(), peerIdClient), connutil.NewLastUsageConn(cc), "", sess)
+	serv = <-servConn
+	return
+}

net/rpc/rpctest/pool.go

@@ -1,122 +0,0 @@
-package rpctest
-
-import (
-	"context"
-	"errors"
-	"github.com/anyproto/any-sync/app"
-	"github.com/anyproto/any-sync/net/peer"
-	"github.com/anyproto/any-sync/net/pool"
-	"math/rand"
-	"storj.io/drpc"
-	"sync"
-	"time"
-)
-
-var ErrCantConnect = errors.New("can't connect to test server")
-
-func NewTestPool() *TestPool {
-	return &TestPool{
-		peers: map[string]peer.Peer{},
-	}
-}
-
-type TestPool struct {
-	ts    *TesServer
-	peers map[string]peer.Peer
-	mu    sync.Mutex
-}
-
-func (t *TestPool) WithServer(ts *TesServer) *TestPool {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	t.ts = ts
-	return t
-}
-
-func (t *TestPool) Get(ctx context.Context, id string) (peer.Peer, error) {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	if p, ok := t.peers[id]; ok {
-		return p, nil
-	}
-	if t.ts == nil {
-		return nil, ErrCantConnect
-	}
-	return &testPeer{id: id, Conn: t.ts.Dial(ctx)}, nil
-}
-
-func (t *TestPool) Dial(ctx context.Context, id string) (peer.Peer, error) {
-	return t.Get(ctx, id)
-}
-
-func (t *TestPool) GetOneOf(ctx context.Context, peerIds []string) (peer.Peer, error) {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	for _, peerId := range peerIds {
-		if p, ok := t.peers[peerId]; ok {
-			return p, nil
-		}
-	}
-	if t.ts == nil {
-		return nil, ErrCantConnect
-	}
-	return &testPeer{id: peerIds[rand.Intn(len(peerIds))], Conn: t.ts.Dial(ctx)}, nil
-}
-
-func (t *TestPool) DialOneOf(ctx context.Context, peerIds []string) (peer.Peer, error) {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	if t.ts == nil {
-		return nil, ErrCantConnect
-	}
-	return &testPeer{id: peerIds[rand.Intn(len(peerIds))], Conn: t.ts.Dial(ctx)}, nil
-}
-
-func (t *TestPool) NewPool(name string) pool.Pool {
-	return t
-}
-
-func (t *TestPool) AddPeer(p peer.Peer) {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	t.peers[p.Id()] = p
-}
-
-func (t *TestPool) Init(a *app.App) (err error) {
-	return nil
-}
-
-func (t *TestPool) Name() (name string) {
-	return pool.CName
-}
-
-func (t *TestPool) Run(ctx context.Context) (err error) {
-	return nil
-}
-
-func (t *TestPool) Close(ctx context.Context) (err error) {
-	return nil
-}
-
-type testPeer struct {
-	id string
-	drpc.Conn
-}
-
-func (t testPeer) Addr() string {
-	return ""
-}
-
-func (t testPeer) TryClose(objectTTL time.Duration) (res bool, err error) {
-	return true, t.Close()
-}
-
-func (t testPeer) Id() string {
-	return t.id
-}
-
-func (t testPeer) LastUsage() time.Time {
-	return time.Now()
-}
-
-func (t testPeer) UpdateLastUsage() {}

net/rpc/rpctest/server.go

@@ -5,43 +5,39 @@ import (
 	"github.com/anyproto/any-sync/app"
 	"github.com/anyproto/any-sync/net/rpc/server"
 	"net"
-	"storj.io/drpc"
-	"storj.io/drpc/drpcconn"
 	"storj.io/drpc/drpcmux"
 	"storj.io/drpc/drpcserver"
 )
 
-func NewTestServer() *TesServer {
-	ts := &TesServer{
+func NewTestServer() *TestServer {
+	ts := &TestServer{
 		Mux: drpcmux.New(),
 	}
 	ts.Server = drpcserver.New(ts.Mux)
 	return ts
 }
 
-type TesServer struct {
+type TestServer struct {
 	*drpcmux.Mux
 	*drpcserver.Server
 }
 
-func (ts *TesServer) Init(a *app.App) (err error) {
+func (ts *TestServer) Init(a *app.App) (err error) {
 	return nil
 }
 
-func (ts *TesServer) Name() (name string) {
+func (ts *TestServer) Name() (name string) {
 	return server.CName
 }
 
-func (ts *TesServer) Run(ctx context.Context) (err error) {
+func (ts *TestServer) Run(ctx context.Context) (err error) {
 	return nil
 }
 
-func (ts *TesServer) Close(ctx context.Context) (err error) {
+func (ts *TestServer) Close(ctx context.Context) (err error) {
 	return nil
 }
 
-func (ts *TesServer) Dial(ctx context.Context) drpc.Conn {
-	sc, cc := net.Pipe()
-	go ts.Server.ServeOne(ctx, sc)
-	return drpcconn.New(cc)
+func (s *TestServer) ServeConn(ctx context.Context, conn net.Conn) (err error) {
+	return s.Server.ServeOne(ctx, conn)
 }

net/rpc/server/baseserver.go

@@ -1,134 +0,0 @@
-package server
-
-import (
-	"context"
-	"github.com/anyproto/any-sync/net/peer"
-	"github.com/anyproto/any-sync/net/secureservice"
-	"github.com/libp2p/go-libp2p/core/sec"
-	"github.com/zeebo/errs"
-	"go.uber.org/zap"
-	"io"
-	"net"
-	"storj.io/drpc"
-	"storj.io/drpc/drpcmanager"
-	"storj.io/drpc/drpcmux"
-	"storj.io/drpc/drpcserver"
-	"storj.io/drpc/drpcwire"
-	"time"
-)
-
-type BaseDrpcServer struct {
-	drpcServer *drpcserver.Server
-	transport  secureservice.SecureService
-	listeners  []net.Listener
-	handshake  func(conn net.Conn) (cCtx context.Context, sc sec.SecureConn, err error)
-	cancel     func()
-	*drpcmux.Mux
-}
-
-type DRPCHandlerWrapper func(handler drpc.Handler) drpc.Handler
-
-type Params struct {
-	BufferSizeMb  int
-	ListenAddrs   []string
-	Wrapper       DRPCHandlerWrapper
-	TimeoutMillis int
-	Handshake     func(conn net.Conn) (cCtx context.Context, sc sec.SecureConn, err error)
-}
-
-func NewBaseDrpcServer() *BaseDrpcServer {
-	return &BaseDrpcServer{Mux: drpcmux.New()}
-}
-
-func (s *BaseDrpcServer) Run(ctx context.Context, params Params) (err error) {
-	s.drpcServer = drpcserver.NewWithOptions(params.Wrapper(s.Mux), drpcserver.Options{Manager: drpcmanager.Options{
-		Reader: drpcwire.ReaderOptions{MaximumBufferSize: params.BufferSizeMb * (1 << 20)},
-	}})
-	s.handshake = params.Handshake
-	ctx, s.cancel = context.WithCancel(ctx)
-	for _, addr := range params.ListenAddrs {
-		list, err := net.Listen("tcp", addr)
-		if err != nil {
-			return err
-		}
-		s.listeners = append(s.listeners, list)
-		go s.serve(ctx, list)
-	}
-	return
-}
-
-func (s *BaseDrpcServer) serve(ctx context.Context, lis net.Listener) {
-	l := log.With(zap.String("localAddr", lis.Addr().String()))
-	l.Info("drpc listener started")
-	defer func() {
-		l.Debug("drpc listener stopped")
-	}()
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		default:
-		}
-		conn, err := lis.Accept()
-		if err != nil {
-			if isTemporary(err) {
-				l.Debug("listener temporary accept error", zap.Error(err))
-				select {
-				case <-time.After(time.Second):
-				case <-ctx.Done():
-					return
-				}
-				continue
-			}
-			l.Error("listener accept error", zap.Error(err))
-			return
-		}
-		go s.serveConn(conn)
-	}
-}
-
-func (s *BaseDrpcServer) serveConn(conn net.Conn) {
-	l := log.With(zap.String("remoteAddr", conn.RemoteAddr().String())).With(zap.String("localAddr", conn.LocalAddr().String()))
-	var (
-		ctx = context.Background()
-		err error
-	)
-	if s.handshake != nil {
-		ctx, conn, err = s.handshake(conn)
-		if err != nil {
-			l.Info("handshake error", zap.Error(err))
-			return
-		}
-		if sc, ok := conn.(sec.SecureConn); ok {
-			ctx = peer.CtxWithPeerId(ctx, sc.RemotePeer().String())
-		}
-	}
-	ctx = peer.CtxWithPeerAddr(ctx, conn.RemoteAddr().String())
-	l.Debug("connection opened")
-	if err := s.drpcServer.ServeOne(ctx, conn); err != nil {
-		if errs.Is(err, context.Canceled) || errs.Is(err, io.EOF) {
-			l.Debug("connection closed")
-		} else {
-			l.Warn("serve connection error", zap.Error(err))
-		}
-	}
-}
-
-func (s *BaseDrpcServer) ListenAddrs() (addrs []net.Addr) {
-	for _, list := range s.listeners {
-		addrs = append(addrs, list.Addr())
-	}
-	return
-}
-
-func (s *BaseDrpcServer) Close(ctx context.Context) (err error) {
-	if s.cancel != nil {
-		s.cancel()
-	}
-	for _, l := range s.listeners {
-		if e := l.Close(); e != nil {
-			log.Warn("close listener error", zap.Error(e))
-		}
-	}
-	return
-}

net/rpc/server/drpcserver.go

@@ -6,11 +6,13 @@ import (
 	"github.com/anyproto/any-sync/app/logger"
 	"github.com/anyproto/any-sync/metric"
 	anyNet "github.com/anyproto/any-sync/net"
-	"github.com/anyproto/any-sync/net/secureservice"
-	"github.com/libp2p/go-libp2p/core/sec"
+	"go.uber.org/zap"
 	"net"
 	"storj.io/drpc"
-	"time"
+	"storj.io/drpc/drpcmanager"
+	"storj.io/drpc/drpcmux"
+	"storj.io/drpc/drpcserver"
+	"storj.io/drpc/drpcwire"
 )
 
 const CName = "common.net.drpcserver"
@@ -18,49 +20,46 @@ const CName = "common.net.drpcserver"
 var log = logger.NewNamed(CName)
 
 func New() DRPCServer {
-	return &drpcServer{BaseDrpcServer: NewBaseDrpcServer()}
+	return &drpcServer{}
 }
 
 type DRPCServer interface {
-	app.ComponentRunnable
+	ServeConn(ctx context.Context, conn net.Conn) (err error)
+	app.Component
 	drpc.Mux
 }
 
 type drpcServer struct {
-	config    anyNet.Config
-	metric    metric.Metric
-	transport secureservice.SecureService
-	*BaseDrpcServer
+	drpcServer *drpcserver.Server
+	*drpcmux.Mux
+	config anyNet.Config
+	metric metric.Metric
 }
 
-func (s *drpcServer) Init(a *app.App) (err error) {
-	s.config = a.MustComponent("config").(anyNet.ConfigGetter).GetNet()
-	s.metric = a.MustComponent(metric.CName).(metric.Metric)
-	s.transport = a.MustComponent(secureservice.CName).(secureservice.SecureService)
-	return nil
-}
+type DRPCHandlerWrapper func(handler drpc.Handler) drpc.Handler
 
 func (s *drpcServer) Name() (name string) {
 	return CName
 }
 
-func (s *drpcServer) Run(ctx context.Context) (err error) {
-	params := Params{
-		BufferSizeMb:  s.config.Stream.MaxMsgSizeMb,
-		TimeoutMillis: s.config.Stream.TimeoutMilliseconds,
-		ListenAddrs:   s.config.Server.ListenAddrs,
-		Wrapper: func(handler drpc.Handler) drpc.Handler {
-			return s.metric.WrapDRPCHandler(handler)
-		},
-		Handshake: func(conn net.Conn) (cCtx context.Context, sc sec.SecureConn, err error) {
-			ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
-			defer cancel()
-			return s.transport.SecureInbound(ctx, conn)
-		},
+func (s *drpcServer) Init(a *app.App) (err error) {
+	s.config = a.MustComponent("config").(anyNet.ConfigGetter).GetNet()
+	s.metric, _ = a.Component(metric.CName).(metric.Metric)
+	s.Mux = drpcmux.New()
+
+	var handler drpc.Handler
+	handler = s
+	if s.metric != nil {
+		handler = s.metric.WrapDRPCHandler(s)
 	}
-	return s.BaseDrpcServer.Run(ctx, params)
+	s.drpcServer = drpcserver.NewWithOptions(handler, drpcserver.Options{Manager: drpcmanager.Options{
+		Reader: drpcwire.ReaderOptions{MaximumBufferSize: s.config.Stream.MaxMsgSizeMb * (1 << 20)},
+	}})
+	return
 }
 
-func (s *drpcServer) Close(ctx context.Context) (err error) {
-	return s.BaseDrpcServer.Close(ctx)
+func (s *drpcServer) ServeConn(ctx context.Context, conn net.Conn) (err error) {
+	l := log.With(zap.String("remoteAddr", conn.RemoteAddr().String())).With(zap.String("localAddr", conn.LocalAddr().String()))
+	l.Debug("drpc serve peer")
+	return s.drpcServer.ServeOne(ctx, conn)
 }

net/secureservice/credential.go

@@ -5,7 +5,6 @@ import (
 	"github.com/anyproto/any-sync/net/secureservice/handshake"
 	"github.com/anyproto/any-sync/net/secureservice/handshake/handshakeproto"
 	"github.com/anyproto/any-sync/util/crypto"
-	"github.com/libp2p/go-libp2p/core/sec"
 	"go.uber.org/zap"
 )
 
@@ -19,11 +18,11 @@ type noVerifyChecker struct {
 	cred *handshakeproto.Credentials
 }
 
-func (n noVerifyChecker) MakeCredentials(sc sec.SecureConn) *handshakeproto.Credentials {
+func (n noVerifyChecker) MakeCredentials(remotePeerId string) *handshakeproto.Credentials {
 	return n.cred
 }
 
-func (n noVerifyChecker) CheckCredential(sc sec.SecureConn, cred *handshakeproto.Credentials) (identity []byte, err error) {
+func (n noVerifyChecker) CheckCredential(remotePeerId string, cred *handshakeproto.Credentials) (identity []byte, err error) {
 	if cred.Version != n.cred.Version {
 		return nil, handshake.ErrIncompatibleVersion
 	}
@@ -42,8 +41,8 @@ type peerSignVerifier struct {
 	account      *accountdata.AccountKeys
 }
 
-func (p *peerSignVerifier) MakeCredentials(sc sec.SecureConn) *handshakeproto.Credentials {
-	sign, err := p.account.SignKey.Sign([]byte(p.account.PeerId + sc.RemotePeer().String()))
+func (p *peerSignVerifier) MakeCredentials(remotePeerId string) *handshakeproto.Credentials {
+	sign, err := p.account.SignKey.Sign([]byte(p.account.PeerId + remotePeerId))
 	if err != nil {
 		log.Warn("can't sign identity credentials", zap.Error(err))
 	}
@@ -61,7 +60,7 @@ func (p *peerSignVerifier) MakeCredentials(sc sec.SecureConn) *handshakeproto.Cr
 	}
 }
 
-func (p *peerSignVerifier) CheckCredential(sc sec.SecureConn, cred *handshakeproto.Credentials) (identity []byte, err error) {
+func (p *peerSignVerifier) CheckCredential(remotePeerId string, cred *handshakeproto.Credentials) (identity []byte, err error) {
 	if cred.Version != p.protoVersion {
 		return nil, handshake.ErrIncompatibleVersion
 	}
@@ -76,7 +75,7 @@ func (p *peerSignVerifier) CheckCredential(sc sec.SecureConn, cred *handshakepro
 	if err != nil {
 		return nil, handshake.ErrInvalidCredentials
 	}
-	ok, err := pubKey.Verify([]byte((sc.RemotePeer().String() + p.account.PeerId)), msg.Sign)
+	ok, err := pubKey.Verify([]byte((remotePeerId + p.account.PeerId)), msg.Sign)
 	if err != nil {
 		return nil, err
 	}

net/secureservice/credential_test.go

@@ -4,13 +4,8 @@ import (
 	"github.com/anyproto/any-sync/commonspace/object/accountdata"
 	"github.com/anyproto/any-sync/net/secureservice/handshake"
 	"github.com/anyproto/any-sync/testutil/accounttest"
-	"github.com/libp2p/go-libp2p/core/crypto"
-	"github.com/libp2p/go-libp2p/core/network"
-	"github.com/libp2p/go-libp2p/core/peer"
-	"github.com/libp2p/go-libp2p/core/sec"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"net"
 	"testing"
 )
 
@@ -23,8 +18,8 @@ func TestPeerSignVerifier_CheckCredential(t *testing.T) {
 	cc1 := newPeerSignVerifier(0, a1)
 	cc2 := newPeerSignVerifier(0, a2)
 
-	c1 := newTestSC(a2.PeerId)
-	c2 := newTestSC(a1.PeerId)
+	c1 := a2.PeerId
+	c2 := a1.PeerId
 
 	cr1 := cc1.MakeCredentials(c1)
 	cr2 := cc2.MakeCredentials(c2)
@@ -48,8 +43,8 @@ func TestIncompatibleVersion(t *testing.T) {
 	cc1 := newPeerSignVerifier(0, a1)
 	cc2 := newPeerSignVerifier(1, a2)
 
-	c1 := newTestSC(a2.PeerId)
-	c2 := newTestSC(a1.PeerId)
+	c1 := a2.PeerId
+	c2 := a1.PeerId
 
 	cr1 := cc1.MakeCredentials(c1)
 	cr2 := cc2.MakeCredentials(c2)
@@ -68,35 +63,3 @@ func newTestAccData(t *testing.T) *accountdata.AccountKeys {
 	require.NoError(t, as.Init(nil))
 	return as.Account()
 }
-
-func newTestSC(peerId string) sec.SecureConn {
-	pid, _ := peer.Decode(peerId)
-	return &testSc{
-		ID: pid,
-	}
-}
-
-type testSc struct {
-	net.Conn
-	peer.ID
-}
-
-func (t *testSc) LocalPeer() peer.ID {
-	return ""
-}
-
-func (t *testSc) LocalPrivateKey() crypto.PrivKey {
-	return nil
-}
-
-func (t *testSc) RemotePeer() peer.ID {
-	return t.ID
-}
-
-func (t *testSc) RemotePublicKey() crypto.PubKey {
-	return nil
-}
-
-func (t *testSc) ConnState() network.ConnectionState {
-	return network.ConnectionState{}
-}

net/secureservice/handshake/credential.go

@@ -0,0 +1,133 @@
+package handshake
+
+import (
+	"context"
+	"github.com/anyproto/any-sync/net/secureservice/handshake/handshakeproto"
+	"io"
+)
+
+func OutgoingHandshake(ctx context.Context, conn io.ReadWriteCloser, peerId string, cc CredentialChecker) (identity []byte, err error) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	h := newHandshake()
+	done := make(chan struct{})
+	var (
+		resIdentity []byte
+		resErr      error
+	)
+	go func() {
+		defer close(done)
+		resIdentity, resErr = outgoingHandshake(h, conn, peerId, cc)
+	}()
+	select {
+	case <-done:
+		return resIdentity, resErr
+	case <-ctx.Done():
+		_ = conn.Close()
+		return nil, ctx.Err()
+	}
+}
+
+func outgoingHandshake(h *handshake, conn io.ReadWriteCloser, peerId string, cc CredentialChecker) (identity []byte, err error) {
+	defer h.release()
+	h.conn = conn
+	localCred := cc.MakeCredentials(peerId)
+	if err = h.writeCredentials(localCred); err != nil {
+		h.tryWriteErrAndClose(err)
+		return
+	}
+	msg, err := h.readMsg(msgTypeAck, msgTypeCred)
+	if err != nil {
+		h.tryWriteErrAndClose(err)
+		return
+	}
+	if msg.ack != nil {
+		if msg.ack.Error == handshakeproto.Error_InvalidCredentials {
+			return nil, ErrPeerDeclinedCredentials
+		}
+		return nil, HandshakeError{e: msg.ack.Error}
+	}
+
+	if identity, err = cc.CheckCredential(peerId, msg.cred); err != nil {
+		h.tryWriteErrAndClose(err)
+		return
+	}
+
+	if err = h.writeAck(handshakeproto.Error_Null); err != nil {
+		h.tryWriteErrAndClose(err)
+		return nil, err
+	}
+
+	msg, err = h.readMsg(msgTypeAck)
+	if err != nil {
+		h.tryWriteErrAndClose(err)
+		return nil, err
+	}
+	if msg.ack.Error == handshakeproto.Error_Null {
+		return identity, nil
+	} else {
+		_ = h.conn.Close()
+		return nil, HandshakeError{e: msg.ack.Error}
+	}
+}
+
+func IncomingHandshake(ctx context.Context, conn io.ReadWriteCloser, peerId string, cc CredentialChecker) (identity []byte, err error) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	h := newHandshake()
+	done := make(chan struct{})
+	var (
+		resIdentity []byte
+		resError    error
+	)
+	go func() {
+		defer close(done)
+		resIdentity, resError = incomingHandshake(h, conn, peerId, cc)
+	}()
+	select {
+	case <-done:
+		return resIdentity, resError
+	case <-ctx.Done():
+		_ = conn.Close()
+		return nil, ctx.Err()
+	}
+}
+
+func incomingHandshake(h *handshake, conn io.ReadWriteCloser, peerId string, cc CredentialChecker) (identity []byte, err error) {
+	defer h.release()
+	h.conn = conn
+
+	msg, err := h.readMsg(msgTypeCred)
+	if err != nil {
+		h.tryWriteErrAndClose(err)
+		return
+	}
+	if identity, err = cc.CheckCredential(peerId, msg.cred); err != nil {
+		h.tryWriteErrAndClose(err)
+		return
+	}
+
+	if err = h.writeCredentials(cc.MakeCredentials(peerId)); err != nil {
+		h.tryWriteErrAndClose(err)
+		return nil, err
+	}
+
+	msg, err = h.readMsg(msgTypeAck)
+	if err != nil {
+		h.tryWriteErrAndClose(err)
+		return nil, err
+	}
+	if msg.ack.Error != handshakeproto.Error_Null {
+		if msg.ack.Error == handshakeproto.Error_InvalidCredentials {
+			return nil, ErrPeerDeclinedCredentials
+		}
+		return nil, HandshakeError{e: msg.ack.Error}
+	}
+	if err = h.writeAck(handshakeproto.Error_Null); err != nil {
+		h.tryWriteErrAndClose(err)
+		return nil, err
+	}
+	return
+}

net/secureservice/handshake/credential_test.go

@@ -7,7 +7,6 @@ import (
 	"github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
-	"github.com/libp2p/go-libp2p/core/sec"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"net"
@@ -17,7 +16,7 @@ import (
 
 var noVerifyChecker = &testCredChecker{
 	makeCred: &handshakeproto.Credentials{Type: handshakeproto.CredentialsType_SkipVerify},
-	checkCred: func(sc sec.SecureConn, cred *handshakeproto.Credentials) (identity []byte, err error) {
+	checkCred: func(peerId string, cred *handshakeproto.Credentials) (identity []byte, err error) {
 		return []byte("identity"), nil
 	},
 }
@@ -32,21 +31,20 @@ func TestOutgoingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := OutgoingHandshake(nil, c1, noVerifyChecker)
+			identity, err := OutgoingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// receive credential message
-		msg, err := h.readMsg()
+		msg, err := h.readMsg(msgTypeCred, msgTypeAck, msgTypeProto)
 		require.NoError(t, err)
-		require.Nil(t, msg.ack)
-		_, err = noVerifyChecker.CheckCredential(c2, msg.cred)
+		_, err = noVerifyChecker.CheckCredential("p1", msg.cred)
 		require.NoError(t, err)
 		// send credential message
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		// receive ack
-		msg, err = h.readMsg()
+		msg, err = h.readMsg(msgTypeAck)
 		require.NoError(t, err)
 		require.Equal(t, handshakeproto.Error_Null, msg.ack.Error)
 		// send ack
@@ -59,7 +57,7 @@ func TestOutgoingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := OutgoingHandshake(nil, c1, noVerifyChecker)
+			identity, err := OutgoingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		_ = c2.Close()
@@ -70,13 +68,13 @@ func TestOutgoingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := OutgoingHandshake(nil, c1, noVerifyChecker)
+			identity, err := OutgoingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// receive credential message
-		_, err := h.readMsg()
+		_, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
 		_ = c2.Close()
 		res := <-handshakeResCh
@@ -86,13 +84,13 @@ func TestOutgoingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := OutgoingHandshake(nil, c1, noVerifyChecker)
+			identity, err := OutgoingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// receive credential message
-		_, err := h.readMsg()
+		_, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
 		require.NoError(t, h.writeAck(ErrInvalidCredentials.e))
 		res := <-handshakeResCh
@@ -102,16 +100,16 @@ func TestOutgoingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := OutgoingHandshake(nil, c1, &testCredChecker{makeCred: noVerifyChecker.makeCred, checkErr: ErrInvalidCredentials})
+			identity, err := OutgoingHandshake(nil, c1, "", &testCredChecker{makeCred: noVerifyChecker.makeCred, checkErr: ErrInvalidCredentials})
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// receive credential message
-		_, err := h.readMsg()
+		_, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
-		msg, err := h.readMsg()
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
+		msg, err := h.readMsg(msgTypeAck)
 		require.NoError(t, err)
 		assert.Equal(t, ErrInvalidCredentials.e, msg.ack.Error)
 		res := <-handshakeResCh
@@ -121,16 +119,16 @@ func TestOutgoingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := OutgoingHandshake(nil, c1, noVerifyChecker)
+			identity, err := OutgoingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// receive credential message
-		_, err := h.readMsg()
+		_, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
 		// write credentials and close conn
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		_ = c2.Close()
 		res := <-handshakeResCh
 		require.Error(t, res.err)
@@ -139,18 +137,18 @@ func TestOutgoingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := OutgoingHandshake(nil, c1, noVerifyChecker)
+			identity, err := OutgoingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// receive credential message
-		_, err := h.readMsg()
+		_, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
 		// write credentials
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		// read ack and close conn
-		_, err = h.readMsg()
+		_, err = h.readMsg(msgTypeAck)
 		require.NoError(t, err)
 		_ = c2.Close()
 		res := <-handshakeResCh
@@ -160,24 +158,23 @@ func TestOutgoingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := OutgoingHandshake(nil, c1, noVerifyChecker)
+			identity, err := OutgoingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// receive credential message
-		_, err := h.readMsg()
+		_, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
 		// write credentials
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		// read ack
-		_, err = h.readMsg()
+		_, err = h.readMsg(msgTypeAck)
 		require.NoError(t, err)
 		// write cred instead ack
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
-		msg, err := h.readMsg()
-		require.NoError(t, err)
-		assert.Equal(t, handshakeproto.Error_UnexpectedPayload, msg.ack.Error)
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
+		_, err = h.readMsg(msgTypeAck)
+		require.Error(t, err)
 		res := <-handshakeResCh
 		require.Error(t, res.err)
 	})
@@ -185,21 +182,21 @@ func TestOutgoingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := OutgoingHandshake(nil, c1, noVerifyChecker)
+			identity, err := OutgoingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// receive credential message
-		msg, err := h.readMsg()
+		msg, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
 		require.Nil(t, msg.ack)
-		_, err = noVerifyChecker.CheckCredential(c2, msg.cred)
+		_, err = noVerifyChecker.CheckCredential("", msg.cred)
 		require.NoError(t, err)
 		// send credential message
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		// receive ack
-		msg, err = h.readMsg()
+		msg, err = h.readMsg(msgTypeAck)
 		require.NoError(t, err)
 		require.Equal(t, handshakeproto.Error_Null, msg.ack.Error)
 		// send ack
@@ -213,13 +210,13 @@ func TestOutgoingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := OutgoingHandshake(ctx, c1, noVerifyChecker)
+			identity, err := OutgoingHandshake(ctx, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// receive credential message
-		_, err := h.readMsg()
+		_, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
 		ctxCancel()
 		res := <-handshakeResCh
@@ -236,22 +233,22 @@ func TestIncomingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := IncomingHandshake(nil, c1, noVerifyChecker)
+			identity, err := IncomingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// write credentials
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		// wait credentials
-		msg, err := h.readMsg()
+		msg, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
 		require.Nil(t, msg.ack)
 		require.Equal(t, handshakeproto.CredentialsType_SkipVerify, msg.cred.Type)
 		// write ack
 		require.NoError(t, h.writeAck(handshakeproto.Error_Null))
 		// wait ack
-		msg, err = h.readMsg()
+		msg, err = h.readMsg(msgTypeAck)
 		require.NoError(t, err)
 		assert.Equal(t, handshakeproto.Error_Null, msg.ack.Error)
 		res := <-handshakeResCh
@@ -262,7 +259,7 @@ func TestIncomingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := IncomingHandshake(nil, c1, noVerifyChecker)
+			identity, err := IncomingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		_ = c2.Close()
@@ -273,13 +270,13 @@ func TestIncomingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := IncomingHandshake(nil, c1, noVerifyChecker)
+			identity, err := IncomingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// write credentials and close conn
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		_ = c2.Close()
 		res := <-handshakeResCh
 		require.Error(t, res.err)
@@ -288,7 +285,7 @@ func TestIncomingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := IncomingHandshake(nil, c1, noVerifyChecker)
+			identity, err := IncomingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
@@ -302,15 +299,15 @@ func TestIncomingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := IncomingHandshake(nil, c1, &testCredChecker{makeCred: noVerifyChecker.makeCred, checkErr: ErrInvalidCredentials})
+			identity, err := IncomingHandshake(nil, c1, "", &testCredChecker{makeCred: noVerifyChecker.makeCred, checkErr: ErrInvalidCredentials})
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// write credentials
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		// except ack with error
-		msg, err := h.readMsg()
+		msg, err := h.readMsg(msgTypeAck)
 		require.NoError(t, err)
 		require.Nil(t, msg.cred)
 		require.Equal(t, handshakeproto.Error_InvalidCredentials, msg.ack.Error)
@@ -322,15 +319,15 @@ func TestIncomingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := IncomingHandshake(nil, c1, &testCredChecker{makeCred: noVerifyChecker.makeCred, checkErr: ErrIncompatibleVersion})
+			identity, err := IncomingHandshake(nil, c1, "", &testCredChecker{makeCred: noVerifyChecker.makeCred, checkErr: ErrIncompatibleVersion})
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// write credentials
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		// except ack with error
-		msg, err := h.readMsg()
+		msg, err := h.readMsg(msgTypeAck)
 		require.NoError(t, err)
 		require.Nil(t, msg.cred)
 		require.Equal(t, handshakeproto.Error_IncompatibleVersion, msg.ack.Error)
@@ -342,21 +339,21 @@ func TestIncomingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := IncomingHandshake(nil, c1, noVerifyChecker)
+			identity, err := IncomingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// write credentials
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		// read cred
-		_, err := h.readMsg()
+		_, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
 		// write cred instead ack
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
-		// expect ack with error
-		msg, err := h.readMsg()
-		require.Equal(t, handshakeproto.Error_UnexpectedPayload, msg.ack.Error)
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
+		// expect EOF
+		_, err = h.readMsg(msgTypeAck)
+		require.Error(t, err)
 		res := <-handshakeResCh
 		require.Error(t, res.err)
 	})
@@ -364,15 +361,15 @@ func TestIncomingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := IncomingHandshake(nil, c1, noVerifyChecker)
+			identity, err := IncomingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// write credentials
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		// read cred and close conn
-		_, err := h.readMsg()
+		_, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
 		_ = c2.Close()
 
@@ -383,15 +380,15 @@ func TestIncomingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := IncomingHandshake(nil, c1, noVerifyChecker)
+			identity, err := IncomingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// write credentials
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		// wait credentials
-		msg, err := h.readMsg()
+		msg, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
 		require.Nil(t, msg.ack)
 		require.Equal(t, handshakeproto.CredentialsType_SkipVerify, msg.cred.Type)
@@ -405,15 +402,15 @@ func TestIncomingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := IncomingHandshake(nil, c1, noVerifyChecker)
+			identity, err := IncomingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// write credentials
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		// wait credentials
-		msg, err := h.readMsg()
+		msg, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
 		require.Nil(t, msg.ack)
 		require.Equal(t, handshakeproto.CredentialsType_SkipVerify, msg.cred.Type)
@@ -427,15 +424,15 @@ func TestIncomingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := IncomingHandshake(nil, c1, noVerifyChecker)
+			identity, err := IncomingHandshake(nil, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// write credentials
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		// wait credentials
-		msg, err := h.readMsg()
+		msg, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
 		require.Nil(t, msg.ack)
 		require.Equal(t, handshakeproto.CredentialsType_SkipVerify, msg.cred.Type)
@@ -450,15 +447,15 @@ func TestIncomingHandshake(t *testing.T) {
 		c1, c2 := newConnPair(t)
 		var handshakeResCh = make(chan handshakeRes, 1)
 		go func() {
-			identity, err := IncomingHandshake(ctx, c1, noVerifyChecker)
+			identity, err := IncomingHandshake(ctx, c1, "", noVerifyChecker)
 			handshakeResCh <- handshakeRes{identity: identity, err: err}
 		}()
 		h := newHandshake()
 		h.conn = c2
 		// write credentials
-		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials(c2)))
+		require.NoError(t, h.writeCredentials(noVerifyChecker.MakeCredentials("")))
 		// wait credentials
-		_, err := h.readMsg()
+		_, err := h.readMsg(msgTypeCred)
 		require.NoError(t, err)
 		ctxCancel()
 		res := <-handshakeResCh
@@ -474,7 +471,7 @@ func TestNotAHandshakeMessage(t *testing.T) {
 	c1, c2 := newConnPair(t)
 	var handshakeResCh = make(chan handshakeRes, 1)
 	go func() {
-		identity, err := IncomingHandshake(nil, c1, noVerifyChecker)
+		identity, err := IncomingHandshake(nil, c1, "", noVerifyChecker)
 		handshakeResCh <- handshakeRes{identity: identity, err: err}
 	}()
 	h := newHandshake()
@@ -482,7 +479,7 @@ func TestNotAHandshakeMessage(t *testing.T) {
 	_, err := c2.Write([]byte("some unexpected bytes"))
 	require.Error(t, err)
 	res := <-handshakeResCh
-	assert.EqualError(t, res.err, ErrGotNotAHandshakeMessage.Error())
+	assert.Error(t, res.err)
 }
 
 func TestEndToEnd(t *testing.T) {
@@ -493,11 +490,11 @@ func TestEndToEnd(t *testing.T) {
 	)
 	st := time.Now()
 	go func() {
-		identity, err := OutgoingHandshake(nil, c1, noVerifyChecker)
+		identity, err := OutgoingHandshake(nil, c1, "", noVerifyChecker)
 		outResCh <- handshakeRes{identity: identity, err: err}
 	}()
 	go func() {
-		identity, err := IncomingHandshake(nil, c2, noVerifyChecker)
+		identity, err := IncomingHandshake(nil, c2, "", noVerifyChecker)
 		inResCh <- handshakeRes{identity: identity, err: err}
 	}()
 
@@ -521,7 +518,7 @@ func BenchmarkHandshake(b *testing.B) {
 	defer close(done)
 	go func() {
 		for {
-			_, _ = OutgoingHandshake(nil, c1, noVerifyChecker)
+			_, _ = OutgoingHandshake(nil, c1, "", noVerifyChecker)
 			select {
 			case outRes <- struct{}{}:
 			case <-done:
@@ -531,7 +528,7 @@ func BenchmarkHandshake(b *testing.B) {
 	}()
 	go func() {
 		for {
-			_, _ = IncomingHandshake(nil, c2, noVerifyChecker)
+			_, _ = IncomingHandshake(nil, c2, "", noVerifyChecker)
 			select {
 			case inRes <- struct{}{}:
 			case <-done:
@@ -551,20 +548,20 @@ func BenchmarkHandshake(b *testing.B) {
 
 type testCredChecker struct {
 	makeCred  *handshakeproto.Credentials
-	checkCred func(sc sec.SecureConn, cred *handshakeproto.Credentials) (identity []byte, err error)
+	checkCred func(peerId string, cred *handshakeproto.Credentials) (identity []byte, err error)
 	checkErr  error
 }
 
-func (t *testCredChecker) MakeCredentials(sc sec.SecureConn) *handshakeproto.Credentials {
+func (t *testCredChecker) MakeCredentials(peerId string) *handshakeproto.Credentials {
 	return t.makeCred
 }
 
-func (t *testCredChecker) CheckCredential(sc sec.SecureConn, cred *handshakeproto.Credentials) (identity []byte, err error) {
+func (t *testCredChecker) CheckCredential(peerId string, cred *handshakeproto.Credentials) (identity []byte, err error) {
 	if t.checkErr != nil {
 		return nil, t.checkErr
 	}
 	if t.checkCred != nil {
-		return t.checkCred(sc, cred)
+		return t.checkCred(peerId, cred)
 	}
 	return nil, nil
 }

net/secureservice/handshake/handshake.go

@@ -1,11 +1,9 @@
 package handshake
 
 import (
-	"context"
 	"encoding/binary"
 	"errors"
 	"github.com/anyproto/any-sync/net/secureservice/handshake/handshakeproto"
-	"github.com/libp2p/go-libp2p/core/sec"
 	"golang.org/x/exp/slices"
 	"io"
 	"sync"
@@ -14,8 +12,17 @@ import (
 const headerSize = 5 // 1 byte for type + 4 byte for uint32 size
 
 const (
-	msgTypeCred = byte(1)
-	msgTypeAck  = byte(2)
+	msgTypeCred  = byte(1)
+	msgTypeAck   = byte(2)
+	msgTypeProto = byte(3)
+
+	sizeLimit = 200 * 1024 // 200 Kb
+)
+
+var (
+	credMsgTypes     = []byte{msgTypeCred, msgTypeAck}
+	protoMsgTypes    = []byte{msgTypeProto, msgTypeAck}
+	protoMsgTypesAck = []byte{msgTypeAck}
 )
 
 type HandshakeError struct {
@@ -38,154 +45,26 @@ var (
 	ErrSkipVerifyNotAllowed    = HandshakeError{e: handshakeproto.Error_SkipVerifyNotAllowed}
 	ErrUnexpected              = HandshakeError{e: handshakeproto.Error_Unexpected}
 
-	ErrIncompatibleVersion = HandshakeError{e: handshakeproto.Error_IncompatibleVersion}
+	ErrIncompatibleVersion     = HandshakeError{e: handshakeproto.Error_IncompatibleVersion}
+	ErrIncompatibleProto       = HandshakeError{e: handshakeproto.Error_IncompatibleProto}
+	ErrRemoteIncompatibleProto = HandshakeError{Err: errors.New("remote peer declined the proto")}
 
-	ErrGotNotAHandshakeMessage = errors.New("go not a handshake message")
+	ErrGotUnexpectedMessage = errors.New("go not a handshake message")
 )
 
 var handshakePool = &sync.Pool{New: func() any {
 	return &handshake{
-		remoteCred: &handshakeproto.Credentials{},
-		remoteAck:  &handshakeproto.Ack{},
-		localAck:   &handshakeproto.Ack{},
-		buf:        make([]byte, 0, 1024),
+		remoteCred:  &handshakeproto.Credentials{},
+		remoteAck:   &handshakeproto.Ack{},
+		localAck:    &handshakeproto.Ack{},
+		remoteProto: &handshakeproto.Proto{},
+		buf:         make([]byte, 0, 1024),
 	}
 }}
 
 type CredentialChecker interface {
-	MakeCredentials(sc sec.SecureConn) *handshakeproto.Credentials
-	CheckCredential(sc sec.SecureConn, cred *handshakeproto.Credentials) (identity []byte, err error)
-}
-
-func OutgoingHandshake(ctx context.Context, sc sec.SecureConn, cc CredentialChecker) (identity []byte, err error) {
-	if ctx == nil {
-		ctx = context.Background()
-	}
-	h := newHandshake()
-	done := make(chan struct{})
-	go func() {
-		defer close(done)
-		identity, err = outgoingHandshake(h, sc, cc)
-	}()
-	select {
-	case <-done:
-		return
-	case <-ctx.Done():
-		_ = sc.Close()
-		return nil, ctx.Err()
-	}
-}
-
-func outgoingHandshake(h *handshake, sc sec.SecureConn, cc CredentialChecker) (identity []byte, err error) {
-	defer h.release()
-	h.conn = sc
-	localCred := cc.MakeCredentials(sc)
-	if err = h.writeCredentials(localCred); err != nil {
-		h.tryWriteErrAndClose(err)
-		return
-	}
-	msg, err := h.readMsg()
-	if err != nil {
-		h.tryWriteErrAndClose(err)
-		return
-	}
-	if msg.ack != nil {
-		if msg.ack.Error == handshakeproto.Error_InvalidCredentials {
-			return nil, ErrPeerDeclinedCredentials
-		}
-		return nil, HandshakeError{e: msg.ack.Error}
-	}
-
-	if identity, err = cc.CheckCredential(sc, msg.cred); err != nil {
-		h.tryWriteErrAndClose(err)
-		return
-	}
-
-	if err = h.writeAck(handshakeproto.Error_Null); err != nil {
-		h.tryWriteErrAndClose(err)
-		return nil, err
-	}
-
-	msg, err = h.readMsg()
-	if err != nil {
-		h.tryWriteErrAndClose(err)
-		return nil, err
-	}
-	if msg.ack == nil {
-		err = ErrUnexpectedPayload
-		h.tryWriteErrAndClose(err)
-		return nil, err
-	}
-	if msg.ack.Error == handshakeproto.Error_Null {
-		return identity, nil
-	} else {
-		_ = h.conn.Close()
-		return nil, HandshakeError{e: msg.ack.Error}
-	}
-}
-
-func IncomingHandshake(ctx context.Context, sc sec.SecureConn, cc CredentialChecker) (identity []byte, err error) {
-	if ctx == nil {
-		ctx = context.Background()
-	}
-	h := newHandshake()
-	done := make(chan struct{})
-	go func() {
-		defer close(done)
-		identity, err = incomingHandshake(h, sc, cc)
-	}()
-	select {
-	case <-done:
-		return
-	case <-ctx.Done():
-		_ = sc.Close()
-		return nil, ctx.Err()
-	}
-}
-
-func incomingHandshake(h *handshake, sc sec.SecureConn, cc CredentialChecker) (identity []byte, err error) {
-	defer h.release()
-	h.conn = sc
-
-	msg, err := h.readMsg()
-	if err != nil {
-		h.tryWriteErrAndClose(err)
-		return
-	}
-	if msg.ack != nil {
-		return nil, ErrUnexpectedPayload
-	}
-	if identity, err = cc.CheckCredential(sc, msg.cred); err != nil {
-		h.tryWriteErrAndClose(err)
-		return
-	}
-
-	if err = h.writeCredentials(cc.MakeCredentials(sc)); err != nil {
-		h.tryWriteErrAndClose(err)
-		return nil, err
-	}
-
-	msg, err = h.readMsg()
-	if err != nil {
-		h.tryWriteErrAndClose(err)
-		return nil, err
-	}
-	if msg.ack == nil {
-		err = ErrUnexpectedPayload
-		h.tryWriteErrAndClose(err)
-		return nil, err
-	}
-	if msg.ack.Error != handshakeproto.Error_Null {
-		if msg.ack.Error == handshakeproto.Error_InvalidCredentials {
-			return nil, ErrPeerDeclinedCredentials
-		}
-		return nil, HandshakeError{e: msg.ack.Error}
-	}
-	if err = h.writeAck(handshakeproto.Error_Null); err != nil {
-		h.tryWriteErrAndClose(err)
-		return nil, err
-	}
-	return
+	MakeCredentials(remotePeerId string) *handshakeproto.Credentials
+	CheckCredential(remotePeerId string, cred *handshakeproto.Credentials) (identity []byte, err error)
 }
 
 func newHandshake() *handshake {
@@ -193,11 +72,12 @@ func newHandshake() *handshake {
 }
 
 type handshake struct {
-	conn       sec.SecureConn
-	remoteCred *handshakeproto.Credentials
-	remoteAck  *handshakeproto.Ack
-	localAck   *handshakeproto.Ack
-	buf        []byte
+	conn        io.ReadWriteCloser
+	remoteCred  *handshakeproto.Credentials
+	remoteProto *handshakeproto.Proto
+	remoteAck   *handshakeproto.Ack
+	localAck    *handshakeproto.Ack
+	buf         []byte
 }
 
 func (h *handshake) writeCredentials(cred *handshakeproto.Credentials) (err error) {
@@ -209,8 +89,17 @@ func (h *handshake) writeCredentials(cred *handshakeproto.Credentials) (err erro
 	return h.writeData(msgTypeCred, n)
 }
 
+func (h *handshake) writeProto(proto *handshakeproto.Proto) (err error) {
+	h.buf = slices.Grow(h.buf, proto.Size()+headerSize)[:proto.Size()+headerSize]
+	n, err := proto.MarshalToSizedBuffer(h.buf[headerSize:])
+	if err != nil {
+		return err
+	}
+	return h.writeData(msgTypeProto, n)
+}
+
 func (h *handshake) tryWriteErrAndClose(err error) {
-	if err == ErrGotNotAHandshakeMessage {
+	if err == ErrUnexpectedPayload {
 		// if we got unexpected message - just close the connection
 		_ = h.conn.Close()
 		return
@@ -243,21 +132,26 @@ func (h *handshake) writeData(tp byte, size int) (err error) {
 }
 
 type message struct {
-	cred *handshakeproto.Credentials
-	ack  *handshakeproto.Ack
+	cred  *handshakeproto.Credentials
+	proto *handshakeproto.Proto
+	ack   *handshakeproto.Ack
 }
 
-func (h *handshake) readMsg() (msg message, err error) {
+func (h *handshake) readMsg(allowedTypes ...byte) (msg message, err error) {
 	h.buf = slices.Grow(h.buf, headerSize)[:headerSize]
 	if _, err = io.ReadFull(h.conn, h.buf[:headerSize]); err != nil {
 		return
 	}
 	tp := h.buf[0]
-	if tp != msgTypeCred && tp != msgTypeAck {
-		err = ErrGotNotAHandshakeMessage
+	if !slices.Contains(allowedTypes, tp) {
+		err = ErrUnexpectedPayload
 		return
 	}
 	size := binary.LittleEndian.Uint32(h.buf[1:headerSize])
+	if size > sizeLimit {
+		err = ErrGotUnexpectedMessage
+		return
+	}
 	h.buf = slices.Grow(h.buf, int(size))[:size]
 	if _, err = io.ReadFull(h.conn, h.buf[:size]); err != nil {
 		return
@@ -273,6 +167,11 @@ func (h *handshake) readMsg() (msg message, err error) {
 			return
 		}
 		msg.ack = h.remoteAck
+	case msgTypeProto:
+		if err = h.remoteProto.Unmarshal(h.buf[:size]); err != nil {
+			return
+		}
+		msg.proto = h.remoteProto
 	}
 	return
 }
@@ -284,5 +183,6 @@ func (h *handshake) release() {
 	h.remoteAck.Error = 0
 	h.remoteCred.Type = 0
 	h.remoteCred.Payload = h.remoteCred.Payload[:0]
+	h.remoteProto.Proto = 0
 	handshakePool.Put(h)
 }

net/secureservice/handshake/handshakeproto/handshake.pb.go

@@ -59,6 +59,7 @@ const (
 	Error_SkipVerifyNotAllowed Error = 4
 	Error_DeadlineExceeded     Error = 5
 	Error_IncompatibleVersion  Error = 6
+	Error_IncompatibleProto    Error = 7
 )
 
 var Error_name = map[int32]string{
@@ -69,6 +70,7 @@ var Error_name = map[int32]string{
 	4: "SkipVerifyNotAllowed",
 	5: "DeadlineExceeded",
 	6: "IncompatibleVersion",
+	7: "IncompatibleProto",
 }
 
 var Error_value = map[string]int32{
@@ -79,6 +81,7 @@ var Error_value = map[string]int32{
 	"SkipVerifyNotAllowed": 4,
 	"DeadlineExceeded":     5,
 	"IncompatibleVersion":  6,
+	"IncompatibleProto":    7,
 }
 
 func (x Error) String() string {
@@ -89,6 +92,28 @@ func (Error) EnumDescriptor() ([]byte, []int) {
 	return fileDescriptor_60283fc75f020893, []int{1}
 }
 
+type ProtoType int32
+
+const (
+	ProtoType_DRPC ProtoType = 0
+)
+
+var ProtoType_name = map[int32]string{
+	0: "DRPC",
+}
+
+var ProtoType_value = map[string]int32{
+	"DRPC": 0,
+}
+
+func (x ProtoType) String() string {
+	return proto.EnumName(ProtoType_name, int32(x))
+}
+
+func (ProtoType) EnumDescriptor() ([]byte, []int) {
+	return fileDescriptor_60283fc75f020893, []int{2}
+}
+
 type Credentials struct {
 	Type    CredentialsType `protobuf:"varint,1,opt,name=type,proto3,enum=anyHandshake.CredentialsType" json:"type,omitempty"`
 	Payload []byte          `protobuf:"bytes,2,opt,name=payload,proto3" json:"payload,omitempty"`
@@ -247,12 +272,58 @@ func (m *Ack) GetError() Error {
 	return Error_Null
 }
 
+type Proto struct {
+	Proto ProtoType `protobuf:"varint,1,opt,name=proto,proto3,enum=anyHandshake.ProtoType" json:"proto,omitempty"`
+}
+
+func (m *Proto) Reset()         { *m = Proto{} }
+func (m *Proto) String() string { return proto.CompactTextString(m) }
+func (*Proto) ProtoMessage()    {}
+func (*Proto) Descriptor() ([]byte, []int) {
+	return fileDescriptor_60283fc75f020893, []int{3}
+}
+func (m *Proto) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *Proto) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_Proto.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *Proto) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Proto.Merge(m, src)
+}
+func (m *Proto) XXX_Size() int {
+	return m.Size()
+}
+func (m *Proto) XXX_DiscardUnknown() {
+	xxx_messageInfo_Proto.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Proto proto.InternalMessageInfo
+
+func (m *Proto) GetProto() ProtoType {
+	if m != nil {
+		return m.Proto
+	}
+	return ProtoType_DRPC
+}
+
 func init() {
 	proto.RegisterEnum("anyHandshake.CredentialsType", CredentialsType_name, CredentialsType_value)
 	proto.RegisterEnum("anyHandshake.Error", Error_name, Error_value)
+	proto.RegisterEnum("anyHandshake.ProtoType", ProtoType_name, ProtoType_value)
 	proto.RegisterType((*Credentials)(nil), "anyHandshake.Credentials")
 	proto.RegisterType((*PayloadSignedPeerIds)(nil), "anyHandshake.PayloadSignedPeerIds")
 	proto.RegisterType((*Ack)(nil), "anyHandshake.Ack")
+	proto.RegisterType((*Proto)(nil), "anyHandshake.Proto")
 }
 
 func init() {
@@ -260,32 +331,35 @@ func init() {
 }
 
 var fileDescriptor_60283fc75f020893 = []byte{
-	// 395 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0xcd, 0x6e, 0x13, 0x31,
-	0x10, 0xc7, 0xd7, 0x4d, 0x52, 0xaa, 0x21, 0x2d, 0xee, 0x34, 0xc0, 0x0a, 0x89, 0x55, 0x94, 0x53,
-	0xc8, 0x21, 0xe1, 0xeb, 0x05, 0x02, 0x2d, 0x22, 0x97, 0xaa, 0xda, 0x42, 0x0f, 0xdc, 0xdc, 0xf5,
-	0xd0, 0x5a, 0x31, 0xf6, 0xca, 0x76, 0x43, 0xf7, 0x2d, 0xb8, 0xf2, 0x46, 0x1c, 0x7b, 0xe4, 0x88,
-	0x92, 0x17, 0x41, 0x71, 0x12, 0x92, 0x70, 0xea, 0xc5, 0x9e, 0x8f, 0x9f, 0xfd, 0xff, 0x8f, 0x65,
-	0x18, 0x1a, 0x0a, 0x03, 0x4f, 0xc5, 0x8d, 0x23, 0x4f, 0x6e, 0xa2, 0x0a, 0x1a, 0x5c, 0x0b, 0x23,
-	0xfd, 0xb5, 0x18, 0x6f, 0x44, 0xa5, 0xb3, 0xc1, 0x0e, 0xe2, 0xea, 0xd7, 0xd5, 0x7e, 0x2c, 0x60,
-	0x53, 0x98, 0xea, 0xe3, 0xaa, 0xd6, 0x09, 0xf0, 0xf0, 0xbd, 0x23, 0x49, 0x26, 0x28, 0xa1, 0x3d,
-	0xbe, 0x82, 0x7a, 0xa8, 0x4a, 0x4a, 0x59, 0x9b, 0x75, 0x0f, 0x5e, 0x3f, 0xef, 0x6f, 0xb2, 0xfd,
-	0x0d, 0xf0, 0x53, 0x55, 0x52, 0x1e, 0x51, 0x4c, 0xe1, 0x41, 0x29, 0x2a, 0x6d, 0x85, 0x4c, 0x77,
-	0xda, 0xac, 0xdb, 0xcc, 0x57, 0xe9, 0xbc, 0x33, 0x21, 0xe7, 0x95, 0x35, 0x69, 0xad, 0xcd, 0xba,
-	0xfb, 0xf9, 0x2a, 0xed, 0x7c, 0x80, 0xd6, 0xd9, 0x02, 0x3a, 0x57, 0x57, 0x86, 0xe4, 0x19, 0x91,
-	0x1b, 0x49, 0x8f, 0xcf, 0x60, 0x4f, 0x45, 0x89, 0x50, 0x45, 0x0b, 0xcd, 0xfc, 0x5f, 0x8e, 0x08,
-	0x75, 0xaf, 0xae, 0xcc, 0x52, 0x24, 0xc6, 0x9d, 0x97, 0x50, 0x1b, 0x16, 0x63, 0x7c, 0x01, 0x0d,
-	0x72, 0xce, 0xba, 0xa5, 0xed, 0xa3, 0x6d, 0xdb, 0x27, 0xf3, 0x56, 0xbe, 0x20, 0x7a, 0x6f, 0xe1,
-	0xd1, 0x7f, 0x63, 0xe0, 0x01, 0xc0, 0xf9, 0x58, 0x95, 0x17, 0xe4, 0xd4, 0xd7, 0x8a, 0x27, 0x78,
-	0x08, 0xfb, 0x5b, 0xae, 0x38, 0xeb, 0xfd, 0x64, 0xd0, 0x88, 0xd7, 0xe0, 0x1e, 0xd4, 0x4f, 0x6f,
-	0xb4, 0xe6, 0xc9, 0xfc, 0xd8, 0x67, 0x43, 0xb7, 0x25, 0x15, 0x81, 0x24, 0x67, 0xf8, 0x04, 0x70,
-	0x64, 0x26, 0x42, 0x2b, 0xb9, 0x21, 0xc0, 0x77, 0xf0, 0x31, 0x1c, 0xae, 0xb9, 0xe5, 0xd4, 0xbc,
-	0x86, 0x29, 0xb4, 0xd6, 0xaa, 0xa7, 0x36, 0x0c, 0xb5, 0xb6, 0xdf, 0x49, 0xf2, 0x3a, 0xb6, 0x80,
-	0x1f, 0x93, 0x90, 0x5a, 0x19, 0x3a, 0xb9, 0x2d, 0x88, 0x24, 0x49, 0xde, 0xc0, 0xa7, 0x70, 0x34,
-	0x32, 0x85, 0xfd, 0x56, 0x8a, 0xa0, 0x2e, 0x35, 0x5d, 0x2c, 0x5e, 0x92, 0xef, 0xbe, 0x3b, 0xfe,
-	0x35, 0xcd, 0xd8, 0xdd, 0x34, 0x63, 0x7f, 0xa6, 0x19, 0xfb, 0x31, 0xcb, 0x92, 0xbb, 0x59, 0x96,
-	0xfc, 0x9e, 0x65, 0xc9, 0x97, 0xde, 0xfd, 0x3f, 0xcb, 0xe5, 0x6e, 0xdc, 0xde, 0xfc, 0x0d, 0x00,
-	0x00, 0xff, 0xff, 0xbf, 0x78, 0x2f, 0x36, 0x61, 0x02, 0x00, 0x00,
+	// 439 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0xcf, 0x6e, 0xd3, 0x40,
+	0x10, 0xc6, 0xbd, 0x4d, 0xdc, 0x96, 0x21, 0x2d, 0xdb, 0x6d, 0x4a, 0x2d, 0x24, 0xac, 0x28, 0xa7,
+	0x10, 0x89, 0x84, 0x7f, 0xe2, 0x1e, 0x9a, 0x22, 0x72, 0xa9, 0x22, 0x17, 0x7a, 0xe0, 0xb6, 0xf5,
+	0x0e, 0xed, 0x2a, 0xcb, 0xda, 0xda, 0xdd, 0x86, 0xfa, 0x2d, 0x78, 0x14, 0x1e, 0x83, 0x63, 0x8f,
+	0x1c, 0x51, 0xf2, 0x22, 0xc8, 0x9b, 0xa4, 0x71, 0x38, 0xf5, 0x62, 0xef, 0xcc, 0xfc, 0xc6, 0xdf,
+	0x7c, 0xe3, 0x85, 0x81, 0x46, 0xd7, 0xb7, 0x98, 0xde, 0x18, 0xb4, 0x68, 0xa6, 0x32, 0xc5, 0xfe,
+	0x35, 0xd7, 0xc2, 0x5e, 0xf3, 0x49, 0xe5, 0x94, 0x9b, 0xcc, 0x65, 0x7d, 0xff, 0xb4, 0xeb, 0x6c,
+	0xcf, 0x27, 0x58, 0x83, 0xeb, 0xe2, 0xd3, 0x2a, 0xd7, 0x76, 0xf0, 0xf8, 0xc4, 0xa0, 0x40, 0xed,
+	0x24, 0x57, 0x96, 0xbd, 0x86, 0xba, 0x2b, 0x72, 0x8c, 0x48, 0x8b, 0x74, 0xf6, 0xdf, 0x3c, 0xef,
+	0x55, 0xd9, 0x5e, 0x05, 0xfc, 0x5c, 0xe4, 0x98, 0x78, 0x94, 0x45, 0xb0, 0x93, 0xf3, 0x42, 0x65,
+	0x5c, 0x44, 0x5b, 0x2d, 0xd2, 0x69, 0x24, 0xab, 0xb0, 0xac, 0x4c, 0xd1, 0x58, 0x99, 0xe9, 0xa8,
+	0xd6, 0x22, 0x9d, 0xbd, 0x64, 0x15, 0xb6, 0x3f, 0x42, 0x73, 0xbc, 0x80, 0xce, 0xe5, 0x95, 0x46,
+	0x31, 0x46, 0x34, 0x23, 0x61, 0xd9, 0x33, 0xd8, 0x95, 0x5e, 0xc2, 0x15, 0x7e, 0x84, 0x46, 0x72,
+	0x1f, 0x33, 0x06, 0x75, 0x2b, 0xaf, 0xf4, 0x52, 0xc4, 0x9f, 0xdb, 0xaf, 0xa0, 0x36, 0x48, 0x27,
+	0xec, 0x05, 0x84, 0x68, 0x4c, 0x66, 0x96, 0x63, 0x1f, 0x6e, 0x8e, 0x7d, 0x5a, 0x96, 0x92, 0x05,
+	0xd1, 0x7e, 0x0f, 0xe1, 0xd8, 0xaf, 0xe1, 0x25, 0x84, 0x7e, 0x1f, 0xcb, 0x9e, 0xe3, 0xcd, 0x1e,
+	0xcf, 0x78, 0x93, 0x0b, 0xaa, 0xfb, 0x0e, 0x9e, 0xfc, 0x67, 0x9f, 0xed, 0x03, 0x9c, 0x4f, 0x64,
+	0x7e, 0x81, 0x46, 0x7e, 0x2b, 0x68, 0xc0, 0x0e, 0x60, 0x6f, 0xc3, 0x0d, 0x25, 0xdd, 0x5f, 0x04,
+	0x42, 0x2f, 0xcf, 0x76, 0xa1, 0x7e, 0x76, 0xa3, 0x14, 0x0d, 0xca, 0xb6, 0x2f, 0x1a, 0x6f, 0x73,
+	0x4c, 0x1d, 0x0a, 0x4a, 0xd8, 0x53, 0x60, 0x23, 0x3d, 0xe5, 0x4a, 0x8a, 0x8a, 0x00, 0xdd, 0x62,
+	0x47, 0x70, 0xb0, 0xe6, 0x96, 0xdb, 0xa2, 0x35, 0x16, 0x41, 0x73, 0xad, 0x7a, 0x96, 0xb9, 0x81,
+	0x52, 0xd9, 0x0f, 0x14, 0xb4, 0xce, 0x9a, 0x40, 0x87, 0xc8, 0x85, 0x92, 0x1a, 0x4f, 0x6f, 0x53,
+	0x44, 0x81, 0x82, 0x86, 0xec, 0x18, 0x0e, 0x47, 0x3a, 0xcd, 0xbe, 0xe7, 0xdc, 0xc9, 0x4b, 0x85,
+	0x17, 0x8b, 0x3f, 0x40, 0xb7, 0xcb, 0xef, 0x57, 0x0b, 0xde, 0x31, 0xdd, 0xe9, 0x1e, 0xc1, 0xa3,
+	0x7b, 0xf3, 0xe5, 0xd4, 0xc3, 0x64, 0x7c, 0x42, 0x83, 0x0f, 0xc3, 0xdf, 0xb3, 0x98, 0xdc, 0xcd,
+	0x62, 0xf2, 0x77, 0x16, 0x93, 0x9f, 0xf3, 0x38, 0xb8, 0x9b, 0xc7, 0xc1, 0x9f, 0x79, 0x1c, 0x7c,
+	0xed, 0x3e, 0xfc, 0x4a, 0x5e, 0x6e, 0xfb, 0xd7, 0xdb, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x53,
+	0x32, 0xf7, 0x79, 0xc7, 0x02, 0x00, 0x00,
 }
 
 func (m *Credentials) Marshal() (dAtA []byte, err error) {
@@ -393,6 +467,34 @@ func (m *Ack) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+func (m *Proto) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *Proto) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *Proto) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Proto != 0 {
+		i = encodeVarintHandshake(dAtA, i, uint64(m.Proto))
+		i--
+		dAtA[i] = 0x8
+	}
+	return len(dAtA) - i, nil
+}
+
 func encodeVarintHandshake(dAtA []byte, offset int, v uint64) int {
 	offset -= sovHandshake(v)
 	base := offset
@@ -452,6 +554,18 @@ func (m *Ack) Size() (n int) {
 	return n
 }
 
+func (m *Proto) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if m.Proto != 0 {
+		n += 1 + sovHandshake(uint64(m.Proto))
+	}
+	return n
+}
+
 func sovHandshake(x uint64) (n int) {
 	return (math_bits.Len64(x|1) + 6) / 7
 }
@@ -767,6 +881,75 @@ func (m *Ack) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *Proto) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowHandshake
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: Proto: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: Proto: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Proto", wireType)
+			}
+			m.Proto = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowHandshake
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Proto |= ProtoType(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipHandshake(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthHandshake
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func skipHandshake(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0

net/secureservice/handshake/handshakeproto/protos/handshake.proto

@@ -5,6 +5,8 @@ option go_package = "net/secureservice/handshake/handshakeproto";
 
 /*
 
+CREDENTIALS HANDSHAKE
+
 Alice opens a new connection with Bob
 1. TLS handshake done successfully; both sides know local and remote peer identifiers.
 
@@ -68,4 +70,20 @@ enum Error {
     SkipVerifyNotAllowed = 4;
     DeadlineExceeded = 5;
     IncompatibleVersion = 6;
+    IncompatibleProto = 7;
+}
+
+
+/*
+
+PROTO HANDSHAKE
+
+ */
+
+message Proto {
+    ProtoType proto = 1;
+}
+
+enum ProtoType {
+    DRPC = 0;
 }

net/secureservice/handshake/proto.go

@@ -0,0 +1,97 @@
+package handshake
+
+import (
+	"context"
+	"github.com/anyproto/any-sync/net/secureservice/handshake/handshakeproto"
+	"golang.org/x/exp/slices"
+	"net"
+)
+
+type ProtoChecker struct {
+	AllowedProtoTypes []handshakeproto.ProtoType
+}
+
+func OutgoingProtoHandshake(ctx context.Context, conn net.Conn, pt handshakeproto.ProtoType) (err error) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	h := newHandshake()
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		err = outgoingProtoHandshake(h, conn, pt)
+	}()
+	select {
+	case <-done:
+		return
+	case <-ctx.Done():
+		_ = conn.Close()
+		return ctx.Err()
+	}
+}
+
+func outgoingProtoHandshake(h *handshake, conn net.Conn, pt handshakeproto.ProtoType) (err error) {
+	defer h.release()
+	h.conn = conn
+	localProto := &handshakeproto.Proto{
+		Proto: pt,
+	}
+	if err = h.writeProto(localProto); err != nil {
+		h.tryWriteErrAndClose(err)
+		return
+	}
+	msg, err := h.readMsg(msgTypeAck)
+	if err != nil {
+		h.tryWriteErrAndClose(err)
+		return
+	}
+	if msg.ack.Error == handshakeproto.Error_IncompatibleProto {
+		return ErrRemoteIncompatibleProto
+	}
+	if msg.ack.Error == handshakeproto.Error_Null {
+		return nil
+	}
+	return HandshakeError{e: msg.ack.Error}
+}
+
+func IncomingProtoHandshake(ctx context.Context, conn net.Conn, pt ProtoChecker) (protoType handshakeproto.ProtoType, err error) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	h := newHandshake()
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		protoType, err = incomingProtoHandshake(h, conn, pt)
+	}()
+	select {
+	case <-done:
+		return
+	case <-ctx.Done():
+		_ = conn.Close()
+		return 0, ctx.Err()
+	}
+}
+
+func incomingProtoHandshake(h *handshake, conn net.Conn, pt ProtoChecker) (protoType handshakeproto.ProtoType, err error) {
+	defer h.release()
+	h.conn = conn
+
+	msg, err := h.readMsg(msgTypeProto)
+	if err != nil {
+		h.tryWriteErrAndClose(err)
+		return
+	}
+	if !slices.Contains(pt.AllowedProtoTypes, msg.proto.Proto) {
+		err = ErrIncompatibleProto
+		h.tryWriteErrAndClose(err)
+		return
+	}
+
+	if err = h.writeAck(handshakeproto.Error_Null); err != nil {
+		h.tryWriteErrAndClose(err)
+		return 0, err
+	} else {
+		return msg.proto.Proto, nil
+	}
+}

net/secureservice/handshake/proto_test.go

@@ -0,0 +1,121 @@
+package handshake
+
+import (
+	"github.com/anyproto/any-sync/net/secureservice/handshake/handshakeproto"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"testing"
+	"time"
+)
+
+type protoRes struct {
+	protoType handshakeproto.ProtoType
+	err       error
+}
+
+func newProtoChecker(types ...handshakeproto.ProtoType) ProtoChecker {
+	return ProtoChecker{AllowedProtoTypes: types}
+}
+func TestIncomingProtoHandshake(t *testing.T) {
+	t.Run("success", func(t *testing.T) {
+		c1, c2 := newConnPair(t)
+		var protoResCh = make(chan protoRes, 1)
+		go func() {
+			protoType, err := IncomingProtoHandshake(nil, c1, newProtoChecker(1))
+			protoResCh <- protoRes{protoType: protoType, err: err}
+		}()
+		h := newHandshake()
+		h.conn = c2
+
+		// write desired proto
+		require.NoError(t, h.writeProto(&handshakeproto.Proto{Proto: handshakeproto.ProtoType(1)}))
+		msg, err := h.readMsg(msgTypeAck)
+		require.NoError(t, err)
+		assert.Equal(t, handshakeproto.Error_Null, msg.ack.Error)
+		res := <-protoResCh
+		require.NoError(t, res.err)
+		assert.Equal(t, handshakeproto.ProtoType(1), res.protoType)
+	})
+	t.Run("incompatible", func(t *testing.T) {
+		c1, c2 := newConnPair(t)
+		var protoResCh = make(chan protoRes, 1)
+		go func() {
+			protoType, err := IncomingProtoHandshake(nil, c1, newProtoChecker(1))
+			protoResCh <- protoRes{protoType: protoType, err: err}
+		}()
+		h := newHandshake()
+		h.conn = c2
+
+		// write desired proto
+		require.NoError(t, h.writeProto(&handshakeproto.Proto{Proto: 0}))
+		msg, err := h.readMsg(msgTypeAck)
+		require.NoError(t, err)
+		assert.Equal(t, handshakeproto.Error_IncompatibleProto, msg.ack.Error)
+		res := <-protoResCh
+		require.Error(t, res.err, ErrIncompatibleProto.Error())
+	})
+}
+
+func TestOutgoingProtoHandshake(t *testing.T) {
+	t.Run("success", func(t *testing.T) {
+		c1, c2 := newConnPair(t)
+		var protoResCh = make(chan protoRes, 1)
+		go func() {
+			err := OutgoingProtoHandshake(nil, c1, 1)
+			protoResCh <- protoRes{err: err}
+		}()
+		h := newHandshake()
+		h.conn = c2
+
+		msg, err := h.readMsg(msgTypeProto)
+		require.NoError(t, err)
+		assert.Equal(t, handshakeproto.ProtoType(1), msg.proto.Proto)
+		require.NoError(t, h.writeAck(handshakeproto.Error_Null))
+
+		res := <-protoResCh
+		assert.NoError(t, res.err)
+	})
+	t.Run("incompatible", func(t *testing.T) {
+		c1, c2 := newConnPair(t)
+		var protoResCh = make(chan protoRes, 1)
+		go func() {
+			err := OutgoingProtoHandshake(nil, c1, 1)
+			protoResCh <- protoRes{err: err}
+		}()
+		h := newHandshake()
+		h.conn = c2
+
+		msg, err := h.readMsg(msgTypeProto)
+		require.NoError(t, err)
+		assert.Equal(t, handshakeproto.ProtoType(1), msg.proto.Proto)
+		require.NoError(t, h.writeAck(handshakeproto.Error_IncompatibleProto))
+
+		res := <-protoResCh
+		assert.EqualError(t, res.err, ErrRemoteIncompatibleProto.Error())
+	})
+}
+
+func TestEndToEndProto(t *testing.T) {
+	c1, c2 := newConnPair(t)
+	var (
+		inResCh  = make(chan protoRes, 1)
+		outResCh = make(chan protoRes, 1)
+	)
+	st := time.Now()
+	go func() {
+		err := OutgoingProtoHandshake(nil, c1, 0)
+		outResCh <- protoRes{err: err}
+	}()
+	go func() {
+		protoType, err := IncomingProtoHandshake(nil, c2, newProtoChecker(0, 1))
+		inResCh <- protoRes{protoType: protoType, err: err}
+	}()
+
+	outRes := <-outResCh
+	assert.NoError(t, outRes.err)
+
+	inRes := <-inResCh
+	assert.NoError(t, inRes.err)
+	assert.Equal(t, handshakeproto.ProtoType(0), inRes.protoType)
+	t.Log("dur", time.Since(st))
+}

net/secureservice/secureservice.go

@@ -2,6 +2,7 @@ package secureservice
 
 import (
 	"context"
+	"crypto/tls"
 	commonaccount "github.com/anyproto/any-sync/accountservice"
 	"github.com/anyproto/any-sync/app"
 	"github.com/anyproto/any-sync/app/logger"
@@ -10,9 +11,9 @@ import (
 	"github.com/anyproto/any-sync/net/secureservice/handshake"
 	"github.com/anyproto/any-sync/nodeconf"
 	"github.com/libp2p/go-libp2p/core/crypto"
-	"github.com/libp2p/go-libp2p/core/sec"
 	libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls"
 	"go.uber.org/zap"
+	"io"
 	"net"
 )
 
@@ -25,8 +26,10 @@ func New() SecureService {
 }
 
 type SecureService interface {
-	SecureOutbound(ctx context.Context, conn net.Conn) (sec.SecureConn, error)
-	SecureInbound(ctx context.Context, conn net.Conn) (cctx context.Context, sc sec.SecureConn, err error)
+	SecureOutbound(ctx context.Context, conn net.Conn) (cctx context.Context, err error)
+	SecureInbound(ctx context.Context, conn net.Conn) (cctx context.Context, err error)
+	HandshakeInbound(ctx context.Context, conn io.ReadWriteCloser, remotePeerId string) (cctx context.Context, err error)
+	ServerTlsConfig() (*tls.Config, error)
 	app.Component
 }
 
@@ -75,25 +78,28 @@ func (s *secureService) Name() (name string) {
 	return CName
 }
 
-func (s *secureService) SecureInbound(ctx context.Context, conn net.Conn) (cctx context.Context, sc sec.SecureConn, err error) {
-	sc, err = s.p2pTr.SecureInbound(ctx, conn, "")
+func (s *secureService) SecureInbound(ctx context.Context, conn net.Conn) (cctx context.Context, err error) {
+	sc, err := s.p2pTr.SecureInbound(ctx, conn, "")
 	if err != nil {
-		return nil, nil, handshake.HandshakeError{
+		return nil, handshake.HandshakeError{
 			Err: err,
 		}
 	}
+	return s.HandshakeInbound(ctx, sc, sc.RemotePeer().String())
+}
 
-	identity, err := handshake.IncomingHandshake(ctx, sc, s.inboundChecker)
+func (s *secureService) HandshakeInbound(ctx context.Context, conn io.ReadWriteCloser, peerId string) (cctx context.Context, err error) {
+	identity, err := handshake.IncomingHandshake(ctx, conn, peerId, s.inboundChecker)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 	cctx = context.Background()
-	cctx = peer.CtxWithPeerId(cctx, sc.RemotePeer().String())
+	cctx = peer.CtxWithPeerId(cctx, peerId)
 	cctx = peer.CtxWithIdentity(cctx, identity)
 	return
 }
 
-func (s *secureService) SecureOutbound(ctx context.Context, conn net.Conn) (sec.SecureConn, error) {
+func (s *secureService) SecureOutbound(ctx context.Context, conn net.Conn) (cctx context.Context, err error) {
 	sc, err := s.p2pTr.SecureOutbound(ctx, conn, "")
 	if err != nil {
 		return nil, handshake.HandshakeError{Err: err}
@@ -106,10 +112,22 @@ func (s *secureService) SecureOutbound(ctx context.Context, conn net.Conn) (sec.
 	} else {
 		checker = s.noVerifyChecker
 	}
-	// ignore identity for outgoing connection because we don't need it at this moment
-	_, err = handshake.OutgoingHandshake(ctx, sc, checker)
+	identity, err := handshake.OutgoingHandshake(ctx, sc, sc.RemotePeer().String(), checker)
 	if err != nil {
 		return nil, err
 	}
-	return sc, nil
+	cctx = context.Background()
+	cctx = peer.CtxWithPeerId(cctx, sc.RemotePeer().String())
+	cctx = peer.CtxWithIdentity(cctx, identity)
+	return cctx, nil
+}
+
+func (s *secureService) ServerTlsConfig() (*tls.Config, error) {
+	p2pIdn, err := libp2ptls.NewIdentity(s.key)
+	if err != nil {
+		return nil, err
+	}
+	conf, _ := p2pIdn.ConfigForPeer("")
+	conf.NextProtos = []string{"anysync"}
+	return conf, nil
 }

net/secureservice/secureservice_test.go

@@ -32,16 +32,18 @@ func TestHandshake(t *testing.T) {
 	resCh := make(chan acceptRes)
 	go func() {
 		var ar acceptRes
-		ar.ctx, ar.conn, ar.err = fxS.SecureInbound(ctx, sc)
+		ar.ctx, ar.err = fxS.SecureInbound(ctx, sc)
 		resCh <- ar
 	}()
 
 	fxC := newFixture(t, nc, nc.GetAccountService(1), 0)
 	defer fxC.Finish(t)
 
-	secConn, err := fxC.SecureOutbound(ctx, cc)
+	cctx, err := fxC.SecureOutbound(ctx, cc)
 	require.NoError(t, err)
-	assert.Equal(t, nc.GetAccountService(0).Account().PeerId, secConn.RemotePeer().String())
+	ctxPeerId, err := peer.CtxPeerId(cctx)
+	require.NoError(t, err)
+	assert.Equal(t, nc.GetAccountService(0).Account().PeerId, ctxPeerId)
 	res := <-resCh
 	require.NoError(t, res.err)
 	peerId, err := peer.CtxPeerId(res.ctx)
@@ -67,7 +69,7 @@ func TestHandshakeIncompatibleVersion(t *testing.T) {
 	resCh := make(chan acceptRes)
 	go func() {
 		var ar acceptRes
-		ar.ctx, ar.conn, ar.err = fxS.SecureInbound(ctx, sc)
+		ar.ctx, ar.err = fxS.SecureInbound(ctx, sc)
 		resCh <- ar
 	}()
 	fxC := newFixture(t, nc, nc.GetAccountService(1), 1)

net/streampool/stream.go

@@ -3,7 +3,7 @@ package streampool
 import (
 	"context"
 	"github.com/anyproto/any-sync/app/logger"
-	"github.com/anyproto/any-sync/util/multiqueue"
+	"github.com/cheggaaa/mb/v3"
 	"go.uber.org/zap"
 	"storj.io/drpc"
 	"sync/atomic"
@@ -17,17 +17,12 @@ type stream struct {
 	streamId uint32
 	closed   atomic.Bool
 	l        logger.CtxLogger
-	queue    multiqueue.MultiQueue[drpc.Message]
+	queue    *mb.MB[drpc.Message]
 	tags     []string
 }
 
 func (sr *stream) write(msg drpc.Message) (err error) {
-	var queueId string
-	if qId, ok := msg.(MessageQueueId); ok {
-		queueId = qId.MessageQueueId()
-		msg = qId.DrpcMessage()
-	}
-	return sr.queue.Add(sr.stream.Context(), queueId, msg)
+	return sr.queue.Add(sr.stream.Context(), msg)
 }
 
 func (sr *stream) readLoop() error {
@@ -50,13 +45,21 @@ func (sr *stream) readLoop() error {
 	}
 }
 
-func (sr *stream) writeToStream(msg drpc.Message) {
-	if err := sr.stream.MsgSend(msg, EncodingProto); err != nil {
-		sr.l.Warn("msg send error", zap.Error(err))
-		sr.streamClose()
-		return
+func (sr *stream) writeLoop() {
+	for {
+		msg, err := sr.queue.WaitOne(sr.peerCtx)
+		if err != nil {
+			if err != mb.ErrClosed {
+				sr.streamClose()
+			}
+			return
+		}
+		if err := sr.stream.MsgSend(msg, EncodingProto); err != nil {
+			sr.l.Warn("msg send error", zap.Error(err))
+			sr.streamClose()
+			return
+		}
 	}
-	return
 }
 
 func (sr *stream) streamClose() {

net/streampool/streampool.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"github.com/anyproto/any-sync/net"
 	"github.com/anyproto/any-sync/net/peer"
-	"github.com/anyproto/any-sync/util/multiqueue"
+	"github.com/cheggaaa/mb/v3"
 	"go.uber.org/zap"
 	"golang.org/x/exp/slices"
 	"golang.org/x/net/context"
@@ -74,6 +74,9 @@ func (s *streamPool) ReadStream(drpcStream drpc.Stream, tags ...string) error {
 	if err != nil {
 		return err
 	}
+	go func() {
+		st.writeLoop()
+	}()
 	return st.readLoop()
 }
 
@@ -85,6 +88,9 @@ func (s *streamPool) AddStream(drpcStream drpc.Stream, tags ...string) error {
 	go func() {
 		_ = st.readLoop()
 	}()
+	go func() {
+		st.writeLoop()
+	}()
 	return nil
 }
 
@@ -122,7 +128,7 @@ func (s *streamPool) addStream(drpcStream drpc.Stream, tags ...string) (*stream,
 		l:        log.With(zap.String("peerId", peerId), zap.Uint32("streamId", streamId)),
 		tags:     tags,
 	}
-	st.queue = multiqueue.New[drpc.Message](st.writeToStream, s.writeQueueSize)
+	st.queue = mb.New[drpc.Message](s.writeQueueSize)
 	s.streams[streamId] = st
 	s.streamIdsByPeer[peerId] = append(s.streamIdsByPeer[peerId], streamId)
 	for _, tag := range tags {
@@ -364,21 +370,3 @@ func removeStream(m map[string][]uint32, key string, streamId uint32) {
 		m[key] = streamIds
 	}
 }
-
-// WithQueueId wraps the message and adds queueId
-func WithQueueId(msg drpc.Message, queueId string) drpc.Message {
-	return &messageWithQueueId{queueId: queueId, Message: msg}
-}
-
-type messageWithQueueId struct {
-	drpc.Message
-	queueId string
-}
-
-func (m messageWithQueueId) MessageQueueId() string {
-	return m.queueId
-}
-
-func (m messageWithQueueId) DrpcMessage() drpc.Message {
-	return m.Message
-}

net/streampool/streampool_test.go

@@ -18,17 +18,25 @@ import (
 
 var ctx = context.Background()
 
+func makePeerPair(t *testing.T, fx *fixture, peerId string) (pS, pC peer.Peer) {
+	mcS, mcC := rpctest.MultiConnPair(peerId+"server", peerId)
+	pS, err := peer.NewPeer(mcS, fx.ts)
+	require.NoError(t, err)
+	pC, err = peer.NewPeer(mcC, fx.ts)
+	require.NoError(t, err)
+	return
+}
+
 func newClientStream(t *testing.T, fx *fixture, peerId string) (st testservice.DRPCTest_TestStreamClient, p peer.Peer) {
-	p, err := fx.tp.Dial(ctx, peerId)
+	_, pC := makePeerPair(t, fx, peerId)
+	drpcConn, err := pC.AcquireDrpcConn(ctx)
 	require.NoError(t, err)
-	ctx = peer.CtxWithPeerId(ctx, peerId)
-	s, err := testservice.NewDRPCTestClient(p).TestStream(ctx)
+	st, err = testservice.NewDRPCTestClient(drpcConn).TestStream(pC.Context())
 	require.NoError(t, err)
-	return s, p
+	return st, pC
 }
 
 func TestStreamPool_AddStream(t *testing.T) {
-
 	t.Run("broadcast incoming", func(t *testing.T) {
 		fx := newFixture(t)
 		defer fx.Finish(t)
@@ -39,7 +47,7 @@ func TestStreamPool_AddStream(t *testing.T) {
 		require.NoError(t, fx.AddStream(s2, "space2", "common"))
 
 		require.NoError(t, fx.Broadcast(ctx, &testservice.StreamMessage{ReqData: "space1"}, "space1"))
-		require.NoError(t, fx.Broadcast(ctx, WithQueueId(&testservice.StreamMessage{ReqData: "space2"}, "q2"), "space2"))
+		require.NoError(t, fx.Broadcast(ctx, &testservice.StreamMessage{ReqData: "space2"}, "space2"))
 		require.NoError(t, fx.Broadcast(ctx, &testservice.StreamMessage{ReqData: "common"}, "common"))
 
 		var serverResults []string
@@ -85,11 +93,10 @@ func TestStreamPool_Send(t *testing.T) {
 		fx := newFixture(t)
 		defer fx.Finish(t)
 
-		p, err := fx.tp.Dial(ctx, "p1")
-		require.NoError(t, err)
+		pS, _ := makePeerPair(t, fx, "p1")
 
 		require.NoError(t, fx.Send(ctx, &testservice.StreamMessage{ReqData: "should open stream"}, func(ctx context.Context) (peers []peer.Peer, err error) {
-			return []peer.Peer{p}, nil
+			return []peer.Peer{pS}, nil
 		}))
 
 		var msg *testservice.StreamMessage
@@ -100,12 +107,12 @@ func TestStreamPool_Send(t *testing.T) {
 		}
 		assert.Equal(t, "should open stream", msg.ReqData)
 	})
+
 	t.Run("parallel open stream", func(t *testing.T) {
 		fx := newFixture(t)
 		defer fx.Finish(t)
 
-		p, err := fx.tp.Dial(ctx, "p1")
-		require.NoError(t, err)
+		pS, _ := makePeerPair(t, fx, "p1")
 
 		fx.th.streamOpenDelay = time.Second / 3
 
@@ -113,7 +120,7 @@ func TestStreamPool_Send(t *testing.T) {
 
 		for i := 0; i < numMsgs; i++ {
 			go require.NoError(t, fx.Send(ctx, &testservice.StreamMessage{ReqData: "should open stream"}, func(ctx context.Context) (peers []peer.Peer, err error) {
-				return []peer.Peer{p}, nil
+				return []peer.Peer{pS}, nil
 			}))
 		}
 
@@ -134,9 +141,8 @@ func TestStreamPool_Send(t *testing.T) {
 		fx := newFixture(t)
 		defer fx.Finish(t)
 
-		p, err := fx.tp.Dial(ctx, "p1")
-		require.NoError(t, err)
-		_ = p.Close()
+		pS, _ := makePeerPair(t, fx, "p1")
+		_ = pS.Close()
 
 		fx.th.streamOpenDelay = time.Second / 3
 
@@ -147,11 +153,12 @@ func TestStreamPool_Send(t *testing.T) {
 			wg.Add(1)
 			go func() {
 				defer wg.Done()
-				assert.Error(t, fx.StreamPool.(*streamPool).sendOne(ctx, p, &testservice.StreamMessage{ReqData: "should open stream"}))
+				assert.Error(t, fx.StreamPool.(*streamPool).sendOne(ctx, pS, &testservice.StreamMessage{ReqData: "should open stream"}))
 			}()
 		}
 		wg.Wait()
 	})
+
 }
 
 func TestStreamPool_SendById(t *testing.T) {
@@ -196,10 +203,9 @@ func TestStreamPool_Tags(t *testing.T) {
 
 func newFixture(t *testing.T) *fixture {
 	fx := &fixture{}
-	ts := rpctest.NewTestServer()
+	fx.ts = rpctest.NewTestServer()
 	fx.tsh = &testServerHandler{receiveCh: make(chan *testservice.StreamMessage, 100)}
-	require.NoError(t, testservice.DRPCRegisterTest(ts, fx.tsh))
-	fx.tp = rpctest.NewTestPool().WithServer(ts)
+	require.NoError(t, testservice.DRPCRegisterTest(fx.ts, fx.tsh))
 	fx.th = &testHandler{}
 	fx.StreamPool = New().NewStreamPool(fx.th, StreamConfig{
 		SendQueueSize:    10,
@@ -211,14 +217,13 @@ func newFixture(t *testing.T) *fixture {
 
 type fixture struct {
 	StreamPool
-	tp  *rpctest.TestPool
 	th  *testHandler
 	tsh *testServerHandler
+	ts  *rpctest.TestServer
 }
 
 func (fx *fixture) Finish(t *testing.T) {
 	require.NoError(t, fx.Close())
-	require.NoError(t, fx.tp.Close(ctx))
 }
 
 type testHandler struct {
@@ -231,7 +236,11 @@ func (t *testHandler) OpenStream(ctx context.Context, p peer.Peer) (stream drpc.
 	if t.streamOpenDelay > 0 {
 		time.Sleep(t.streamOpenDelay)
 	}
-	stream, err = testservice.NewDRPCTestClient(p).TestStream(ctx)
+	conn, err := p.AcquireDrpcConn(ctx)
+	if err != nil {
+		return
+	}
+	stream, err = testservice.NewDRPCTestClient(conn).TestStream(p.Context())
 	return
 }
 

net/streampool/testservice/testservice_drpc.pb.go

@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go-drpc. DO NOT EDIT.
-// protoc-gen-go-drpc version: v0.0.32
+// protoc-gen-go-drpc version: v0.0.33
 // source: net/streampool/testservice/protos/testservice.proto
 
 package testservice
@@ -72,6 +72,10 @@ type drpcTest_TestStreamClient struct {
 	drpc.Stream
 }
 
+func (x *drpcTest_TestStreamClient) GetStream() drpc.Stream {
+	return x.Stream
+}
+
 func (x *drpcTest_TestStreamClient) Send(m *StreamMessage) error {
 	return x.MsgSend(m, drpcEncoding_File_net_streampool_testservice_protos_testservice_proto{})
 }

net/transport/mock_transport/mock_transport.go

@@ -0,0 +1,188 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/anyproto/any-sync/net/transport (interfaces: Transport,MultiConn)
+
+// Package mock_transport is a generated GoMock package.
+package mock_transport
+
+import (
+	context "context"
+	net "net"
+	reflect "reflect"
+	time "time"
+
+	transport "github.com/anyproto/any-sync/net/transport"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockTransport is a mock of Transport interface.
+type MockTransport struct {
+	ctrl     *gomock.Controller
+	recorder *MockTransportMockRecorder
+}
+
+// MockTransportMockRecorder is the mock recorder for MockTransport.
+type MockTransportMockRecorder struct {
+	mock *MockTransport
+}
+
+// NewMockTransport creates a new mock instance.
+func NewMockTransport(ctrl *gomock.Controller) *MockTransport {
+	mock := &MockTransport{ctrl: ctrl}
+	mock.recorder = &MockTransportMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockTransport) EXPECT() *MockTransportMockRecorder {
+	return m.recorder
+}
+
+// Dial mocks base method.
+func (m *MockTransport) Dial(arg0 context.Context, arg1 string) (transport.MultiConn, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Dial", arg0, arg1)
+	ret0, _ := ret[0].(transport.MultiConn)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Dial indicates an expected call of Dial.
+func (mr *MockTransportMockRecorder) Dial(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Dial", reflect.TypeOf((*MockTransport)(nil).Dial), arg0, arg1)
+}
+
+// SetAccepter mocks base method.
+func (m *MockTransport) SetAccepter(arg0 transport.Accepter) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "SetAccepter", arg0)
+}
+
+// SetAccepter indicates an expected call of SetAccepter.
+func (mr *MockTransportMockRecorder) SetAccepter(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetAccepter", reflect.TypeOf((*MockTransport)(nil).SetAccepter), arg0)
+}
+
+// MockMultiConn is a mock of MultiConn interface.
+type MockMultiConn struct {
+	ctrl     *gomock.Controller
+	recorder *MockMultiConnMockRecorder
+}
+
+// MockMultiConnMockRecorder is the mock recorder for MockMultiConn.
+type MockMultiConnMockRecorder struct {
+	mock *MockMultiConn
+}
+
+// NewMockMultiConn creates a new mock instance.
+func NewMockMultiConn(ctrl *gomock.Controller) *MockMultiConn {
+	mock := &MockMultiConn{ctrl: ctrl}
+	mock.recorder = &MockMultiConnMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockMultiConn) EXPECT() *MockMultiConnMockRecorder {
+	return m.recorder
+}
+
+// Accept mocks base method.
+func (m *MockMultiConn) Accept() (net.Conn, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Accept")
+	ret0, _ := ret[0].(net.Conn)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Accept indicates an expected call of Accept.
+func (mr *MockMultiConnMockRecorder) Accept() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Accept", reflect.TypeOf((*MockMultiConn)(nil).Accept))
+}
+
+// Addr mocks base method.
+func (m *MockMultiConn) Addr() string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Addr")
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// Addr indicates an expected call of Addr.
+func (mr *MockMultiConnMockRecorder) Addr() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Addr", reflect.TypeOf((*MockMultiConn)(nil).Addr))
+}
+
+// Close mocks base method.
+func (m *MockMultiConn) Close() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Close")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Close indicates an expected call of Close.
+func (mr *MockMultiConnMockRecorder) Close() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockMultiConn)(nil).Close))
+}
+
+// Context mocks base method.
+func (m *MockMultiConn) Context() context.Context {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Context")
+	ret0, _ := ret[0].(context.Context)
+	return ret0
+}
+
+// Context indicates an expected call of Context.
+func (mr *MockMultiConnMockRecorder) Context() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Context", reflect.TypeOf((*MockMultiConn)(nil).Context))
+}
+
+// IsClosed mocks base method.
+func (m *MockMultiConn) IsClosed() bool {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "IsClosed")
+	ret0, _ := ret[0].(bool)
+	return ret0
+}
+
+// IsClosed indicates an expected call of IsClosed.
+func (mr *MockMultiConnMockRecorder) IsClosed() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsClosed", reflect.TypeOf((*MockMultiConn)(nil).IsClosed))
+}
+
+// LastUsage mocks base method.
+func (m *MockMultiConn) LastUsage() time.Time {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "LastUsage")
+	ret0, _ := ret[0].(time.Time)
+	return ret0
+}
+
+// LastUsage indicates an expected call of LastUsage.
+func (mr *MockMultiConnMockRecorder) LastUsage() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LastUsage", reflect.TypeOf((*MockMultiConn)(nil).LastUsage))
+}
+
+// Open mocks base method.
+func (m *MockMultiConn) Open(arg0 context.Context) (net.Conn, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Open", arg0)
+	ret0, _ := ret[0].(net.Conn)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Open indicates an expected call of Open.
+func (mr *MockMultiConnMockRecorder) Open(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Open", reflect.TypeOf((*MockMultiConn)(nil).Open), arg0)
+}

net/transport/transport.go

@@ -0,0 +1,44 @@
+//go:generate mockgen -destination mock_transport/mock_transport.go github.com/anyproto/any-sync/net/transport Transport,MultiConn
+package transport
+
+import (
+	"context"
+	"errors"
+	"net"
+	"time"
+)
+
+var (
+	ErrConnClosed = errors.New("connection closed")
+)
+
+// Transport is a common interface for a network transport
+type Transport interface {
+	// SetAccepter sets accepter that will be called for new connections
+	// this method should be called before app start
+	SetAccepter(accepter Accepter)
+	// Dial creates a new connection by given address
+	Dial(ctx context.Context, addr string) (mc MultiConn, err error)
+}
+
+// MultiConn is an object of multiplexing connection containing handshake info
+type MultiConn interface {
+	// Context returns the connection context that contains handshake details
+	Context() context.Context
+	// Accept accepts new sub connections
+	Accept() (conn net.Conn, err error)
+	// Open opens new sub connection
+	Open(ctx context.Context) (conn net.Conn, err error)
+	// LastUsage returns the time of the last connection activity
+	LastUsage() time.Time
+	// Addr returns remote peer address
+	Addr() string
+	// IsClosed returns true when connection is closed
+	IsClosed() bool
+	// Close closes the connection and all sub connections
+	Close() error
+}
+
+type Accepter interface {
+	Accept(mc MultiConn) (err error)
+}

net/transport/yamux/config.go

@@ -0,0 +1,11 @@
+package yamux
+
+type configGetter interface {
+	GetYamux() Config
+}
+
+type Config struct {
+	ListenAddrs     []string `yaml:"listenAddrs"`
+	WriteTimeoutSec int      `yaml:"writeTimeoutSec"`
+	DialTimeoutSec  int      `yaml:"dialTimeoutSec"`
+}

net/transport/yamux/conn.go

@@ -0,0 +1,55 @@
+package yamux
+
+import (
+	"context"
+	"github.com/anyproto/any-sync/net/connutil"
+	"github.com/anyproto/any-sync/net/transport"
+	"github.com/hashicorp/yamux"
+	"net"
+	"time"
+)
+
+func NewMultiConn(cctx context.Context, luConn *connutil.LastUsageConn, addr string, sess *yamux.Session) transport.MultiConn {
+	return &yamuxConn{
+		ctx:     cctx,
+		luConn:  luConn,
+		addr:    addr,
+		Session: sess,
+	}
+}
+
+type yamuxConn struct {
+	ctx    context.Context
+	luConn *connutil.LastUsageConn
+	addr   string
+	*yamux.Session
+}
+
+func (y *yamuxConn) Open(ctx context.Context) (conn net.Conn, err error) {
+	if conn, err = y.Session.Open(); err != nil {
+		return
+	}
+	return
+}
+
+func (y *yamuxConn) LastUsage() time.Time {
+	return y.luConn.LastUsage()
+}
+
+func (y *yamuxConn) Context() context.Context {
+	return y.ctx
+}
+
+func (y *yamuxConn) Addr() string {
+	return y.addr
+}
+
+func (y *yamuxConn) Accept() (conn net.Conn, err error) {
+	if conn, err = y.Session.Accept(); err != nil {
+		if err == yamux.ErrSessionShutdown {
+			err = transport.ErrConnClosed
+		}
+		return
+	}
+	return
+}

net/transport/yamux/util.go

@@ -1,6 +1,6 @@
 //go:build !windows
 
-package server
+package yamux
 
 import (
 	"errors"

net/transport/yamux/util_windows.go

@@ -1,6 +1,6 @@
 //go:build windows
 
-package server
+package yamux
 
 import (
 	"errors"

net/transport/yamux/yamux.go

@@ -0,0 +1,157 @@
+package yamux
+
+import (
+	"context"
+	"fmt"
+	"github.com/anyproto/any-sync/app"
+	"github.com/anyproto/any-sync/app/logger"
+	"github.com/anyproto/any-sync/net/connutil"
+	"github.com/anyproto/any-sync/net/secureservice"
+	"github.com/anyproto/any-sync/net/transport"
+	"github.com/hashicorp/yamux"
+	"go.uber.org/zap"
+	"net"
+	"time"
+)
+
+const CName = "net.transport.yamux"
+
+var log = logger.NewNamed(CName)
+
+func New() Yamux {
+	return new(yamuxTransport)
+}
+
+// Yamux implements transport.Transport with tcp+yamux
+type Yamux interface {
+	transport.Transport
+	app.ComponentRunnable
+}
+
+type yamuxTransport struct {
+	secure   secureservice.SecureService
+	accepter transport.Accepter
+	conf     Config
+
+	listeners     []net.Listener
+	listCtx       context.Context
+	listCtxCancel context.CancelFunc
+	yamuxConf     *yamux.Config
+}
+
+func (y *yamuxTransport) Init(a *app.App) (err error) {
+	y.secure = a.MustComponent(secureservice.CName).(secureservice.SecureService)
+	y.conf = a.MustComponent("config").(configGetter).GetYamux()
+	y.yamuxConf = yamux.DefaultConfig()
+	y.yamuxConf.EnableKeepAlive = false
+	y.yamuxConf.StreamOpenTimeout = time.Duration(y.conf.DialTimeoutSec) * time.Second
+	y.yamuxConf.ConnectionWriteTimeout = time.Duration(y.conf.WriteTimeoutSec) * time.Second
+	return
+}
+
+func (y *yamuxTransport) Name() string {
+	return CName
+}
+
+func (y *yamuxTransport) Run(ctx context.Context) (err error) {
+	if y.accepter == nil {
+		return fmt.Errorf("can't run service without accepter")
+	}
+	for _, listAddr := range y.conf.ListenAddrs {
+		list, err := net.Listen("tcp", listAddr)
+		if err != nil {
+			return err
+		}
+		y.listeners = append(y.listeners, list)
+	}
+	y.listCtx, y.listCtxCancel = context.WithCancel(context.Background())
+	for _, list := range y.listeners {
+		go y.acceptLoop(y.listCtx, list)
+	}
+	return
+}
+
+func (y *yamuxTransport) SetAccepter(accepter transport.Accepter) {
+	y.accepter = accepter
+}
+
+func (y *yamuxTransport) Dial(ctx context.Context, addr string) (mc transport.MultiConn, err error) {
+	dialTimeout := time.Duration(y.conf.DialTimeoutSec) * time.Second
+	conn, err := net.DialTimeout("tcp", addr, dialTimeout)
+	if err != nil {
+		return nil, err
+	}
+	ctx, cancel := context.WithTimeout(ctx, dialTimeout)
+	defer cancel()
+	cctx, err := y.secure.SecureOutbound(ctx, conn)
+	if err != nil {
+		_ = conn.Close()
+		return nil, err
+	}
+	luc := connutil.NewLastUsageConn(conn)
+	sess, err := yamux.Client(luc, y.yamuxConf)
+	if err != nil {
+		return
+	}
+	mc = NewMultiConn(cctx, luc, addr, sess)
+	return
+}
+
+func (y *yamuxTransport) acceptLoop(ctx context.Context, list net.Listener) {
+	l := log.With(zap.String("localAddr", list.Addr().String()))
+	l.Info("yamux listener started")
+	defer func() {
+		l.Debug("yamux listener stopped")
+	}()
+	for {
+		conn, err := list.Accept()
+		if err != nil {
+			if isTemporary(err) {
+				l.Debug("listener temporary accept error", zap.Error(err))
+				select {
+				case <-time.After(time.Second):
+				case <-ctx.Done():
+					return
+				}
+				continue
+			}
+			if err != net.ErrClosed {
+				l.Error("listener closed with error", zap.Error(err))
+			} else {
+				l.Info("listener closed")
+			}
+			return
+		}
+		go y.accept(conn)
+	}
+}
+
+func (y *yamuxTransport) accept(conn net.Conn) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(y.conf.DialTimeoutSec)*time.Second)
+	defer cancel()
+	cctx, err := y.secure.SecureInbound(ctx, conn)
+	if err != nil {
+		log.Warn("incoming connection handshake error", zap.Error(err))
+		return
+	}
+	luc := connutil.NewLastUsageConn(conn)
+	sess, err := yamux.Server(luc, y.yamuxConf)
+	if err != nil {
+		log.Warn("incoming connection yamux session error", zap.Error(err))
+		return
+	}
+	mc := NewMultiConn(cctx, luc, conn.RemoteAddr().String(), sess)
+	if err = y.accepter.Accept(mc); err != nil {
+		log.Warn("connection accept error", zap.Error(err))
+	}
+}
+
+func (y *yamuxTransport) Close(ctx context.Context) (err error) {
+	if y.listCtxCancel != nil {
+		y.listCtxCancel()
+	}
+	for _, l := range y.listeners {
+		_ = l.Close()
+	}
+	return
+}

net/transport/yamux/yamux_test.go

@@ -0,0 +1,194 @@
+package yamux
+
+import (
+	"bytes"
+	"context"
+	"github.com/anyproto/any-sync/app"
+	"github.com/anyproto/any-sync/net/secureservice"
+	"github.com/anyproto/any-sync/net/transport"
+	"github.com/anyproto/any-sync/nodeconf"
+	"github.com/anyproto/any-sync/nodeconf/mock_nodeconf"
+	"github.com/anyproto/any-sync/testutil/accounttest"
+	"github.com/anyproto/any-sync/testutil/testnodeconf"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"io"
+	"net"
+	"sync"
+	"testing"
+	"time"
+)
+
+var ctx = context.Background()
+
+func TestYamuxTransport_Dial(t *testing.T) {
+	fxS := newFixture(t)
+	defer fxS.finish(t)
+	fxC := newFixture(t)
+	defer fxC.finish(t)
+
+	mcC, err := fxC.Dial(ctx, fxS.addr)
+	require.NoError(t, err)
+	require.Len(t, fxS.accepter.mcs, 1)
+	mcS := <-fxS.accepter.mcs
+
+	var (
+		sData     string
+		acceptErr error
+		copyErr   error
+		done      = make(chan struct{})
+	)
+
+	go func() {
+		defer close(done)
+		conn, serr := mcS.Accept()
+		if serr != nil {
+			acceptErr = serr
+			return
+		}
+		buf := bytes.NewBuffer(nil)
+		_, copyErr = io.Copy(buf, conn)
+		sData = buf.String()
+		return
+	}()
+
+	conn, err := mcC.Open(ctx)
+	require.NoError(t, err)
+	data := "some data"
+	_, err = conn.Write([]byte(data))
+	require.NoError(t, err)
+	require.NoError(t, conn.Close())
+	<-done
+
+	assert.NoError(t, acceptErr)
+	assert.Equal(t, data, sData)
+	assert.NoError(t, copyErr)
+}
+
+// no deadline - 69100 rps
+// common write deadline - 66700 rps
+// subconn write deadline - 67100 rps
+func TestWriteBench(t *testing.T) {
+	t.Skip()
+	var (
+		numSubConn = 10
+		numWrites  = 100000
+	)
+
+	fxS := newFixture(t)
+	defer fxS.finish(t)
+	fxC := newFixture(t)
+	defer fxC.finish(t)
+
+	mcC, err := fxC.Dial(ctx, fxS.addr)
+	require.NoError(t, err)
+	mcS := <-fxS.accepter.mcs
+
+	go func() {
+		for i := 0; i < numSubConn; i++ {
+			conn, err := mcS.Accept()
+			require.NoError(t, err)
+			go func(sc net.Conn) {
+				var b = make([]byte, 1024)
+				for {
+					n, _ := sc.Read(b)
+					if n > 0 {
+						sc.Write(b[:n])
+					} else {
+						break
+					}
+				}
+			}(conn)
+		}
+	}()
+
+	var wg sync.WaitGroup
+	wg.Add(numSubConn)
+	st := time.Now()
+	for i := 0; i < numSubConn; i++ {
+		conn, err := mcC.Open(ctx)
+		require.NoError(t, err)
+		go func(sc net.Conn) {
+			defer sc.Close()
+			defer wg.Done()
+			for j := 0; j < numWrites; j++ {
+				var b = []byte("some data some data some data some data some data some data some data some data some data")
+				sc.Write(b)
+				sc.Read(b)
+			}
+		}(conn)
+	}
+	wg.Wait()
+	dur := time.Since(st)
+	t.Logf("%.2f req per sec", float64(numWrites*numSubConn)/dur.Seconds())
+}
+
+type fixture struct {
+	*yamuxTransport
+	a            *app.App
+	ctrl         *gomock.Controller
+	mockNodeConf *mock_nodeconf.MockService
+	acc          *accounttest.AccountTestService
+	accepter     *testAccepter
+	addr         string
+}
+
+func newFixture(t *testing.T) *fixture {
+	fx := &fixture{
+		yamuxTransport: New().(*yamuxTransport),
+		ctrl:           gomock.NewController(t),
+		acc:            &accounttest.AccountTestService{},
+		accepter:       &testAccepter{mcs: make(chan transport.MultiConn, 100)},
+		a:              new(app.App),
+	}
+
+	fx.mockNodeConf = mock_nodeconf.NewMockService(fx.ctrl)
+	fx.mockNodeConf.EXPECT().Init(gomock.Any())
+	fx.mockNodeConf.EXPECT().Name().Return(nodeconf.CName).AnyTimes()
+	fx.mockNodeConf.EXPECT().Run(ctx)
+	fx.mockNodeConf.EXPECT().Close(ctx)
+	fx.mockNodeConf.EXPECT().NodeTypes(gomock.Any()).Return([]nodeconf.NodeType{nodeconf.NodeTypeTree}).AnyTimes()
+	fx.a.Register(fx.acc).Register(newTestConf()).Register(fx.mockNodeConf).Register(secureservice.New()).Register(fx.yamuxTransport).Register(fx.accepter)
+	require.NoError(t, fx.a.Start(ctx))
+	fx.addr = fx.listeners[0].Addr().String()
+	return fx
+}
+
+func (fx *fixture) finish(t *testing.T) {
+	require.NoError(t, fx.a.Close(ctx))
+	fx.ctrl.Finish()
+}
+
+func newTestConf() *testConf {
+	return &testConf{testnodeconf.GenNodeConfig(1)}
+}
+
+type testConf struct {
+	*testnodeconf.Config
+}
+
+func (c *testConf) GetYamux() Config {
+	return Config{
+		ListenAddrs:     []string{"127.0.0.1:0"},
+		WriteTimeoutSec: 10,
+		DialTimeoutSec:  10,
+	}
+}
+
+type testAccepter struct {
+	err error
+	mcs chan transport.MultiConn
+}
+
+func (t *testAccepter) Accept(mc transport.MultiConn) (err error) {
+	t.mcs <- mc
+	return t.err
+}
+
+func (t *testAccepter) Init(a *app.App) (err error) {
+	a.MustComponent(CName).(transport.Transport).SetAccepter(t)
+	return nil
+}
+
+func (t *testAccepter) Name() (name string) { return "testAccepter" }