From accf68fc6000d470747d8b15fcb0469c479e2806 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 13 Mar 2023 21:47:07 +0100 Subject: [PATCH 01/27] Start sync tests --- .../object/acl/list/aclrecordbuilder_test.go | 8 +- .../object/tree/objecttree/objecttree_test.go | 142 ++++++------------ .../object/tree/objecttree/testutils.go | 76 ++++++++++ .../object/tree/synctree/syncclient.go | 20 +-- .../object/tree/synctree/syncclient_test.go | 54 +++++++ commonspace/object/tree/synctree/synctree.go | 3 +- 6 files changed, 190 insertions(+), 113 deletions(-) create mode 100644 commonspace/object/tree/objecttree/testutils.go create mode 100644 commonspace/object/tree/synctree/syncclient_test.go diff --git a/commonspace/object/acl/list/aclrecordbuilder_test.go b/commonspace/object/acl/list/aclrecordbuilder_test.go index 5afdb2a8..0a5d3a77 100644 --- a/commonspace/object/acl/list/aclrecordbuilder_test.go +++ b/commonspace/object/acl/list/aclrecordbuilder_test.go @@ -3,7 +3,7 @@ package list import ( "github.com/anytypeio/any-sync/commonspace/object/accountdata" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" - acllistbuilder2 "github.com/anytypeio/any-sync/commonspace/object/acl/testutils/acllistbuilder" + acllistbuilder "github.com/anytypeio/any-sync/commonspace/object/acl/testutils/acllistbuilder" "github.com/anytypeio/any-sync/commonspace/object/keychain" "github.com/anytypeio/any-sync/util/cidutil" "github.com/anytypeio/any-sync/util/keys/asymmetric/signingkey" @@ -12,10 +12,10 @@ import ( ) func TestAclRecordBuilder_BuildUserJoin(t *testing.T) { - st, err := acllistbuilder2.NewListStorageWithTestName("userjoinexample.yml") + st, err := acllistbuilder.NewListStorageWithTestName("userjoinexample.yml") require.NoError(t, err, "building storage should not result in error") - testKeychain := st.(*acllistbuilder2.AclListStorageBuilder).GetKeychain() + testKeychain := st.(*acllistbuilder.AclListStorageBuilder).GetKeychain() identity := testKeychain.GeneratedIdentities["D"] signPrivKey := testKeychain.SigningKeysByYAMLName["D"] encPrivKey := testKeychain.EncryptionKeysByYAMLName["D"] @@ -28,7 +28,7 @@ func TestAclRecordBuilder_BuildUserJoin(t *testing.T) { aclList, err := BuildAclListWithIdentity(acc, st) require.NoError(t, err, "building acl list should be without error") recordBuilder := newAclRecordBuilder(aclList.Id(), keychain.NewKeychain()) - rk, err := testKeychain.GetKey("key.Read.EncKey").(*acllistbuilder2.SymKey).Key.Raw() + rk, err := testKeychain.GetKey("key.Read.EncKey").(*acllistbuilder.SymKey).Key.Raw() require.NoError(t, err) privKey, err := testKeychain.GetKey("key.Sign.Onetime1").(signingkey.PrivKey).Raw() require.NoError(t, err) diff --git a/commonspace/object/tree/objecttree/objecttree_test.go b/commonspace/object/tree/objecttree/objecttree_test.go index ee8989e9..26a8995b 100644 --- a/commonspace/object/tree/objecttree/objecttree_test.go +++ b/commonspace/object/tree/objecttree/objecttree_test.go @@ -11,59 +11,11 @@ import ( "testing" ) -type mockChangeCreator struct{} - -func (c *mockChangeCreator) createRoot(id, aclId string) *treechangeproto.RawTreeChangeWithId { - aclChange := &treechangeproto.RootChange{ - AclHeadId: aclId, - } - res, _ := aclChange.Marshal() - - raw := &treechangeproto.RawTreeChange{ - Payload: res, - Signature: nil, - } - rawMarshalled, _ := raw.Marshal() - - return &treechangeproto.RawTreeChangeWithId{ - RawChange: rawMarshalled, - Id: id, - } -} - -func (c *mockChangeCreator) createRaw(id, aclId, snapshotId string, isSnapshot bool, prevIds ...string) *treechangeproto.RawTreeChangeWithId { - aclChange := &treechangeproto.TreeChange{ - TreeHeadIds: prevIds, - AclHeadId: aclId, - SnapshotBaseId: snapshotId, - ChangesData: nil, - IsSnapshot: isSnapshot, - } - res, _ := aclChange.Marshal() - - raw := &treechangeproto.RawTreeChange{ - Payload: res, - Signature: nil, - } - rawMarshalled, _ := raw.Marshal() - - return &treechangeproto.RawTreeChangeWithId{ - RawChange: rawMarshalled, - Id: id, - } -} - -func (c *mockChangeCreator) createNewTreeStorage(treeId, aclHeadId string) treestorage.TreeStorage { - root := c.createRoot(treeId, aclHeadId) - treeStorage, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root}) - return treeStorage -} - type testTreeContext struct { aclList list.AclList treeStorage treestorage.TreeStorage changeBuilder ChangeBuilder - changeCreator *mockChangeCreator + changeCreator *MockChangeCreator objTree ObjectTree } @@ -77,9 +29,9 @@ func prepareAclList(t *testing.T) list.AclList { return aclList } -func prepareTreeDeps(aclList list.AclList) (*mockChangeCreator, objectTreeDeps) { - changeCreator := &mockChangeCreator{} - treeStorage := changeCreator.createNewTreeStorage("0", aclList.Head().Id) +func prepareTreeDeps(aclList list.AclList) (*MockChangeCreator, objectTreeDeps) { + changeCreator := &MockChangeCreator{} + treeStorage := changeCreator.CreateNewTreeStorage("0", aclList.Head().Id) root, _ := treeStorage.Root() changeBuilder := &nonVerifiableChangeBuilder{ ChangeBuilder: NewChangeBuilder(nil, root), @@ -96,8 +48,8 @@ func prepareTreeDeps(aclList list.AclList) (*mockChangeCreator, objectTreeDeps) } func prepareTreeContext(t *testing.T, aclList list.AclList) testTreeContext { - changeCreator := &mockChangeCreator{} - treeStorage := changeCreator.createNewTreeStorage("0", aclList.Head().Id) + changeCreator := &MockChangeCreator{} + treeStorage := changeCreator.CreateNewTreeStorage("0", aclList.Head().Id) root, _ := treeStorage.Root() changeBuilder := &nonVerifiableChangeBuilder{ ChangeBuilder: NewChangeBuilder(nil, root), @@ -142,8 +94,8 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -187,7 +139,7 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("0", aclList.Head().Id, "", true, ""), + changeCreator.CreateRaw("0", aclList.Head().Id, "", true, ""), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -211,7 +163,7 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -237,10 +189,10 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "3", false, "3"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "3", false, "3"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -287,9 +239,9 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -311,12 +263,12 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } payload := RawChangesPayload{ @@ -390,13 +342,13 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), // main difference from tree example - changeCreator.createRaw("6", aclList.Head().Id, "0", true, "3", "4", "5"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", true, "3", "4", "5"), } payload := RawChangesPayload{ @@ -471,9 +423,9 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -485,9 +437,9 @@ func TestObjectTree(t *testing.T) { require.Equal(t, "3", objTree.Root().Id) rawChanges = []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } payload = RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -531,12 +483,12 @@ func TestObjectTree(t *testing.T) { changeCreator, deps := prepareTreeDeps(aclList) rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } deps.treeStorage.TransactionAdd(rawChanges, []string{"6"}) hTree, err := buildHistoryTree(deps, HistoryTreeParams{ @@ -562,12 +514,12 @@ func TestObjectTree(t *testing.T) { changeCreator, deps := prepareTreeDeps(aclList) rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } deps.treeStorage.TransactionAdd(rawChanges, []string{"6"}) hTree, err := buildHistoryTree(deps, HistoryTreeParams{ diff --git a/commonspace/object/tree/objecttree/testutils.go b/commonspace/object/tree/objecttree/testutils.go new file mode 100644 index 00000000..61cd0fe7 --- /dev/null +++ b/commonspace/object/tree/objecttree/testutils.go @@ -0,0 +1,76 @@ +package objecttree + +import ( + "github.com/anytypeio/any-sync/commonspace/object/acl/list" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" +) + +type MockChangeCreator struct{} + +func NewMockChangeCreator() *MockChangeCreator { + return &MockChangeCreator{} +} + +func (c *MockChangeCreator) CreateRoot(id, aclId string) *treechangeproto.RawTreeChangeWithId { + aclChange := &treechangeproto.RootChange{ + AclHeadId: aclId, + } + res, _ := aclChange.Marshal() + + raw := &treechangeproto.RawTreeChange{ + Payload: res, + Signature: nil, + } + rawMarshalled, _ := raw.Marshal() + + return &treechangeproto.RawTreeChangeWithId{ + RawChange: rawMarshalled, + Id: id, + } +} + +func (c *MockChangeCreator) CreateRaw(id, aclId, snapshotId string, isSnapshot bool, prevIds ...string) *treechangeproto.RawTreeChangeWithId { + aclChange := &treechangeproto.TreeChange{ + TreeHeadIds: prevIds, + AclHeadId: aclId, + SnapshotBaseId: snapshotId, + ChangesData: nil, + IsSnapshot: isSnapshot, + } + res, _ := aclChange.Marshal() + + raw := &treechangeproto.RawTreeChange{ + Payload: res, + Signature: nil, + } + rawMarshalled, _ := raw.Marshal() + + return &treechangeproto.RawTreeChangeWithId{ + RawChange: rawMarshalled, + Id: id, + } +} + +func (c *MockChangeCreator) CreateNewTreeStorage(treeId, aclHeadId string) treestorage.TreeStorage { + root := c.CreateRoot(treeId, aclHeadId) + treeStorage, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root}) + return treeStorage +} + +func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { + root, _ := treeStorage.Root() + changeBuilder := &nonVerifiableChangeBuilder{ + ChangeBuilder: NewChangeBuilder(nil, root), + } + deps := objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: newTreeBuilder(treeStorage, changeBuilder), + treeStorage: treeStorage, + rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), + validator: &noOpTreeValidator{}, + aclList: aclList, + } + + return buildObjectTree(deps) +} diff --git a/commonspace/object/tree/synctree/syncclient.go b/commonspace/object/tree/synctree/syncclient.go index 6ad98c7c..f6e6b990 100644 --- a/commonspace/object/tree/synctree/syncclient.go +++ b/commonspace/object/tree/synctree/syncclient.go @@ -4,9 +4,8 @@ package synctree import ( "context" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/objectsync" + "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" - "github.com/anytypeio/any-sync/nodeconf" ) type SyncClient interface { @@ -16,21 +15,18 @@ type SyncClient interface { } type syncClient struct { - objectsync.MessagePool + peermanager.PeerManager RequestFactory - spaceId string - configuration nodeconf.Configuration + spaceId string } func newSyncClient( spaceId string, - pool objectsync.MessagePool, - factory RequestFactory, - configuration nodeconf.Configuration) SyncClient { + peerManager peermanager.PeerManager, + factory RequestFactory) SyncClient { return &syncClient{ - MessagePool: pool, + PeerManager: peerManager, RequestFactory: factory, - configuration: configuration, spaceId: spaceId, } } @@ -40,7 +36,7 @@ func (s *syncClient) Broadcast(ctx context.Context, msg *treechangeproto.TreeSyn if err != nil { return } - return s.MessagePool.Broadcast(ctx, objMsg) + return s.PeerManager.Broadcast(ctx, objMsg) } func (s *syncClient) SendWithReply(ctx context.Context, peerId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) { @@ -48,7 +44,7 @@ func (s *syncClient) SendWithReply(ctx context.Context, peerId string, msg *tree if err != nil { return } - return s.MessagePool.SendPeer(ctx, peerId, objMsg) + return s.PeerManager.SendPeer(ctx, peerId, objMsg) } func marshallTreeMessage(message *treechangeproto.TreeSyncMessage, spaceId, objectId, replyId string) (objMsg *spacesyncproto.ObjectSyncMessage, err error) { diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go new file mode 100644 index 00000000..cc7f7e18 --- /dev/null +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -0,0 +1,54 @@ +package synctree + +import ( + "context" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" + "github.com/anytypeio/any-sync/commonspace/peermanager" + "github.com/anytypeio/any-sync/commonspace/spacesyncproto" + "github.com/anytypeio/any-sync/commonspace/syncstatus" + "github.com/anytypeio/any-sync/net/peer" +) + +type mockPeerManager struct { +} + +func (m *mockPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { + //TODO implement me + panic("implement me") +} + +func (m *mockPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { + //TODO implement me + panic("implement me") +} + +func (m *mockPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { + //TODO implement me + panic("implement me") +} + +type broadcastTree struct { + objecttree.ObjectTree + SyncClient +} + +func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) (objecttree.AddResult, error) { + res, err := b.ObjectTree.AddRawChanges(ctx, changes) + if err != nil { + return objecttree.AddResult{}, err + } + upd := b.SyncClient.CreateHeadUpdate(b.ObjectTree, res.Added) + b.SyncClient.Broadcast(ctx, upd) + return res, nil +} + +func build(spaceId string, objTree objecttree.ObjectTree, peerManager peermanager.PeerManager) synchandler.SyncHandler { + factory := GetRequestFactory() + syncClient := newSyncClient(spaceId, peerManager, factory) + netTree := &broadcastTree{ + ObjectTree: objTree, + SyncClient: syncClient, + } + return newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) +} diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index 56d07c1c..a1374b48 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -101,8 +101,7 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t Sy syncClient := createSyncClient( deps.SpaceId, deps.ObjectSync.MessagePool(), - sharedFactory, - deps.Configuration) + sharedFactory) syncTree := &syncTree{ ObjectTree: objTree, syncClient: syncClient, From 5f5aefef53627b1db810efa6eb41dac40d14ec04 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 13 Mar 2023 22:03:38 +0100 Subject: [PATCH 02/27] Add test component structure --- .../object/tree/synctree/syncclient_test.go | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index cc7f7e18..14e8cacc 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -2,29 +2,31 @@ package synctree import ( "context" + "github.com/anytypeio/any-sync/commonspace/object/acl/list" + "github.com/anytypeio/any-sync/commonspace/object/acl/testutils/acllistbuilder" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/anytypeio/any-sync/net/peer" + "github.com/stretchr/testify/require" + "testing" ) type mockPeerManager struct { } func (m *mockPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { - //TODO implement me panic("implement me") } func (m *mockPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { - //TODO implement me panic("implement me") } func (m *mockPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { - //TODO implement me panic("implement me") } @@ -43,7 +45,7 @@ func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.Ra return res, nil } -func build(spaceId string, objTree objecttree.ObjectTree, peerManager peermanager.PeerManager) synchandler.SyncHandler { +func createSyncHandler(spaceId string, objTree objecttree.ObjectTree, peerManager peermanager.PeerManager) synchandler.SyncHandler { factory := GetRequestFactory() syncClient := newSyncClient(spaceId, peerManager, factory) netTree := &broadcastTree{ @@ -52,3 +54,33 @@ func build(spaceId string, objTree objecttree.ObjectTree, peerManager peermanage } return newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) } + +func createAclList() (list.AclList, error) { + st, err := acllistbuilder.NewListStorageWithTestName("userjoinexample.yml") + if err != nil { + return nil, err + } + return list.BuildAclList(st) +} + +func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage { + changeCreator := objecttree.NewMockChangeCreator() + st := changeCreator.CreateNewTreeStorage(treeId, aclList.Head().Id) + return st +} + +func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (objecttree.ObjectTree, error) { + return objecttree.BuildTestableTree(aclList, storage) +} + +func TestSyncProtocol(t *testing.T) { + aclList, err := createAclList() + require.NoError(t, err) + treeId := "treeId" + spaceId := "spaceId" + storage := createStorage(treeId, aclList) + testTree, err := createTestTree(aclList, storage) + require.NoError(t, err) + peerManager := &mockPeerManager{} + _ = createSyncHandler(spaceId, testTree, peerManager) +} From 829159690fc4c91c30c5c405afdf66f4fa1c45c2 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 14 Mar 2023 10:33:59 +0100 Subject: [PATCH 03/27] Add more components for async communication --- .../object/tree/synctree/syncclient_test.go | 136 +++++++++++++++--- 1 file changed, 113 insertions(+), 23 deletions(-) diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index 14e8cacc..d7617987 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -2,32 +2,113 @@ package synctree import ( "context" + "fmt" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/acl/testutils/acllistbuilder" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" - "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/anytypeio/any-sync/net/peer" - "github.com/stretchr/testify/require" - "testing" + "github.com/cheggaaa/mb/v3" + "sync" ) -type mockPeerManager struct { +type processMsg struct { + msg *spacesyncproto.ObjectSyncMessage + senderId string + receiverId string } -func (m *mockPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { - panic("implement me") +type messageLog struct { + batcher *mb.MB[processMsg] } -func (m *mockPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { - panic("implement me") +func newMessageLog() *messageLog { + return &messageLog{batcher: mb.New[processMsg](0)} } -func (m *mockPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { - panic("implement me") +func (m *messageLog) addMessage(msg processMsg) { + m.batcher.Add(context.Background(), msg) +} + +type processSyncHandler struct { + synchandler.SyncHandler + batcher *mb.MB[processMsg] + peerId string +} + +func newProcessSyncHandler(peerId string, syncHandler synchandler.SyncHandler) *processSyncHandler { + batcher := mb.New[processMsg](0) + return &processSyncHandler{syncHandler, batcher, peerId} +} + +func (p *processSyncHandler) manager() *processPeerManager { + return p.SyncHandler.(*syncTreeHandler).syncClient.(*syncClient).PeerManager.(*processPeerManager) +} + +func (p *processSyncHandler) send(ctx context.Context, msg processMsg) (err error) { + return p.batcher.Add(ctx, msg) +} + +func (p *processSyncHandler) run(ctx context.Context, wg *sync.WaitGroup) { + wg.Add(1) + go func() { + defer wg.Done() + for { + res, err := p.batcher.WaitOne(ctx) + if err != nil { + return + } + err = p.SyncHandler.HandleMessage(ctx, res.senderId, res.msg) + if err != nil { + fmt.Println("error handling message", err.Error()) + continue + } + } + }() +} + +type processPeerManager struct { + peerId string + handlers map[string]*processSyncHandler + log *messageLog +} + +func newProcessPeerManager(peerId string, log *messageLog) *processPeerManager { + return &processPeerManager{handlers: map[string]*processSyncHandler{}, peerId: peerId, log: log} +} + +func (m *processPeerManager) addHandler(peerId string, handler *processSyncHandler) { + m.handlers[peerId] = handler +} + +func (m *processPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { + pMsg := processMsg{ + msg: msg, + senderId: m.peerId, + receiverId: peerId, + } + m.log.addMessage(pMsg) + return m.handlers[peerId].send(context.Background(), pMsg) +} + +func (m *processPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { + for _, handler := range m.handlers { + pMsg := processMsg{ + msg: msg, + senderId: m.peerId, + receiverId: handler.peerId, + } + m.log.addMessage(pMsg) + handler.send(context.Background(), pMsg) + } + return +} + +func (m *processPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { + panic("should not be called") } type broadcastTree struct { @@ -45,14 +126,15 @@ func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.Ra return res, nil } -func createSyncHandler(spaceId string, objTree objecttree.ObjectTree, peerManager peermanager.PeerManager) synchandler.SyncHandler { +func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *processSyncHandler { factory := GetRequestFactory() - syncClient := newSyncClient(spaceId, peerManager, factory) + syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) netTree := &broadcastTree{ ObjectTree: objTree, SyncClient: syncClient, } - return newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) + handler := newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) + return newProcessSyncHandler(peerId, handler) } func createAclList() (list.AclList, error) { @@ -73,14 +155,22 @@ func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (obje return objecttree.BuildTestableTree(aclList, storage) } -func TestSyncProtocol(t *testing.T) { - aclList, err := createAclList() - require.NoError(t, err) - treeId := "treeId" - spaceId := "spaceId" - storage := createStorage(treeId, aclList) - testTree, err := createTestTree(aclList, storage) - require.NoError(t, err) - peerManager := &mockPeerManager{} - _ = createSyncHandler(spaceId, testTree, peerManager) +type processFixture struct { + handlers []*processSyncHandler + log *messageLog + wg *sync.WaitGroup + ctx context.Context + cancel context.CancelFunc } + +//func TestSyncProtocol(t *testing.T) { +// aclList, err := createAclList() +// require.NoError(t, err) +// treeId := "treeId" +// spaceId := "spaceId" +// storage := createStorage(treeId, aclList) +// testTree, err := createTestTree(aclList, storage) +// require.NoError(t, err) +// peerManager := &processPeerManager{} +// _ = createSyncHandler(spaceId, testTree, peerManager) +//} From 0746526170d491ac8c05963fea05e69c26dc49e5 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 14 Mar 2023 11:41:29 +0100 Subject: [PATCH 04/27] Create fixture draft --- .../object/tree/synctree/syncclient_test.go | 71 ++++++++++++++++++- .../object/tree/treestorage/inmemory.go | 33 +++++---- 2 files changed, 90 insertions(+), 14 deletions(-) diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index d7617987..7a76b71f 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -12,13 +12,16 @@ import ( "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/anytypeio/any-sync/net/peer" "github.com/cheggaaa/mb/v3" + "github.com/stretchr/testify/require" "sync" + "testing" ) type processMsg struct { msg *spacesyncproto.ObjectSyncMessage senderId string receiverId string + userMsg *objecttree.RawChangesPayload } type messageLog struct { @@ -48,11 +51,19 @@ func (p *processSyncHandler) manager() *processPeerManager { return p.SyncHandler.(*syncTreeHandler).syncClient.(*syncClient).PeerManager.(*processPeerManager) } +func (p *processSyncHandler) tree() *broadcastTree { + return p.SyncHandler.(*syncTreeHandler).objTree.(*broadcastTree) +} + func (p *processSyncHandler) send(ctx context.Context, msg processMsg) (err error) { return p.batcher.Add(ctx, msg) } -func (p *processSyncHandler) run(ctx context.Context, wg *sync.WaitGroup) { +func (p *processSyncHandler) sendRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) { + p.batcher.Add(ctx, processMsg{userMsg: &changes}) +} + +func (p *processSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.WaitGroup) { wg.Add(1) go func() { defer wg.Done() @@ -61,6 +72,14 @@ func (p *processSyncHandler) run(ctx context.Context, wg *sync.WaitGroup) { if err != nil { return } + if res.userMsg != nil { + p.tree().Lock() + userRes, err := p.tree().AddRawChanges(ctx, *res.userMsg) + require.NoError(t, err) + fmt.Println("user add result", userRes.Heads) + p.tree().Unlock() + continue + } err = p.SyncHandler.HandleMessage(ctx, res.senderId, res.msg) if err != nil { fmt.Println("error handling message", err.Error()) @@ -155,14 +174,62 @@ func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (obje return objecttree.BuildTestableTree(aclList, storage) } +type fixtureDeps struct { + aclList list.AclList + initStorage *treestorage.InMemoryTreeStorage + connectionMap map[string][]string +} + type processFixture struct { - handlers []*processSyncHandler + handlers map[string]*processSyncHandler log *messageLog wg *sync.WaitGroup ctx context.Context cancel context.CancelFunc } +func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processFixture { + var ( + handlers = map[string]*processSyncHandler{} + log = newMessageLog() + wg = sync.WaitGroup{} + ctx, cancel = context.WithCancel(context.Background()) + ) + + for peerId := range deps.connectionMap { + stCopy := deps.initStorage.Copy() + testTree, err := createTestTree(deps.aclList, stCopy) + require.NoError(t, err) + handler := createSyncHandler(peerId, spaceId, testTree, log) + handlers[peerId] = handler + } + for peerId, connectionMap := range deps.connectionMap { + handler := handlers[peerId] + manager := handler.manager() + for _, connectionId := range connectionMap { + manager.addHandler(connectionId, handlers[connectionId]) + } + } + return &processFixture{ + handlers: handlers, + log: log, + wg: &wg, + ctx: ctx, + cancel: cancel, + } +} + +func (p *processFixture) runProcessFixture(t *testing.T) { + for _, handler := range p.handlers { + handler.run(p.ctx, t, p.wg) + } +} + +func (p *processFixture) stop() { + p.cancel() + p.wg.Wait() +} + //func TestSyncProtocol(t *testing.T) { // aclList, err := createAclList() // require.NoError(t, err) diff --git a/commonspace/object/tree/treestorage/inmemory.go b/commonspace/object/tree/treestorage/inmemory.go index c5c5a5cc..0f22037e 100644 --- a/commonspace/object/tree/treestorage/inmemory.go +++ b/commonspace/object/tree/treestorage/inmemory.go @@ -7,7 +7,7 @@ import ( "sync" ) -type inMemoryTreeStorage struct { +type InMemoryTreeStorage struct { id string root *treechangeproto.RawTreeChangeWithId heads []string @@ -16,7 +16,7 @@ type inMemoryTreeStorage struct { sync.RWMutex } -func (t *inMemoryTreeStorage) TransactionAdd(changes []*treechangeproto.RawTreeChangeWithId, heads []string) error { +func (t *InMemoryTreeStorage) TransactionAdd(changes []*treechangeproto.RawTreeChangeWithId, heads []string) error { t.RLock() defer t.RUnlock() @@ -37,46 +37,46 @@ func NewInMemoryTreeStorage( } allChanges[root.Id] = root - return &inMemoryTreeStorage{ + return &InMemoryTreeStorage{ id: root.Id, root: root, - heads: heads, + heads: append([]string(nil), heads...), changes: allChanges, RWMutex: sync.RWMutex{}, }, nil } -func (t *inMemoryTreeStorage) HasChange(ctx context.Context, id string) (bool, error) { +func (t *InMemoryTreeStorage) HasChange(ctx context.Context, id string) (bool, error) { _, exists := t.changes[id] return exists, nil } -func (t *inMemoryTreeStorage) Id() string { +func (t *InMemoryTreeStorage) Id() string { t.RLock() defer t.RUnlock() return t.id } -func (t *inMemoryTreeStorage) Root() (*treechangeproto.RawTreeChangeWithId, error) { +func (t *InMemoryTreeStorage) Root() (*treechangeproto.RawTreeChangeWithId, error) { t.RLock() defer t.RUnlock() return t.root, nil } -func (t *inMemoryTreeStorage) Heads() ([]string, error) { +func (t *InMemoryTreeStorage) Heads() ([]string, error) { t.RLock() defer t.RUnlock() return t.heads, nil } -func (t *inMemoryTreeStorage) SetHeads(heads []string) error { +func (t *InMemoryTreeStorage) SetHeads(heads []string) error { t.Lock() defer t.Unlock() t.heads = append(t.heads[:0], heads...) return nil } -func (t *inMemoryTreeStorage) AddRawChange(change *treechangeproto.RawTreeChangeWithId) error { +func (t *InMemoryTreeStorage) AddRawChange(change *treechangeproto.RawTreeChangeWithId) error { t.Lock() defer t.Unlock() // TODO: better to do deep copy @@ -84,7 +84,7 @@ func (t *inMemoryTreeStorage) AddRawChange(change *treechangeproto.RawTreeChange return nil } -func (t *inMemoryTreeStorage) GetRawChange(ctx context.Context, changeId string) (*treechangeproto.RawTreeChangeWithId, error) { +func (t *InMemoryTreeStorage) GetRawChange(ctx context.Context, changeId string) (*treechangeproto.RawTreeChangeWithId, error) { t.RLock() defer t.RUnlock() if res, exists := t.changes[changeId]; exists { @@ -93,6 +93,15 @@ func (t *inMemoryTreeStorage) GetRawChange(ctx context.Context, changeId string) return nil, fmt.Errorf("could not get change with id: %s", changeId) } -func (t *inMemoryTreeStorage) Delete() error { +func (t *InMemoryTreeStorage) Delete() error { return nil } + +func (t *InMemoryTreeStorage) Copy() *InMemoryTreeStorage { + var changes []*treechangeproto.RawTreeChangeWithId + for _, ch := range t.changes { + changes = append(changes, ch) + } + other, _ := NewInMemoryTreeStorage(t.root, t.heads, changes) + return other.(*InMemoryTreeStorage) +} From afad73fe8bb08e5d8b1cc15d17d49dce88b88c9c Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 20 Mar 2023 16:00:44 +0100 Subject: [PATCH 05/27] Add simple test --- .../object/tree/synctree/syncclient_test.go | 54 ++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index 7a76b71f..c1f4fccb 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -6,6 +6,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/acl/testutils/acllistbuilder" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" @@ -13,8 +14,10 @@ import ( "github.com/anytypeio/any-sync/net/peer" "github.com/cheggaaa/mb/v3" "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" "sync" "testing" + "time" ) type processMsg struct { @@ -219,7 +222,7 @@ func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processF } } -func (p *processFixture) runProcessFixture(t *testing.T) { +func (p *processFixture) run(t *testing.T) { for _, handler := range p.handlers { handler.run(p.ctx, t, p.wg) } @@ -230,14 +233,41 @@ func (p *processFixture) stop() { p.wg.Wait() } -//func TestSyncProtocol(t *testing.T) { -// aclList, err := createAclList() -// require.NoError(t, err) -// treeId := "treeId" -// spaceId := "spaceId" -// storage := createStorage(treeId, aclList) -// testTree, err := createTestTree(aclList, storage) -// require.NoError(t, err) -// peerManager := &processPeerManager{} -// _ = createSyncHandler(spaceId, testTree, peerManager) -//} +func TestSimple(t *testing.T) { + aclList, err := createAclList() + require.NoError(t, err) + treeId := "treeId" + spaceId := "spaceId" + storage := createStorage(treeId, aclList) + changeCreator := objecttree.NewMockChangeCreator() + deps := fixtureDeps{ + aclList: aclList, + initStorage: storage.(*treestorage.InMemoryTreeStorage), + connectionMap: map[string][]string{ + "peer1": []string{"peer2"}, + "peer2": []string{"peer1"}, + }, + } + fx := newProcessFixture(t, spaceId, deps) + fx.run(t) + fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: nil, + RawChanges: []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("1", aclList.Id(), treeId, false, treeId), + }, + }) + fx.handlers["peer2"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: nil, + RawChanges: []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("2", aclList.Id(), treeId, false, treeId), + }, + }) + time.Sleep(100 * time.Millisecond) + fx.stop() + firstHeads := fx.handlers["peer1"].tree().Heads() + secondHeads := fx.handlers["peer2"].tree().Heads() + slices.Sort(firstHeads) + slices.Sort(secondHeads) + require.Equal(t, firstHeads, secondHeads) + require.Equal(t, []string{"1", "2"}, firstHeads) +} From 0a1dc26a9506f2d8ea1218615b2bc920fea62f34 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 20 Mar 2023 16:24:07 +0100 Subject: [PATCH 06/27] Add second test and description --- .../object/tree/synctree/syncclient_test.go | 85 ++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index c1f4fccb..06c96c25 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -13,6 +13,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/anytypeio/any-sync/net/peer" "github.com/cheggaaa/mb/v3" + "github.com/gogo/protobuf/proto" "github.com/stretchr/testify/require" "golang.org/x/exp/slices" "sync" @@ -27,6 +28,40 @@ type processMsg struct { userMsg *objecttree.RawChangesPayload } +type msgDescription struct { + name string + from string + to string + heads []string +} + +func (p *processMsg) description() (descr msgDescription) { + unmarshalled := &treechangeproto.TreeSyncMessage{} + err := proto.Unmarshal(p.msg.Payload, unmarshalled) + if err != nil { + panic(err) + } + descr = msgDescription{ + from: p.senderId, + to: p.receiverId, + } + switch { + case unmarshalled.GetContent().GetHeadUpdate() != nil: + cnt := unmarshalled.GetContent().GetHeadUpdate() + descr.name = "HeadUpdate" + descr.heads = cnt.Heads + case unmarshalled.GetContent().GetFullSyncRequest() != nil: + cnt := unmarshalled.GetContent().GetFullSyncRequest() + descr.name = "FullSyncRequest" + descr.heads = cnt.Heads + case unmarshalled.GetContent().GetFullSyncResponse() != nil: + cnt := unmarshalled.GetContent().GetFullSyncResponse() + descr.name = "FullSyncResponse" + descr.heads = cnt.Heads + } + return +} + type messageLog struct { batcher *mb.MB[processMsg] } @@ -233,7 +268,7 @@ func (p *processFixture) stop() { p.wg.Wait() } -func TestSimple(t *testing.T) { +func TestSimple_TwoPeers(t *testing.T) { aclList, err := createAclList() require.NoError(t, err) treeId := "treeId" @@ -270,4 +305,52 @@ func TestSimple(t *testing.T) { slices.Sort(secondHeads) require.Equal(t, firstHeads, secondHeads) require.Equal(t, []string{"1", "2"}, firstHeads) + logMsgs := fx.log.batcher.GetAll() + for _, msg := range logMsgs { + fmt.Println(msg.description()) + } +} + +func TestSimple_ThreePeers(t *testing.T) { + aclList, err := createAclList() + require.NoError(t, err) + treeId := "treeId" + spaceId := "spaceId" + storage := createStorage(treeId, aclList) + changeCreator := objecttree.NewMockChangeCreator() + deps := fixtureDeps{ + aclList: aclList, + initStorage: storage.(*treestorage.InMemoryTreeStorage), + connectionMap: map[string][]string{ + "peer1": []string{"node1"}, + "peer2": []string{"node1"}, + "node1": []string{"peer1", "peer2"}, + }, + } + fx := newProcessFixture(t, spaceId, deps) + fx.run(t) + fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: nil, + RawChanges: []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("1", aclList.Id(), treeId, false, treeId), + }, + }) + fx.handlers["peer2"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: nil, + RawChanges: []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("2", aclList.Id(), treeId, false, treeId), + }, + }) + time.Sleep(100 * time.Millisecond) + fx.stop() + firstHeads := fx.handlers["peer1"].tree().Heads() + secondHeads := fx.handlers["peer2"].tree().Heads() + slices.Sort(firstHeads) + slices.Sort(secondHeads) + require.Equal(t, firstHeads, secondHeads) + require.Equal(t, []string{"1", "2"}, firstHeads) + logMsgs := fx.log.batcher.GetAll() + for _, msg := range logMsgs { + fmt.Println(msg.description()) + } } From e101389af072038743936e96afebd35eec4f1ba1 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 13 Apr 2023 21:32:39 +0200 Subject: [PATCH 07/27] Add empty client logic --- .../object/tree/synctree/syncclient_test.go | 120 ++++++++++++++++-- 1 file changed, 111 insertions(+), 9 deletions(-) diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index 7feb15d5..086cb883 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -76,17 +76,59 @@ func (m *messageLog) addMessage(msg processMsg) { type processSyncHandler struct { synchandler.SyncHandler - batcher *mb.MB[processMsg] - peerId string + batcher *mb.MB[processMsg] + peerId string + aclList list.AclList + log *messageLog + syncClient SyncClient +} + +func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { + if p.SyncHandler != nil { + return p.SyncHandler.HandleMessage(ctx, senderId, request) + } + unmarshalled := &treechangeproto.TreeSyncMessage{} + err = proto.Unmarshal(request.Payload, unmarshalled) + if err != nil { + return + } + if unmarshalled.Content.GetFullSyncResponse() == nil { + newTreeRequest := GetRequestFactory().CreateNewTreeRequest() + var objMsg *spacesyncproto.ObjectSyncMessage + objMsg, err = marshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") + if err != nil { + return + } + return p.manager().SendPeer(context.Background(), senderId, objMsg) + } + fullSyncResponse := unmarshalled.Content.GetFullSyncResponse() + treeStorage, _ := treestorage.NewInMemoryTreeStorage(unmarshalled.RootChange, fullSyncResponse.Heads, fullSyncResponse.Changes) + tree, err := createTestTree(p.aclList, treeStorage) + if err != nil { + return + } + netTree := &broadcastTree{ + ObjectTree: tree, + SyncClient: p.syncClient, + } + p.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, p.syncClient, syncstatus.NewNoOpSyncStatus()) + return } func newProcessSyncHandler(peerId string, syncHandler synchandler.SyncHandler) *processSyncHandler { batcher := mb.New[processMsg](0) - return &processSyncHandler{syncHandler, batcher, peerId} + return &processSyncHandler{ + SyncHandler: syncHandler, + batcher: batcher, + peerId: peerId, + } } func (p *processSyncHandler) manager() *processPeerManager { - return p.SyncHandler.(*syncTreeHandler).syncClient.(*syncClient).PeerManager.(*processPeerManager) + if p.SyncHandler != nil { + return p.SyncHandler.(*syncTreeHandler).syncClient.(*syncClient).PeerManager.(*processPeerManager) + } + return p.syncClient.(*syncClient).PeerManager.(*processPeerManager) } func (p *processSyncHandler) tree() *broadcastTree { @@ -118,7 +160,7 @@ func (p *processSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.Wai p.tree().Unlock() continue } - err = p.SyncHandler.HandleMessage(ctx, res.senderId, res.msg) + err = p.HandleMessage(ctx, res.senderId, res.msg) if err != nil { fmt.Println("error handling message", err.Error()) continue @@ -194,6 +236,20 @@ func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, lo return newProcessSyncHandler(peerId, handler) } +func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *processSyncHandler { + factory := GetRequestFactory() + syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) + + batcher := mb.New[processMsg](0) + return &processSyncHandler{ + batcher: batcher, + peerId: peerId, + aclList: aclList, + log: log, + syncClient: syncClient, + } +} + func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage { changeCreator := objecttree.NewMockChangeCreator() st := changeCreator.CreateNewTreeStorage(treeId, aclList.Head().Id) @@ -208,6 +264,7 @@ type fixtureDeps struct { aclList list.AclList initStorage *treestorage.InMemoryTreeStorage connectionMap map[string][]string + emptyTrees []string } type processFixture struct { @@ -227,10 +284,15 @@ func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processF ) for peerId := range deps.connectionMap { - stCopy := deps.initStorage.Copy() - testTree, err := createTestTree(deps.aclList, stCopy) - require.NoError(t, err) - handler := createSyncHandler(peerId, spaceId, testTree, log) + var handler *processSyncHandler + if slices.Contains(deps.emptyTrees, peerId) { + handler = createEmptySyncHandler(peerId, spaceId, deps.aclList, log) + } else { + stCopy := deps.initStorage.Copy() + testTree, err := createTestTree(deps.aclList, stCopy) + require.NoError(t, err) + handler = createSyncHandler(peerId, spaceId, testTree, log) + } handlers[peerId] = handler } for peerId, connectionMap := range deps.connectionMap { @@ -260,6 +322,46 @@ func (p *processFixture) stop() { p.wg.Wait() } +func TestSend_EmptyClient(t *testing.T) { + treeId := "treeId" + spaceId := "spaceId" + keys, err := accountdata.NewRandom() + require.NoError(t, err) + aclList, err := list.NewTestDerivedAcl(spaceId, keys) + require.NoError(t, err) + storage := createStorage(treeId, aclList) + changeCreator := objecttree.NewMockChangeCreator() + deps := fixtureDeps{ + aclList: aclList, + initStorage: storage.(*treestorage.InMemoryTreeStorage), + connectionMap: map[string][]string{ + "peer1": []string{"peer2"}, + "peer2": []string{"peer1"}, + }, + emptyTrees: []string{"peer2"}, + } + fx := newProcessFixture(t, spaceId, deps) + fx.run(t) + fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: nil, + RawChanges: []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("1", aclList.Id(), treeId, false, treeId), + }, + }) + time.Sleep(100 * time.Millisecond) + fx.stop() + firstHeads := fx.handlers["peer1"].tree().Heads() + secondHeads := fx.handlers["peer2"].tree().Heads() + slices.Sort(firstHeads) + slices.Sort(secondHeads) + require.Equal(t, firstHeads, secondHeads) + require.Equal(t, []string{"1"}, firstHeads) + logMsgs := fx.log.batcher.GetAll() + for _, msg := range logMsgs { + fmt.Println(msg.description()) + } +} + func TestSimple_TwoPeers(t *testing.T) { treeId := "treeId" spaceId := "spaceId" From 673925df84b70c722b8f11dbaf8237c7230b19e5 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 13 Apr 2023 23:55:28 +0200 Subject: [PATCH 08/27] Improve validator logic and add tests --- .../object/tree/objecttree/objecttree_test.go | 224 +++++++++--------- .../tree/objecttree/objecttreefactory.go | 6 +- .../tree/objecttree/objecttreevalidator.go | 20 +- .../object/tree/objecttree/testutils.go | 56 ++++- .../object/tree/synctree/syncclient_test.go | 26 +- 5 files changed, 199 insertions(+), 133 deletions(-) diff --git a/commonspace/object/tree/objecttree/objecttree_test.go b/commonspace/object/tree/objecttree/objecttree_test.go index 3ecd33ff..7ec9fac9 100644 --- a/commonspace/object/tree/objecttree/objecttree_test.go +++ b/commonspace/object/tree/objecttree/objecttree_test.go @@ -11,59 +11,10 @@ import ( "testing" ) -type mockChangeCreator struct{} - -func (c *mockChangeCreator) createRoot(id, aclId string) *treechangeproto.RawTreeChangeWithId { - aclChange := &treechangeproto.RootChange{ - AclHeadId: aclId, - } - res, _ := aclChange.Marshal() - - raw := &treechangeproto.RawTreeChange{ - Payload: res, - Signature: nil, - } - rawMarshalled, _ := raw.Marshal() - - return &treechangeproto.RawTreeChangeWithId{ - RawChange: rawMarshalled, - Id: id, - } -} - -func (c *mockChangeCreator) createRaw(id, aclId, snapshotId string, isSnapshot bool, prevIds ...string) *treechangeproto.RawTreeChangeWithId { - aclChange := &treechangeproto.TreeChange{ - TreeHeadIds: prevIds, - AclHeadId: aclId, - SnapshotBaseId: snapshotId, - ChangesData: nil, - IsSnapshot: isSnapshot, - } - res, _ := aclChange.Marshal() - - raw := &treechangeproto.RawTreeChange{ - Payload: res, - Signature: nil, - } - rawMarshalled, _ := raw.Marshal() - - return &treechangeproto.RawTreeChangeWithId{ - RawChange: rawMarshalled, - Id: id, - } -} - -func (c *mockChangeCreator) createNewTreeStorage(treeId, aclHeadId string) treestorage.TreeStorage { - root := c.createRoot(treeId, aclHeadId) - treeStorage, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root}) - return treeStorage -} - type testTreeContext struct { aclList list.AclList treeStorage treestorage.TreeStorage - changeBuilder ChangeBuilder - changeCreator *mockChangeCreator + changeCreator *MockChangeCreator objTree ObjectTree } @@ -76,12 +27,12 @@ func prepareAclList(t *testing.T) list.AclList { return aclList } -func prepareTreeDeps(aclList list.AclList) (*mockChangeCreator, objectTreeDeps) { - changeCreator := &mockChangeCreator{} - treeStorage := changeCreator.createNewTreeStorage("0", aclList.Head().Id) +func prepareTreeDeps(aclList list.AclList) (*MockChangeCreator, objectTreeDeps) { + changeCreator := NewMockChangeCreator() + treeStorage := changeCreator.CreateNewTreeStorage("0", aclList.Head().Id) root, _ := treeStorage.Root() changeBuilder := &nonVerifiableChangeBuilder{ - ChangeBuilder: NewChangeBuilder(newKeyStorage(), root), + ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root), } deps := objectTreeDeps{ changeBuilder: changeBuilder, @@ -95,23 +46,9 @@ func prepareTreeDeps(aclList list.AclList) (*mockChangeCreator, objectTreeDeps) } func prepareTreeContext(t *testing.T, aclList list.AclList) testTreeContext { - changeCreator := &mockChangeCreator{} - treeStorage := changeCreator.createNewTreeStorage("0", aclList.Head().Id) - root, _ := treeStorage.Root() - changeBuilder := &nonVerifiableChangeBuilder{ - ChangeBuilder: NewChangeBuilder(newKeyStorage(), root), - } - deps := objectTreeDeps{ - changeBuilder: changeBuilder, - treeBuilder: newTreeBuilder(treeStorage, changeBuilder), - treeStorage: treeStorage, - rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), - validator: &noOpTreeValidator{}, - aclList: aclList, - } - - // check build - objTree, err := buildObjectTree(deps) + changeCreator := NewMockChangeCreator() + treeStorage := changeCreator.CreateNewTreeStorage("0", aclList.Head().Id) + objTree, err := BuildTestableTree(aclList, treeStorage) require.NoError(t, err, "building tree should be without error") // check tree iterate @@ -125,7 +62,6 @@ func prepareTreeContext(t *testing.T, aclList list.AclList) testTreeContext { return testTreeContext{ aclList: aclList, treeStorage: treeStorage, - changeBuilder: changeBuilder, changeCreator: changeCreator, objTree: objTree, } @@ -141,8 +77,8 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -186,7 +122,7 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("0", aclList.Head().Id, "", true, ""), + changeCreator.CreateRaw("0", aclList.Head().Id, "", true, ""), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -210,7 +146,33 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + } + payload := RawChangesPayload{ + NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, + RawChanges: rawChanges, + } + res, err := objTree.AddRawChanges(context.Background(), payload) + require.NoError(t, err, "adding changes should be without error") + + // check result + assert.Equal(t, []string{"0"}, res.OldHeads) + assert.Equal(t, []string{"0"}, res.Heads) + assert.Equal(t, 0, len(res.Added)) + assert.Equal(t, Nothing, res.Mode) + + // check tree heads + assert.Equal(t, []string{"0"}, objTree.Heads()) + }) + + t.Run("add not connected changes", func(t *testing.T) { + ctx := prepareTreeContext(t, aclList) + changeCreator := ctx.changeCreator + objTree := ctx.objTree + + // this change could in theory replace current snapshot, we should prevent that + rawChanges := []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("2", aclList.Head().Id, "0", true, "1"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -236,10 +198,10 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "3", false, "3"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "3", false, "3"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -286,9 +248,9 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -310,12 +272,12 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } payload := RawChangesPayload{ @@ -389,13 +351,13 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), // main difference from tree example - changeCreator.createRaw("6", aclList.Head().Id, "0", true, "3", "4", "5"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", true, "3", "4", "5"), } payload := RawChangesPayload{ @@ -470,9 +432,9 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -484,9 +446,9 @@ func TestObjectTree(t *testing.T) { require.Equal(t, "3", objTree.Root().Id) rawChanges = []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } payload = RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -530,12 +492,12 @@ func TestObjectTree(t *testing.T) { changeCreator, deps := prepareTreeDeps(aclList) rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } deps.treeStorage.TransactionAdd(rawChanges, []string{"6"}) hTree, err := buildHistoryTree(deps, HistoryTreeParams{ @@ -561,12 +523,12 @@ func TestObjectTree(t *testing.T) { changeCreator, deps := prepareTreeDeps(aclList) rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } deps.treeStorage.TransactionAdd(rawChanges, []string{"6"}) hTree, err := buildHistoryTree(deps, HistoryTreeParams{ @@ -608,4 +570,40 @@ func TestObjectTree(t *testing.T) { assert.Equal(t, []string{"0"}, iterChangesId) assert.Equal(t, "0", hTree.Root().Id) }) + + t.Run("validate correct tree", func(t *testing.T) { + ctx := prepareTreeContext(t, aclList) + changeCreator := ctx.changeCreator + + rawChanges := []*treechangeproto.RawTreeChangeWithId{ + ctx.objTree.Header(), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + } + defaultObjectTreeDeps = nonVerifiableTreeDeps + err := ValidateRawTree(treestorage.TreeStorageCreatePayload{ + RootRawChange: ctx.objTree.Header(), + Heads: []string{"3"}, + Changes: rawChanges, + }, ctx.aclList) + require.NoError(t, err) + }) + + t.Run("fail to validate not connected tree", func(t *testing.T) { + ctx := prepareTreeContext(t, aclList) + changeCreator := ctx.changeCreator + + rawChanges := []*treechangeproto.RawTreeChangeWithId{ + ctx.objTree.Header(), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + } + defaultObjectTreeDeps = nonVerifiableTreeDeps + err := ValidateRawTree(treestorage.TreeStorageCreatePayload{ + RootRawChange: ctx.objTree.Header(), + Heads: []string{"3"}, + Changes: rawChanges, + }, ctx.aclList) + require.Equal(t, ErrHasInvalidChanges, err) + }) } diff --git a/commonspace/object/tree/objecttree/objecttreefactory.go b/commonspace/object/tree/objecttree/objecttreefactory.go index b3b757fb..0ab51ec1 100644 --- a/commonspace/object/tree/objecttree/objecttreefactory.go +++ b/commonspace/object/tree/objecttree/objecttreefactory.go @@ -33,7 +33,9 @@ type objectTreeDeps struct { aclList list.AclList } -func defaultObjectTreeDeps( +var defaultObjectTreeDeps = verifiableTreeDeps + +func verifiableTreeDeps( rootChange *treechangeproto.RawTreeChangeWithId, treeStorage treestorage.TreeStorage, aclList list.AclList) objectTreeDeps { @@ -53,7 +55,7 @@ func nonVerifiableTreeDeps( rootChange *treechangeproto.RawTreeChangeWithId, treeStorage treestorage.TreeStorage, aclList list.AclList) objectTreeDeps { - changeBuilder := &nonVerifiableChangeBuilder{NewChangeBuilder(nil, rootChange)} + changeBuilder := &nonVerifiableChangeBuilder{NewChangeBuilder(newMockKeyStorage(), rootChange)} treeBuilder := newTreeBuilder(treeStorage, changeBuilder) return objectTreeDeps{ changeBuilder: changeBuilder, diff --git a/commonspace/object/tree/objecttree/objecttreevalidator.go b/commonspace/object/tree/objecttree/objecttreevalidator.go index 937dcc96..e2816246 100644 --- a/commonspace/object/tree/objecttree/objecttreevalidator.go +++ b/commonspace/object/tree/objecttree/objecttreevalidator.go @@ -1,10 +1,12 @@ package objecttree import ( + "context" "fmt" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/util/slice" ) type ObjectTreeValidator interface { @@ -88,11 +90,23 @@ func (v *objectTreeValidator) validateChange(tree *Tree, aclList list.AclList, c } func ValidateRawTree(payload treestorage.TreeStorageCreatePayload, aclList list.AclList) (err error) { - treeStorage, err := treestorage.NewInMemoryTreeStorage(payload.RootRawChange, payload.Heads, payload.Changes) + treeStorage, err := treestorage.NewInMemoryTreeStorage(payload.RootRawChange, []string{payload.RootRawChange.Id}, nil) if err != nil { return } - - _, err = BuildObjectTree(treeStorage, aclList) + tree, err := BuildObjectTree(treeStorage, aclList) + if err != nil { + return + } + res, err := tree.AddRawChanges(context.Background(), RawChangesPayload{ + NewHeads: payload.Heads, + RawChanges: payload.Changes, + }) + if err != nil { + return + } + if !slice.UnsortedEquals(res.Heads, payload.Heads) { + return ErrHasInvalidChanges + } return } diff --git a/commonspace/object/tree/objecttree/testutils.go b/commonspace/object/tree/objecttree/testutils.go index 9fbeed5f..6385b9f7 100644 --- a/commonspace/object/tree/objecttree/testutils.go +++ b/commonspace/object/tree/objecttree/testutils.go @@ -1,24 +1,64 @@ package objecttree import ( - "crypto/rand" + "fmt" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/util/crypto" + libcrypto "github.com/libp2p/go-libp2p/core/crypto" ) -type mockKeyStorage struct { - key crypto.PubKey +type mockPubKey struct { } -func newKeyStorage() mockKeyStorage { - _, pk, _ := crypto.GenerateEd25519Key(rand.Reader) - return mockKeyStorage{pk} +const mockKeyValue = "mockKey" + +func (m mockPubKey) Equals(key crypto.Key) bool { + return true +} + +func (m mockPubKey) Raw() ([]byte, error) { + return []byte(mockKeyValue), nil +} + +func (m mockPubKey) Encrypt(message []byte) ([]byte, error) { + return message, nil +} + +func (m mockPubKey) Verify(data []byte, sig []byte) (bool, error) { + return true, nil +} + +func (m mockPubKey) Marshall() ([]byte, error) { + return []byte(mockKeyValue), nil +} + +func (m mockPubKey) Storage() []byte { + return []byte(mockKeyValue) +} + +func (m mockPubKey) Account() string { + return mockKeyValue +} + +func (m mockPubKey) PeerId() string { + return mockKeyValue +} + +func (m mockPubKey) LibP2P() (libcrypto.PubKey, error) { + return nil, fmt.Errorf("can't be converted in libp2p") +} + +type mockKeyStorage struct { +} + +func newMockKeyStorage() mockKeyStorage { + return mockKeyStorage{} } func (m mockKeyStorage) PubKeyFromProto(protoBytes []byte) (crypto.PubKey, error) { - return m.key, nil + return mockPubKey{}, nil } type MockChangeCreator struct{} @@ -76,7 +116,7 @@ func (c *MockChangeCreator) CreateNewTreeStorage(treeId, aclHeadId string) trees func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { root, _ := treeStorage.Root() changeBuilder := &nonVerifiableChangeBuilder{ - ChangeBuilder: NewChangeBuilder(newKeyStorage(), root), + ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root), } deps := objectTreeDeps{ changeBuilder: changeBuilder, diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index 086cb883..73db3f42 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -29,10 +29,11 @@ type processMsg struct { } type msgDescription struct { - name string - from string - to string - heads []string + name string + from string + to string + heads []string + changes []*treechangeproto.RawTreeChangeWithId } func (p *processMsg) description() (descr msgDescription) { @@ -50,14 +51,17 @@ func (p *processMsg) description() (descr msgDescription) { cnt := unmarshalled.GetContent().GetHeadUpdate() descr.name = "HeadUpdate" descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetHeadUpdate().Changes case unmarshalled.GetContent().GetFullSyncRequest() != nil: cnt := unmarshalled.GetContent().GetFullSyncRequest() descr.name = "FullSyncRequest" descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetFullSyncRequest().Changes case unmarshalled.GetContent().GetFullSyncResponse() != nil: cnt := unmarshalled.GetContent().GetFullSyncResponse() descr.name = "FullSyncResponse" descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetFullSyncResponse().Changes } return } @@ -322,7 +326,7 @@ func (p *processFixture) stop() { p.wg.Wait() } -func TestSend_EmptyClient(t *testing.T) { +func TestSend_EmptyClientGetsFullHistory(t *testing.T) { treeId := "treeId" spaceId := "spaceId" keys, err := accountdata.NewRandom() @@ -345,7 +349,7 @@ func TestSend_EmptyClient(t *testing.T) { fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ NewHeads: nil, RawChanges: []*treechangeproto.RawTreeChangeWithId{ - changeCreator.CreateRaw("1", aclList.Id(), treeId, false, treeId), + changeCreator.CreateRaw("1", aclList.Id(), treeId, true, treeId), }, }) time.Sleep(100 * time.Millisecond) @@ -357,9 +361,17 @@ func TestSend_EmptyClient(t *testing.T) { require.Equal(t, firstHeads, secondHeads) require.Equal(t, []string{"1"}, firstHeads) logMsgs := fx.log.batcher.GetAll() + + var fullResponseMsg *processMsg for _, msg := range logMsgs { - fmt.Println(msg.description()) + descr := msg.description() + if descr.name == "FullSyncResponse" { + fullResponseMsg = &msg + } } + require.NotNil(t, fullResponseMsg) + // that means that we got not only the last snapshot, but also the first one + require.Len(t, fullResponseMsg.description().changes, 2) } func TestSimple_TwoPeers(t *testing.T) { From 125d8b46269fd16dae53dac63d62bbef604026a1 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 17 Apr 2023 21:49:51 +0200 Subject: [PATCH 09/27] Add test with random trees --- .../object/tree/synctree/syncclient_test.go | 441 +++-------------- .../object/tree/synctree/synctreehandler.go | 22 +- .../object/tree/synctree/utils_test.go | 446 ++++++++++++++++++ 3 files changed, 518 insertions(+), 391 deletions(-) create mode 100644 commonspace/object/tree/synctree/utils_test.go diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index 73db3f42..ed6dd710 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -8,325 +8,14 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" - "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" - "github.com/anytypeio/any-sync/commonspace/spacesyncproto" - "github.com/anytypeio/any-sync/commonspace/syncstatus" - "github.com/anytypeio/any-sync/net/peer" - "github.com/cheggaaa/mb/v3" - "github.com/gogo/protobuf/proto" + "github.com/anytypeio/any-sync/util/slice" "github.com/stretchr/testify/require" - "golang.org/x/exp/slices" - "sync" + "math/rand" "testing" "time" ) -type processMsg struct { - msg *spacesyncproto.ObjectSyncMessage - senderId string - receiverId string - userMsg *objecttree.RawChangesPayload -} - -type msgDescription struct { - name string - from string - to string - heads []string - changes []*treechangeproto.RawTreeChangeWithId -} - -func (p *processMsg) description() (descr msgDescription) { - unmarshalled := &treechangeproto.TreeSyncMessage{} - err := proto.Unmarshal(p.msg.Payload, unmarshalled) - if err != nil { - panic(err) - } - descr = msgDescription{ - from: p.senderId, - to: p.receiverId, - } - switch { - case unmarshalled.GetContent().GetHeadUpdate() != nil: - cnt := unmarshalled.GetContent().GetHeadUpdate() - descr.name = "HeadUpdate" - descr.heads = cnt.Heads - descr.changes = unmarshalled.GetContent().GetHeadUpdate().Changes - case unmarshalled.GetContent().GetFullSyncRequest() != nil: - cnt := unmarshalled.GetContent().GetFullSyncRequest() - descr.name = "FullSyncRequest" - descr.heads = cnt.Heads - descr.changes = unmarshalled.GetContent().GetFullSyncRequest().Changes - case unmarshalled.GetContent().GetFullSyncResponse() != nil: - cnt := unmarshalled.GetContent().GetFullSyncResponse() - descr.name = "FullSyncResponse" - descr.heads = cnt.Heads - descr.changes = unmarshalled.GetContent().GetFullSyncResponse().Changes - } - return -} - -type messageLog struct { - batcher *mb.MB[processMsg] -} - -func newMessageLog() *messageLog { - return &messageLog{batcher: mb.New[processMsg](0)} -} - -func (m *messageLog) addMessage(msg processMsg) { - m.batcher.Add(context.Background(), msg) -} - -type processSyncHandler struct { - synchandler.SyncHandler - batcher *mb.MB[processMsg] - peerId string - aclList list.AclList - log *messageLog - syncClient SyncClient -} - -func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { - if p.SyncHandler != nil { - return p.SyncHandler.HandleMessage(ctx, senderId, request) - } - unmarshalled := &treechangeproto.TreeSyncMessage{} - err = proto.Unmarshal(request.Payload, unmarshalled) - if err != nil { - return - } - if unmarshalled.Content.GetFullSyncResponse() == nil { - newTreeRequest := GetRequestFactory().CreateNewTreeRequest() - var objMsg *spacesyncproto.ObjectSyncMessage - objMsg, err = marshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") - if err != nil { - return - } - return p.manager().SendPeer(context.Background(), senderId, objMsg) - } - fullSyncResponse := unmarshalled.Content.GetFullSyncResponse() - treeStorage, _ := treestorage.NewInMemoryTreeStorage(unmarshalled.RootChange, fullSyncResponse.Heads, fullSyncResponse.Changes) - tree, err := createTestTree(p.aclList, treeStorage) - if err != nil { - return - } - netTree := &broadcastTree{ - ObjectTree: tree, - SyncClient: p.syncClient, - } - p.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, p.syncClient, syncstatus.NewNoOpSyncStatus()) - return -} - -func newProcessSyncHandler(peerId string, syncHandler synchandler.SyncHandler) *processSyncHandler { - batcher := mb.New[processMsg](0) - return &processSyncHandler{ - SyncHandler: syncHandler, - batcher: batcher, - peerId: peerId, - } -} - -func (p *processSyncHandler) manager() *processPeerManager { - if p.SyncHandler != nil { - return p.SyncHandler.(*syncTreeHandler).syncClient.(*syncClient).PeerManager.(*processPeerManager) - } - return p.syncClient.(*syncClient).PeerManager.(*processPeerManager) -} - -func (p *processSyncHandler) tree() *broadcastTree { - return p.SyncHandler.(*syncTreeHandler).objTree.(*broadcastTree) -} - -func (p *processSyncHandler) send(ctx context.Context, msg processMsg) (err error) { - return p.batcher.Add(ctx, msg) -} - -func (p *processSyncHandler) sendRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) { - p.batcher.Add(ctx, processMsg{userMsg: &changes}) -} - -func (p *processSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.WaitGroup) { - wg.Add(1) - go func() { - defer wg.Done() - for { - res, err := p.batcher.WaitOne(ctx) - if err != nil { - return - } - if res.userMsg != nil { - p.tree().Lock() - userRes, err := p.tree().AddRawChanges(ctx, *res.userMsg) - require.NoError(t, err) - fmt.Println("user add result", userRes.Heads) - p.tree().Unlock() - continue - } - err = p.HandleMessage(ctx, res.senderId, res.msg) - if err != nil { - fmt.Println("error handling message", err.Error()) - continue - } - } - }() -} - -type processPeerManager struct { - peerId string - handlers map[string]*processSyncHandler - log *messageLog -} - -func newProcessPeerManager(peerId string, log *messageLog) *processPeerManager { - return &processPeerManager{handlers: map[string]*processSyncHandler{}, peerId: peerId, log: log} -} - -func (m *processPeerManager) addHandler(peerId string, handler *processSyncHandler) { - m.handlers[peerId] = handler -} - -func (m *processPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { - pMsg := processMsg{ - msg: msg, - senderId: m.peerId, - receiverId: peerId, - } - m.log.addMessage(pMsg) - return m.handlers[peerId].send(context.Background(), pMsg) -} - -func (m *processPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { - for _, handler := range m.handlers { - pMsg := processMsg{ - msg: msg, - senderId: m.peerId, - receiverId: handler.peerId, - } - m.log.addMessage(pMsg) - handler.send(context.Background(), pMsg) - } - return -} - -func (m *processPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { - panic("should not be called") -} - -type broadcastTree struct { - objecttree.ObjectTree - SyncClient -} - -func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) (objecttree.AddResult, error) { - res, err := b.ObjectTree.AddRawChanges(ctx, changes) - if err != nil { - return objecttree.AddResult{}, err - } - upd := b.SyncClient.CreateHeadUpdate(b.ObjectTree, res.Added) - b.SyncClient.Broadcast(ctx, upd) - return res, nil -} - -func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *processSyncHandler { - factory := GetRequestFactory() - syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) - netTree := &broadcastTree{ - ObjectTree: objTree, - SyncClient: syncClient, - } - handler := newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) - return newProcessSyncHandler(peerId, handler) -} - -func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *processSyncHandler { - factory := GetRequestFactory() - syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) - - batcher := mb.New[processMsg](0) - return &processSyncHandler{ - batcher: batcher, - peerId: peerId, - aclList: aclList, - log: log, - syncClient: syncClient, - } -} - -func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage { - changeCreator := objecttree.NewMockChangeCreator() - st := changeCreator.CreateNewTreeStorage(treeId, aclList.Head().Id) - return st -} - -func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (objecttree.ObjectTree, error) { - return objecttree.BuildTestableTree(aclList, storage) -} - -type fixtureDeps struct { - aclList list.AclList - initStorage *treestorage.InMemoryTreeStorage - connectionMap map[string][]string - emptyTrees []string -} - -type processFixture struct { - handlers map[string]*processSyncHandler - log *messageLog - wg *sync.WaitGroup - ctx context.Context - cancel context.CancelFunc -} - -func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processFixture { - var ( - handlers = map[string]*processSyncHandler{} - log = newMessageLog() - wg = sync.WaitGroup{} - ctx, cancel = context.WithCancel(context.Background()) - ) - - for peerId := range deps.connectionMap { - var handler *processSyncHandler - if slices.Contains(deps.emptyTrees, peerId) { - handler = createEmptySyncHandler(peerId, spaceId, deps.aclList, log) - } else { - stCopy := deps.initStorage.Copy() - testTree, err := createTestTree(deps.aclList, stCopy) - require.NoError(t, err) - handler = createSyncHandler(peerId, spaceId, testTree, log) - } - handlers[peerId] = handler - } - for peerId, connectionMap := range deps.connectionMap { - handler := handlers[peerId] - manager := handler.manager() - for _, connectionId := range connectionMap { - manager.addHandler(connectionId, handlers[connectionId]) - } - } - return &processFixture{ - handlers: handlers, - log: log, - wg: &wg, - ctx: ctx, - cancel: cancel, - } -} - -func (p *processFixture) run(t *testing.T) { - for _, handler := range p.handlers { - handler.run(p.ctx, t, p.wg) - } -} - -func (p *processFixture) stop() { - p.cancel() - p.wg.Wait() -} - -func TestSend_EmptyClientGetsFullHistory(t *testing.T) { +func TestEmptyClientGetsFullHistory(t *testing.T) { treeId := "treeId" spaceId := "spaceId" keys, err := accountdata.NewRandom() @@ -356,70 +45,22 @@ func TestSend_EmptyClientGetsFullHistory(t *testing.T) { fx.stop() firstHeads := fx.handlers["peer1"].tree().Heads() secondHeads := fx.handlers["peer2"].tree().Heads() - slices.Sort(firstHeads) - slices.Sort(secondHeads) - require.Equal(t, firstHeads, secondHeads) + require.True(t, slice.SortedEquals(firstHeads, secondHeads)) require.Equal(t, []string{"1"}, firstHeads) logMsgs := fx.log.batcher.GetAll() - var fullResponseMsg *processMsg + var fullResponseMsg msgDescription for _, msg := range logMsgs { descr := msg.description() if descr.name == "FullSyncResponse" { - fullResponseMsg = &msg + fullResponseMsg = descr } } - require.NotNil(t, fullResponseMsg) // that means that we got not only the last snapshot, but also the first one - require.Len(t, fullResponseMsg.description().changes, 2) + require.Len(t, fullResponseMsg.changes, 2) } -func TestSimple_TwoPeers(t *testing.T) { - treeId := "treeId" - spaceId := "spaceId" - keys, err := accountdata.NewRandom() - require.NoError(t, err) - aclList, err := list.NewTestDerivedAcl(spaceId, keys) - require.NoError(t, err) - storage := createStorage(treeId, aclList) - changeCreator := objecttree.NewMockChangeCreator() - deps := fixtureDeps{ - aclList: aclList, - initStorage: storage.(*treestorage.InMemoryTreeStorage), - connectionMap: map[string][]string{ - "peer1": []string{"peer2"}, - "peer2": []string{"peer1"}, - }, - } - fx := newProcessFixture(t, spaceId, deps) - fx.run(t) - fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ - NewHeads: nil, - RawChanges: []*treechangeproto.RawTreeChangeWithId{ - changeCreator.CreateRaw("1", aclList.Id(), treeId, false, treeId), - }, - }) - fx.handlers["peer2"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ - NewHeads: nil, - RawChanges: []*treechangeproto.RawTreeChangeWithId{ - changeCreator.CreateRaw("2", aclList.Id(), treeId, false, treeId), - }, - }) - time.Sleep(100 * time.Millisecond) - fx.stop() - firstHeads := fx.handlers["peer1"].tree().Heads() - secondHeads := fx.handlers["peer2"].tree().Heads() - slices.Sort(firstHeads) - slices.Sort(secondHeads) - require.Equal(t, firstHeads, secondHeads) - require.Equal(t, []string{"1", "2"}, firstHeads) - logMsgs := fx.log.batcher.GetAll() - for _, msg := range logMsgs { - fmt.Println(msg.description()) - } -} - -func TestSimple_ThreePeers(t *testing.T) { +func TestRandomTreeMerge(t *testing.T) { treeId := "treeId" spaceId := "spaceId" keys, err := accountdata.NewRandom() @@ -427,6 +68,21 @@ func TestSimple_ThreePeers(t *testing.T) { aclList, err := list.NewTestDerivedAcl(spaceId, keys) storage := createStorage(treeId, aclList) changeCreator := objecttree.NewMockChangeCreator() + rnd := rand.New(rand.NewSource(time.Now().Unix())) + params := genParams{ + prefix: "peer1", + aclId: aclList.Id(), + startIdx: 0, + levels: 10, + snapshotId: treeId, + prevHeads: []string{treeId}, + isSnapshot: func() bool { + return rnd.Intn(100) > 80 + }, + } + initialRes := genChanges(changeCreator, params) + err = storage.TransactionAdd(initialRes.changes, initialRes.heads) + require.NoError(t, err) deps := fixtureDeps{ aclList: aclList, initStorage: storage.(*treestorage.InMemoryTreeStorage), @@ -435,31 +91,44 @@ func TestSimple_ThreePeers(t *testing.T) { "peer2": []string{"node1"}, "node1": []string{"peer1", "peer2"}, }, + emptyTrees: []string{"peer2", "node1"}, } fx := newProcessFixture(t, spaceId, deps) fx.run(t) fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ - NewHeads: nil, - RawChanges: []*treechangeproto.RawTreeChangeWithId{ - changeCreator.CreateRaw("1", aclList.Id(), treeId, false, treeId), - }, + NewHeads: initialRes.heads, + RawChanges: initialRes.changes, }) - fx.handlers["peer2"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ - NewHeads: nil, - RawChanges: []*treechangeproto.RawTreeChangeWithId{ - changeCreator.CreateRaw("2", aclList.Id(), treeId, false, treeId), - }, - }) - time.Sleep(100 * time.Millisecond) - fx.stop() + time.Sleep(1000 * time.Millisecond) firstHeads := fx.handlers["peer1"].tree().Heads() secondHeads := fx.handlers["peer2"].tree().Heads() - slices.Sort(firstHeads) - slices.Sort(secondHeads) - require.Equal(t, firstHeads, secondHeads) - require.Equal(t, []string{"1", "2"}, firstHeads) - logMsgs := fx.log.batcher.GetAll() - for _, msg := range logMsgs { - fmt.Println(msg.description()) + require.True(t, slice.UnsortedEquals(firstHeads, secondHeads)) + params = genParams{ + prefix: "peer1", + aclId: aclList.Id(), + startIdx: 11, + levels: 10, + snapshotId: initialRes.snapshotId, + prevHeads: initialRes.heads, + isSnapshot: func() bool { + return rnd.Intn(100) > 80 + }, } + peer1Res := genChanges(changeCreator, params) + params.prefix = "peer2" + peer2Res := genChanges(changeCreator, params) + fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: peer1Res.heads, + RawChanges: peer1Res.changes, + }) + fx.handlers["peer2"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: peer2Res.heads, + RawChanges: peer2Res.changes, + }) + time.Sleep(1000 * time.Millisecond) + fx.stop() + firstHeads = fx.handlers["peer1"].tree().Heads() + secondHeads = fx.handlers["peer2"].tree().Heads() + fmt.Println(firstHeads) + fmt.Println(secondHeads) } diff --git a/commonspace/object/tree/synctree/synctreehandler.go b/commonspace/object/tree/synctree/synctreehandler.go index de79d0de..a41a2bad 100644 --- a/commonspace/object/tree/synctree/synctreehandler.go +++ b/commonspace/object/tree/synctree/synctreehandler.go @@ -83,13 +83,19 @@ func (s *syncTreeHandler) handleHeadUpdate( objTree = s.objTree ) - log := log.With(zap.Strings("heads", objTree.Heads()), zap.String("treeId", objTree.Id()), zap.String("spaceId", s.spaceId)) + log := log.With( + zap.Strings("update heads", update.Heads), + zap.String("treeId", objTree.Id()), + zap.String("spaceId", s.spaceId), + zap.Int("len(update changes)", len(update.Changes))) log.DebugCtx(ctx, "received head update message") defer func() { if err != nil { log.With(zap.Error(err)).Debug("head update finished with error") } else if fullRequest != nil { + cnt := fullRequest.Content.GetFullSyncRequest() + log = log.With(zap.Strings("request heads", cnt.Heads), zap.Int("len(request changes)", len(cnt.Changes))) log.DebugCtx(ctx, "sending full sync request") } else { if !isEmptyUpdate { @@ -151,19 +157,21 @@ func (s *syncTreeHandler) handleFullSyncRequest( ) log := log.With(zap.String("senderId", senderId), - zap.Strings("heads", request.Heads), + zap.Strings("request heads", request.Heads), zap.String("treeId", s.objTree.Id()), zap.String("replyId", replyId), - zap.String("spaceId", s.spaceId)) + zap.String("spaceId", s.spaceId), + zap.Int("len(request changes)", len(request.Changes))) log.DebugCtx(ctx, "received full sync request message") defer func() { if err != nil { log.With(zap.Error(err)).DebugCtx(ctx, "full sync request finished with error") - s.syncClient.SendWithReply(ctx, senderId, treechangeproto.WrapError(err, header), replyId) return } else if fullResponse != nil { + cnt := fullResponse.Content.GetFullSyncResponse() + log = log.With(zap.Strings("response heads", cnt.Heads), zap.Int("len(response changes)", len(cnt.Changes))) log.DebugCtx(ctx, "full sync response sent") } }() @@ -192,7 +200,11 @@ func (s *syncTreeHandler) handleFullSyncResponse( var ( objTree = s.objTree ) - log := log.With(zap.Strings("heads", response.Heads), zap.String("treeId", s.objTree.Id()), zap.String("spaceId", s.spaceId)) + log := log.With( + zap.Strings("heads", response.Heads), + zap.String("treeId", s.objTree.Id()), + zap.String("spaceId", s.spaceId), + zap.Int("len(changes)", len(response.Changes))) log.DebugCtx(ctx, "received full sync response message") defer func() { diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go new file mode 100644 index 00000000..0399d239 --- /dev/null +++ b/commonspace/object/tree/synctree/utils_test.go @@ -0,0 +1,446 @@ +package synctree + +import ( + "context" + "fmt" + "github.com/anytypeio/any-sync/commonspace/object/accountdata" + "github.com/anytypeio/any-sync/commonspace/object/acl/list" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" + "github.com/anytypeio/any-sync/commonspace/spacesyncproto" + "github.com/anytypeio/any-sync/commonspace/syncstatus" + "github.com/anytypeio/any-sync/net/peer" + "github.com/cheggaaa/mb/v3" + "github.com/gogo/protobuf/proto" + "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" + "math/rand" + "sync" + "testing" + "time" +) + +type processMsg struct { + msg *spacesyncproto.ObjectSyncMessage + senderId string + receiverId string + userMsg *objecttree.RawChangesPayload +} + +type msgDescription struct { + name string + from string + to string + heads []string + changes []*treechangeproto.RawTreeChangeWithId +} + +func (p *processMsg) description() (descr msgDescription) { + unmarshalled := &treechangeproto.TreeSyncMessage{} + err := proto.Unmarshal(p.msg.Payload, unmarshalled) + if err != nil { + panic(err) + } + descr = msgDescription{ + from: p.senderId, + to: p.receiverId, + } + switch { + case unmarshalled.GetContent().GetHeadUpdate() != nil: + cnt := unmarshalled.GetContent().GetHeadUpdate() + descr.name = "HeadUpdate" + descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetHeadUpdate().Changes + case unmarshalled.GetContent().GetFullSyncRequest() != nil: + cnt := unmarshalled.GetContent().GetFullSyncRequest() + descr.name = "FullSyncRequest" + descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetFullSyncRequest().Changes + case unmarshalled.GetContent().GetFullSyncResponse() != nil: + cnt := unmarshalled.GetContent().GetFullSyncResponse() + descr.name = "FullSyncResponse" + descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetFullSyncResponse().Changes + } + return +} + +type messageLog struct { + batcher *mb.MB[processMsg] +} + +func newMessageLog() *messageLog { + return &messageLog{batcher: mb.New[processMsg](0)} +} + +func (m *messageLog) addMessage(msg processMsg) { + m.batcher.Add(context.Background(), msg) +} + +type processSyncHandler struct { + synchandler.SyncHandler + batcher *mb.MB[processMsg] + peerId string + aclList list.AclList + log *messageLog + syncClient SyncClient +} + +func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { + if p.SyncHandler != nil { + return p.SyncHandler.HandleMessage(ctx, senderId, request) + } + unmarshalled := &treechangeproto.TreeSyncMessage{} + err = proto.Unmarshal(request.Payload, unmarshalled) + if err != nil { + return + } + if unmarshalled.Content.GetFullSyncResponse() == nil { + newTreeRequest := GetRequestFactory().CreateNewTreeRequest() + var objMsg *spacesyncproto.ObjectSyncMessage + objMsg, err = marshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") + if err != nil { + return + } + return p.manager().SendPeer(context.Background(), senderId, objMsg) + } + fullSyncResponse := unmarshalled.Content.GetFullSyncResponse() + treeStorage, _ := treestorage.NewInMemoryTreeStorage(unmarshalled.RootChange, []string{unmarshalled.RootChange.Id}, nil) + tree, err := createTestTree(p.aclList, treeStorage) + if err != nil { + return + } + netTree := &broadcastTree{ + ObjectTree: tree, + SyncClient: p.syncClient, + } + res, err := netTree.AddRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: fullSyncResponse.Heads, + RawChanges: fullSyncResponse.Changes, + }) + if err != nil { + return + } + p.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, p.syncClient, syncstatus.NewNoOpSyncStatus()) + var objMsg *spacesyncproto.ObjectSyncMessage + newTreeRequest := GetRequestFactory().CreateHeadUpdate(netTree, res.Added) + objMsg, err = marshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") + if err != nil { + return + } + return p.manager().Broadcast(context.Background(), objMsg) +} + +func newProcessSyncHandler(peerId string, syncHandler synchandler.SyncHandler) *processSyncHandler { + batcher := mb.New[processMsg](0) + return &processSyncHandler{ + SyncHandler: syncHandler, + batcher: batcher, + peerId: peerId, + } +} + +func (p *processSyncHandler) manager() *processPeerManager { + if p.SyncHandler != nil { + return p.SyncHandler.(*syncTreeHandler).syncClient.(*syncClient).PeerManager.(*processPeerManager) + } + return p.syncClient.(*syncClient).PeerManager.(*processPeerManager) +} + +func (p *processSyncHandler) tree() *broadcastTree { + return p.SyncHandler.(*syncTreeHandler).objTree.(*broadcastTree) +} + +func (p *processSyncHandler) send(ctx context.Context, msg processMsg) (err error) { + return p.batcher.Add(ctx, msg) +} + +func (p *processSyncHandler) sendRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) { + p.batcher.Add(ctx, processMsg{userMsg: &changes}) +} + +func (p *processSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.WaitGroup) { + wg.Add(1) + go func() { + defer wg.Done() + for { + res, err := p.batcher.WaitOne(ctx) + if err != nil { + return + } + if res.userMsg != nil { + p.tree().Lock() + userRes, err := p.tree().AddRawChanges(ctx, *res.userMsg) + require.NoError(t, err) + fmt.Println("user add result", userRes.Heads) + p.tree().Unlock() + continue + } + err = p.HandleMessage(ctx, res.senderId, res.msg) + if err != nil { + fmt.Println("error handling message", err.Error()) + continue + } + } + }() +} + +type processPeerManager struct { + peerId string + handlers map[string]*processSyncHandler + log *messageLog +} + +func newProcessPeerManager(peerId string, log *messageLog) *processPeerManager { + return &processPeerManager{handlers: map[string]*processSyncHandler{}, peerId: peerId, log: log} +} + +func (m *processPeerManager) addHandler(peerId string, handler *processSyncHandler) { + m.handlers[peerId] = handler +} + +func (m *processPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { + pMsg := processMsg{ + msg: msg, + senderId: m.peerId, + receiverId: peerId, + } + m.log.addMessage(pMsg) + return m.handlers[peerId].send(context.Background(), pMsg) +} + +func (m *processPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { + for _, handler := range m.handlers { + pMsg := processMsg{ + msg: msg, + senderId: m.peerId, + receiverId: handler.peerId, + } + m.log.addMessage(pMsg) + handler.send(context.Background(), pMsg) + } + return +} + +func (m *processPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { + panic("should not be called") +} + +type broadcastTree struct { + objecttree.ObjectTree + SyncClient +} + +func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) (objecttree.AddResult, error) { + res, err := b.ObjectTree.AddRawChanges(ctx, changes) + if err != nil { + return objecttree.AddResult{}, err + } + upd := b.SyncClient.CreateHeadUpdate(b.ObjectTree, res.Added) + b.SyncClient.Broadcast(ctx, upd) + return res, nil +} + +func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *processSyncHandler { + factory := GetRequestFactory() + syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) + netTree := &broadcastTree{ + ObjectTree: objTree, + SyncClient: syncClient, + } + handler := newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) + return newProcessSyncHandler(peerId, handler) +} + +func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *processSyncHandler { + factory := GetRequestFactory() + syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) + + batcher := mb.New[processMsg](0) + return &processSyncHandler{ + batcher: batcher, + peerId: peerId, + aclList: aclList, + log: log, + syncClient: syncClient, + } +} + +func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage { + changeCreator := objecttree.NewMockChangeCreator() + st := changeCreator.CreateNewTreeStorage(treeId, aclList.Head().Id) + return st +} + +func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (objecttree.ObjectTree, error) { + return objecttree.BuildTestableTree(aclList, storage) +} + +type fixtureDeps struct { + aclList list.AclList + initStorage *treestorage.InMemoryTreeStorage + connectionMap map[string][]string + emptyTrees []string +} + +type processFixture struct { + handlers map[string]*processSyncHandler + log *messageLog + wg *sync.WaitGroup + ctx context.Context + cancel context.CancelFunc +} + +func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processFixture { + var ( + handlers = map[string]*processSyncHandler{} + log = newMessageLog() + wg = sync.WaitGroup{} + ctx, cancel = context.WithCancel(context.Background()) + ) + + for peerId := range deps.connectionMap { + var handler *processSyncHandler + if slices.Contains(deps.emptyTrees, peerId) { + handler = createEmptySyncHandler(peerId, spaceId, deps.aclList, log) + } else { + stCopy := deps.initStorage.Copy() + testTree, err := createTestTree(deps.aclList, stCopy) + require.NoError(t, err) + handler = createSyncHandler(peerId, spaceId, testTree, log) + } + handlers[peerId] = handler + } + for peerId, connectionMap := range deps.connectionMap { + handler := handlers[peerId] + manager := handler.manager() + for _, connectionId := range connectionMap { + manager.addHandler(connectionId, handlers[connectionId]) + } + } + return &processFixture{ + handlers: handlers, + log: log, + wg: &wg, + ctx: ctx, + cancel: cancel, + } +} + +func (p *processFixture) run(t *testing.T) { + for _, handler := range p.handlers { + handler.run(p.ctx, t, p.wg) + } +} + +func (p *processFixture) stop() { + p.cancel() + p.wg.Wait() +} + +type genParams struct { + prefix string + aclId string + startIdx int + levels int + snapshotId string + prevHeads []string + isSnapshot func() bool +} + +type genResult struct { + changes []*treechangeproto.RawTreeChangeWithId + heads []string + snapshotId string +} + +func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res genResult) { + src := rand.NewSource(time.Now().Unix()) + rnd := rand.New(src) + var ( + prevHeads []string + snapshotId = params.snapshotId + ) + prevHeads = append(prevHeads, params.prevHeads...) + + for i := 0; i < params.levels; i++ { + if params.isSnapshot() { + newId := fmt.Sprintf("%s.%d.%d", params.prefix, params.startIdx+i, 0) + newCh := creator.CreateRaw(newId, params.aclId, snapshotId, true, prevHeads...) + res.changes = append(res.changes, newCh) + prevHeads = []string{newId} + snapshotId = newId + continue + } + perLevel := rnd.Intn(10) + if perLevel == 0 { + perLevel = 1 + } + var ( + newHeads []string + usedIds = map[string]struct{}{} + ) + for j := 0; j < perLevel; j++ { + // if we didn't connect with all prev ones + prevConns := rnd.Intn(len(prevHeads)) + if prevConns == 0 { + prevConns = 1 + } + rnd.Shuffle(len(prevHeads), func(i, j int) { + prevHeads[i], prevHeads[j] = prevHeads[j], prevHeads[i] + }) + if j == perLevel-1 && len(usedIds) != len(prevHeads) { + var unusedIds []string + for _, id := range prevHeads { + if _, exists := usedIds[id]; !exists { + unusedIds = append(unusedIds, id) + } + } + prevHeads = unusedIds + prevConns = len(prevHeads) + } + var prevChId []string + for k := 0; k < prevConns; k++ { + prevChId = append(prevChId, prevHeads[k]) + usedIds[prevHeads[k]] = struct{}{} + } + newId := fmt.Sprintf("%s.%d.%d", params.prefix, params.startIdx+i, j) + newCh := creator.CreateRaw(newId, params.aclId, snapshotId, false, prevChId...) + res.changes = append(res.changes, newCh) + newHeads = append(newHeads, newId) + } + prevHeads = newHeads + } + res.heads = prevHeads + res.snapshotId = snapshotId + return +} + +func TestGenChanges(t *testing.T) { + treeId := "treeId" + spaceId := "spaceId" + keys, err := accountdata.NewRandom() + require.NoError(t, err) + aclList, err := list.NewTestDerivedAcl(spaceId, keys) + storage := createStorage(treeId, aclList) + creator := objecttree.NewMockChangeCreator() + rnd := rand.New(rand.NewSource(time.Now().Unix())) + params := genParams{ + prefix: "peerId", + aclId: aclList.Id(), + startIdx: 0, + levels: 10, + snapshotId: treeId, + prevHeads: []string{treeId}, + isSnapshot: func() bool { + return rnd.Intn(100) > 80 + }, + } + res := genChanges(creator, params) + storage.TransactionAdd(res.changes, res.heads) + tr, err := createTestTree(aclList, storage) + require.NoError(t, err) + fmt.Println(tr.Debug(objecttree.NoOpDescriptionParser)) +} From 678ec256bf8347e45651b90a2adea113fe42b6d0 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 17 Apr 2023 23:51:17 +0200 Subject: [PATCH 10/27] Add tests and don't load data into changes --- .../object/tree/objecttree/changebuilder.go | 33 ++++++++++- .../object/tree/objecttree/objecttree.go | 14 +---- .../tree/objecttree/objecttreefactory.go | 59 +++++++++++++++++++ .../object/tree/objecttree/rawloader.go | 23 ++++++-- .../object/tree/objecttree/testutils.go | 18 ------ ...yncclient_test.go => syncprotocol_test.go} | 59 ++++++++++++------- .../object/tree/synctree/utils_test.go | 35 ++--------- .../object/tree/treestorage/inmemory.go | 19 ++++++ 8 files changed, 171 insertions(+), 89 deletions(-) rename commonspace/object/tree/synctree/{syncclient_test.go => syncprotocol_test.go} (75%) diff --git a/commonspace/object/tree/objecttree/changebuilder.go b/commonspace/object/tree/objecttree/changebuilder.go index cb22d249..2f10fca9 100644 --- a/commonspace/object/tree/objecttree/changebuilder.go +++ b/commonspace/object/tree/objecttree/changebuilder.go @@ -52,6 +52,18 @@ func (c *nonVerifiableChangeBuilder) Marshall(ch *Change) (raw *treechangeproto. return c.ChangeBuilder.Marshall(ch) } +type emptyDataChangeBuilder struct { + ChangeBuilder +} + +func (c *emptyDataChangeBuilder) Build(payload BuilderContent) (ch *Change, raw *treechangeproto.RawTreeChangeWithId, err error) { + panic("should not be called") +} + +func (c *emptyDataChangeBuilder) Marshall(ch *Change) (raw *treechangeproto.RawTreeChangeWithId, err error) { + panic("should not be called") +} + type ChangeBuilder interface { Unmarshall(rawIdChange *treechangeproto.RawTreeChangeWithId, verify bool) (ch *Change, err error) Build(payload BuilderContent) (ch *Change, raw *treechangeproto.RawTreeChangeWithId, err error) @@ -59,13 +71,28 @@ type ChangeBuilder interface { Marshall(ch *Change) (*treechangeproto.RawTreeChangeWithId, error) } +type newChangeFunc = func(id string, identity crypto.PubKey, ch *treechangeproto.TreeChange, signature []byte) *Change + type changeBuilder struct { rootChange *treechangeproto.RawTreeChangeWithId keys crypto.KeyStorage + newChange newChangeFunc +} + +func NewEmptyDataBuilder(keys crypto.KeyStorage, rootChange *treechangeproto.RawTreeChangeWithId) ChangeBuilder { + return &emptyDataChangeBuilder{&changeBuilder{ + rootChange: rootChange, + keys: keys, + newChange: func(id string, identity crypto.PubKey, ch *treechangeproto.TreeChange, signature []byte) *Change { + c := NewChange(id, identity, ch, signature) + c.Data = nil + return c + }, + }} } func NewChangeBuilder(keys crypto.KeyStorage, rootChange *treechangeproto.RawTreeChangeWithId) ChangeBuilder { - return &changeBuilder{keys: keys, rootChange: rootChange} + return &changeBuilder{keys: keys, rootChange: rootChange, newChange: NewChange} } func (c *changeBuilder) Unmarshall(rawIdChange *treechangeproto.RawTreeChangeWithId, verify bool) (ch *Change, err error) { @@ -197,7 +224,7 @@ func (c *changeBuilder) Build(payload BuilderContent) (ch *Change, rawIdChange * if err != nil { return } - ch = NewChange(id, payload.PrivKey.GetPublic(), change, signature) + ch = c.newChange(id, payload.PrivKey.GetPublic(), change, signature) rawIdChange = &treechangeproto.RawTreeChangeWithId{ RawChange: marshalledRawChange, Id: id, @@ -268,7 +295,7 @@ func (c *changeBuilder) unmarshallRawChange(raw *treechangeproto.RawTreeChange, if err != nil { return } - ch = NewChange(id, key, unmarshalled, raw.Signature) + ch = c.newChange(id, key, unmarshalled, raw.Signature) return } diff --git a/commonspace/object/tree/objecttree/objecttree.go b/commonspace/object/tree/objecttree/objecttree.go index 430329c2..5f2d98e4 100644 --- a/commonspace/object/tree/objecttree/objecttree.go +++ b/commonspace/object/tree/objecttree/objecttree.go @@ -621,19 +621,7 @@ func (ot *objectTree) ChangesAfterCommonSnapshot(theirPath, theirHeads []string) } } - if commonSnapshot == ot.tree.RootId() { - return ot.getChangesFromTree(theirHeads) - } else { - return ot.getChangesFromDB(commonSnapshot, theirHeads) - } -} - -func (ot *objectTree) getChangesFromTree(theirHeads []string) (rawChanges []*treechangeproto.RawTreeChangeWithId, err error) { - return ot.rawChangeLoader.LoadFromTree(ot.tree, theirHeads) -} - -func (ot *objectTree) getChangesFromDB(commonSnapshot string, theirHeads []string) (rawChanges []*treechangeproto.RawTreeChangeWithId, err error) { - return ot.rawChangeLoader.LoadFromStorage(commonSnapshot, ot.tree.headIds, theirHeads) + return ot.rawChangeLoader.Load(commonSnapshot, ot.tree, theirHeads) } func (ot *objectTree) snapshotPathIsActual() bool { diff --git a/commonspace/object/tree/objecttree/objecttreefactory.go b/commonspace/object/tree/objecttree/objecttreefactory.go index 0ab51ec1..5b9196e4 100644 --- a/commonspace/object/tree/objecttree/objecttreefactory.go +++ b/commonspace/object/tree/objecttree/objecttreefactory.go @@ -51,6 +51,22 @@ func verifiableTreeDeps( } } +func emptyDataTreeDeps( + rootChange *treechangeproto.RawTreeChangeWithId, + treeStorage treestorage.TreeStorage, + aclList list.AclList) objectTreeDeps { + changeBuilder := NewEmptyDataBuilder(crypto.NewKeyStorage(), rootChange) + treeBuilder := newTreeBuilder(treeStorage, changeBuilder) + return objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: treeBuilder, + treeStorage: treeStorage, + validator: newTreeValidator(), + rawChangeLoader: newStorageLoader(treeStorage, changeBuilder), + aclList: aclList, + } +} + func nonVerifiableTreeDeps( rootChange *treechangeproto.RawTreeChangeWithId, treeStorage treestorage.TreeStorage, @@ -80,6 +96,49 @@ func DeriveObjectTreeRoot(payload ObjectTreeCreatePayload, aclList list.AclList) return createObjectTreeRoot(payload, 0, nil, aclList) } +func BuildEmptyDataObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) { + rootChange, err := treeStorage.Root() + if err != nil { + return nil, err + } + deps := emptyDataTreeDeps(rootChange, treeStorage, aclList) + return buildObjectTree(deps) +} + +func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { + root, _ := treeStorage.Root() + changeBuilder := &nonVerifiableChangeBuilder{ + ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root), + } + deps := objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: newTreeBuilder(treeStorage, changeBuilder), + treeStorage: treeStorage, + rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), + validator: &noOpTreeValidator{}, + aclList: aclList, + } + + return buildObjectTree(deps) +} + +func BuildEmptyDataTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { + root, _ := treeStorage.Root() + changeBuilder := &nonVerifiableChangeBuilder{ + ChangeBuilder: NewEmptyDataBuilder(newMockKeyStorage(), root), + } + deps := objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: newTreeBuilder(treeStorage, changeBuilder), + treeStorage: treeStorage, + rawChangeLoader: newStorageLoader(treeStorage, changeBuilder), + validator: &noOpTreeValidator{}, + aclList: aclList, + } + + return buildObjectTree(deps) +} + func BuildObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) { rootChange, err := treeStorage.Root() if err != nil { diff --git a/commonspace/object/tree/objecttree/rawloader.go b/commonspace/object/tree/objecttree/rawloader.go index cae32cb3..c9234b5b 100644 --- a/commonspace/object/tree/objecttree/rawloader.go +++ b/commonspace/object/tree/objecttree/rawloader.go @@ -9,8 +9,9 @@ import ( ) type rawChangeLoader struct { - treeStorage treestorage.TreeStorage - changeBuilder ChangeBuilder + treeStorage treestorage.TreeStorage + changeBuilder ChangeBuilder + alwaysFromStorage bool // buffers idStack []string @@ -23,6 +24,12 @@ type rawCacheEntry struct { position int } +func newStorageLoader(treeStorage treestorage.TreeStorage, changeBuilder ChangeBuilder) *rawChangeLoader { + loader := newRawChangeLoader(treeStorage, changeBuilder) + loader.alwaysFromStorage = true + return loader +} + func newRawChangeLoader(treeStorage treestorage.TreeStorage, changeBuilder ChangeBuilder) *rawChangeLoader { return &rawChangeLoader{ treeStorage: treeStorage, @@ -30,7 +37,15 @@ func newRawChangeLoader(treeStorage treestorage.TreeStorage, changeBuilder Chang } } -func (r *rawChangeLoader) LoadFromTree(t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { +func (r *rawChangeLoader) Load(commonSnapshot string, t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { + if commonSnapshot == t.root.Id && !r.alwaysFromStorage { + return r.loadFromTree(t, breakpoints) + } else { + return r.loadFromStorage(commonSnapshot, t.Heads(), breakpoints) + } +} + +func (r *rawChangeLoader) loadFromTree(t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { var stack []*Change for _, h := range t.headIds { stack = append(stack, t.attached[h]) @@ -98,7 +113,7 @@ func (r *rawChangeLoader) LoadFromTree(t *Tree, breakpoints []string) ([]*treech return convert(results) } -func (r *rawChangeLoader) LoadFromStorage(commonSnapshot string, heads, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { +func (r *rawChangeLoader) loadFromStorage(commonSnapshot string, heads, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { // resetting cache r.cache = make(map[string]rawCacheEntry) defer func() { diff --git a/commonspace/object/tree/objecttree/testutils.go b/commonspace/object/tree/objecttree/testutils.go index 6385b9f7..fdcd4a9d 100644 --- a/commonspace/object/tree/objecttree/testutils.go +++ b/commonspace/object/tree/objecttree/testutils.go @@ -2,7 +2,6 @@ package objecttree import ( "fmt" - "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/util/crypto" @@ -112,20 +111,3 @@ func (c *MockChangeCreator) CreateNewTreeStorage(treeId, aclHeadId string) trees treeStorage, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root}) return treeStorage } - -func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { - root, _ := treeStorage.Root() - changeBuilder := &nonVerifiableChangeBuilder{ - ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root), - } - deps := objectTreeDeps{ - changeBuilder: changeBuilder, - treeBuilder: newTreeBuilder(treeStorage, changeBuilder), - treeStorage: treeStorage, - rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), - validator: &noOpTreeValidator{}, - aclList: aclList, - } - - return buildObjectTree(deps) -} diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncprotocol_test.go similarity index 75% rename from commonspace/object/tree/synctree/syncclient_test.go rename to commonspace/object/tree/synctree/syncprotocol_test.go index ed6dd710..4b81b166 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncprotocol_test.go @@ -2,7 +2,6 @@ package synctree import ( "context" - "fmt" "github.com/anytypeio/any-sync/commonspace/object/accountdata" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" @@ -60,7 +59,24 @@ func TestEmptyClientGetsFullHistory(t *testing.T) { require.Len(t, fullResponseMsg.changes, 2) } -func TestRandomTreeMerge(t *testing.T) { +func TestRandomMerge(t *testing.T) { + var ( + rnd = rand.New(rand.NewSource(time.Now().Unix())) + levels = 20 + perLevel = 20 + ) + testTreeMerge(t, levels, perLevel, func() bool { + return true + }) + testTreeMerge(t, levels, perLevel, func() bool { + return false + }) + testTreeMerge(t, levels, perLevel, func() bool { + return rnd.Intn(10) > 8 + }) +} + +func testTreeMerge(t *testing.T, levels, perlevel int, isSnapshot func() bool) { treeId := "treeId" spaceId := "spaceId" keys, err := accountdata.NewRandom() @@ -68,17 +84,15 @@ func TestRandomTreeMerge(t *testing.T) { aclList, err := list.NewTestDerivedAcl(spaceId, keys) storage := createStorage(treeId, aclList) changeCreator := objecttree.NewMockChangeCreator() - rnd := rand.New(rand.NewSource(time.Now().Unix())) params := genParams{ prefix: "peer1", aclId: aclList.Id(), startIdx: 0, - levels: 10, + levels: levels, + perLevel: perlevel, snapshotId: treeId, prevHeads: []string{treeId}, - isSnapshot: func() bool { - return rnd.Intn(100) > 80 - }, + isSnapshot: isSnapshot, } initialRes := genChanges(changeCreator, params) err = storage.TransactionAdd(initialRes.changes, initialRes.heads) @@ -99,20 +113,20 @@ func TestRandomTreeMerge(t *testing.T) { NewHeads: initialRes.heads, RawChanges: initialRes.changes, }) - time.Sleep(1000 * time.Millisecond) + time.Sleep(100 * time.Millisecond) firstHeads := fx.handlers["peer1"].tree().Heads() secondHeads := fx.handlers["peer2"].tree().Heads() require.True(t, slice.UnsortedEquals(firstHeads, secondHeads)) params = genParams{ - prefix: "peer1", - aclId: aclList.Id(), - startIdx: 11, - levels: 10, + prefix: "peer1", + aclId: aclList.Id(), + startIdx: levels, + levels: levels, + perLevel: perlevel, + snapshotId: initialRes.snapshotId, prevHeads: initialRes.heads, - isSnapshot: func() bool { - return rnd.Intn(100) > 80 - }, + isSnapshot: isSnapshot, } peer1Res := genChanges(changeCreator, params) params.prefix = "peer2" @@ -125,10 +139,15 @@ func TestRandomTreeMerge(t *testing.T) { NewHeads: peer2Res.heads, RawChanges: peer2Res.changes, }) - time.Sleep(1000 * time.Millisecond) + time.Sleep(100 * time.Millisecond) fx.stop() - firstHeads = fx.handlers["peer1"].tree().Heads() - secondHeads = fx.handlers["peer2"].tree().Heads() - fmt.Println(firstHeads) - fmt.Println(secondHeads) + firstTree := fx.handlers["peer1"].tree() + secondTree := fx.handlers["peer2"].tree() + firstHeads = firstTree.Heads() + secondHeads = secondTree.Heads() + require.True(t, slice.UnsortedEquals(firstHeads, secondHeads)) + require.True(t, slice.UnsortedEquals(firstHeads, append(peer1Res.heads, peer2Res.heads...))) + firstStorage := firstTree.Storage().(*treestorage.InMemoryTreeStorage) + secondStorage := secondTree.Storage().(*treestorage.InMemoryTreeStorage) + require.True(t, firstStorage.Equal(secondStorage)) } diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index 0399d239..ff2f5c06 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -3,7 +3,6 @@ package synctree import ( "context" "fmt" - "github.com/anytypeio/any-sync/commonspace/object/accountdata" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" @@ -275,7 +274,7 @@ func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage } func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (objecttree.ObjectTree, error) { - return objecttree.BuildTestableTree(aclList, storage) + return objecttree.BuildEmptyDataTestableTree(aclList, storage) } type fixtureDeps struct { @@ -345,6 +344,7 @@ type genParams struct { aclId string startIdx int levels int + perLevel int snapshotId string prevHeads []string isSnapshot func() bool @@ -374,7 +374,7 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge snapshotId = newId continue } - perLevel := rnd.Intn(10) + perLevel := rnd.Intn(params.perLevel) if perLevel == 0 { perLevel = 1 } @@ -383,7 +383,6 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge usedIds = map[string]struct{}{} ) for j := 0; j < perLevel; j++ { - // if we didn't connect with all prev ones prevConns := rnd.Intn(len(prevHeads)) if prevConns == 0 { prevConns = 1 @@ -391,6 +390,7 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge rnd.Shuffle(len(prevHeads), func(i, j int) { prevHeads[i], prevHeads[j] = prevHeads[j], prevHeads[i] }) + // if we didn't connect with all prev ones if j == perLevel-1 && len(usedIds) != len(prevHeads) { var unusedIds []string for _, id := range prevHeads { @@ -417,30 +417,3 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge res.snapshotId = snapshotId return } - -func TestGenChanges(t *testing.T) { - treeId := "treeId" - spaceId := "spaceId" - keys, err := accountdata.NewRandom() - require.NoError(t, err) - aclList, err := list.NewTestDerivedAcl(spaceId, keys) - storage := createStorage(treeId, aclList) - creator := objecttree.NewMockChangeCreator() - rnd := rand.New(rand.NewSource(time.Now().Unix())) - params := genParams{ - prefix: "peerId", - aclId: aclList.Id(), - startIdx: 0, - levels: 10, - snapshotId: treeId, - prevHeads: []string{treeId}, - isSnapshot: func() bool { - return rnd.Intn(100) > 80 - }, - } - res := genChanges(creator, params) - storage.TransactionAdd(res.changes, res.heads) - tr, err := createTestTree(aclList, storage) - require.NoError(t, err) - fmt.Println(tr.Debug(objecttree.NoOpDescriptionParser)) -} diff --git a/commonspace/object/tree/treestorage/inmemory.go b/commonspace/object/tree/treestorage/inmemory.go index 0f22037e..f5b192ba 100644 --- a/commonspace/object/tree/treestorage/inmemory.go +++ b/commonspace/object/tree/treestorage/inmemory.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/util/slice" "sync" ) @@ -105,3 +106,21 @@ func (t *InMemoryTreeStorage) Copy() *InMemoryTreeStorage { other, _ := NewInMemoryTreeStorage(t.root, t.heads, changes) return other.(*InMemoryTreeStorage) } + +func (t *InMemoryTreeStorage) Equal(other *InMemoryTreeStorage) bool { + if !slice.UnsortedEquals(t.heads, other.heads) { + return false + } + if len(t.changes) != len(other.changes) { + return false + } + for k, v := range t.changes { + if otherV, exists := other.changes[k]; exists { + if otherV.Id == v.Id { + continue + } + } + return false + } + return true +} From 8fc1f03f54b5f891c5f67ec318f566beb8c12dcd Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 18 Apr 2023 00:34:14 +0200 Subject: [PATCH 11/27] Change treegetter to treemanager and return BuildObjectTreeFunc --- commonspace/commongetter.go | 12 +- commonspace/headsync/diffsyncer.go | 6 +- commonspace/headsync/diffsyncer_test.go | 4 +- commonspace/headsync/headsync.go | 4 +- .../object/tree/objecttree/changebuilder.go | 2 +- .../tree/objecttree/objecttreefactory.go | 2 + commonspace/object/tree/synctree/synctree.go | 4 +- .../object/tree/synctree/synctreehandler.go | 1 - .../tree/synctree/synctreehandler_test.go | 20 ++- .../mock_treegetter/mock_treegetter.go | 136 ---------------- .../mock_treemanager/mock_treemanager.go | 152 ++++++++++++++++++ .../treemanager.go} | 9 +- commonspace/settings/deleter.go | 6 +- commonspace/settings/deleter_test.go | 12 +- commonspace/settings/deletionmanager.go | 10 +- commonspace/settings/deletionmanager_test.go | 14 +- commonspace/settings/settings.go | 24 +-- commonspace/settings/settings_test.go | 10 +- commonspace/space.go | 30 ++-- commonspace/spaceservice.go | 10 +- 20 files changed, 248 insertions(+), 220 deletions(-) delete mode 100644 commonspace/object/treegetter/mock_treegetter/mock_treegetter.go create mode 100644 commonspace/object/treemanager/mock_treemanager/mock_treemanager.go rename commonspace/object/{treegetter/treegetter.go => treemanager/treemanager.go} (55%) diff --git a/commonspace/commongetter.go b/commonspace/commongetter.go index 6807265b..74356281 100644 --- a/commonspace/commongetter.go +++ b/commonspace/commongetter.go @@ -4,20 +4,20 @@ import ( "context" "github.com/anytypeio/any-sync/commonspace/object/syncobjectgetter" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "sync/atomic" ) type commonGetter struct { - treegetter.TreeGetter + treemanager.TreeManager spaceId string reservedObjects []syncobjectgetter.SyncObject spaceIsClosed *atomic.Bool } -func newCommonGetter(spaceId string, getter treegetter.TreeGetter, spaceIsClosed *atomic.Bool) *commonGetter { +func newCommonGetter(spaceId string, getter treemanager.TreeManager, spaceIsClosed *atomic.Bool) *commonGetter { return &commonGetter{ - TreeGetter: getter, + TreeManager: getter, spaceId: spaceId, spaceIsClosed: spaceIsClosed, } @@ -34,7 +34,7 @@ func (c *commonGetter) GetTree(ctx context.Context, spaceId, treeId string) (obj if obj := c.getReservedObject(treeId); obj != nil { return obj.(objecttree.ObjectTree), nil } - return c.TreeGetter.GetTree(ctx, spaceId, treeId) + return c.TreeManager.GetTree(ctx, spaceId, treeId) } func (c *commonGetter) getReservedObject(id string) syncobjectgetter.SyncObject { @@ -53,7 +53,7 @@ func (c *commonGetter) GetObject(ctx context.Context, objectId string) (obj sync if obj := c.getReservedObject(objectId); obj != nil { return obj, nil } - t, err := c.TreeGetter.GetTree(ctx, c.spaceId, objectId) + t, err := c.TreeManager.GetTree(ctx, c.spaceId, objectId) if err != nil { return } diff --git a/commonspace/headsync/diffsyncer.go b/commonspace/headsync/diffsyncer.go index 9d3434c1..1b8a15dc 100644 --- a/commonspace/headsync/diffsyncer.go +++ b/commonspace/headsync/diffsyncer.go @@ -7,7 +7,7 @@ import ( "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/commonspace/credentialprovider" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" @@ -30,7 +30,7 @@ func newDiffSyncer( spaceId string, diff ldiff.Diff, peerManager peermanager.PeerManager, - cache treegetter.TreeGetter, + cache treemanager.TreeManager, storage spacestorage.SpaceStorage, clientFactory spacesyncproto.ClientFactory, syncStatus syncstatus.StatusUpdater, @@ -53,7 +53,7 @@ type diffSyncer struct { spaceId string diff ldiff.Diff peerManager peermanager.PeerManager - cache treegetter.TreeGetter + cache treemanager.TreeManager storage spacestorage.SpaceStorage clientFactory spacesyncproto.ClientFactory log logger.CtxLogger diff --git a/commonspace/headsync/diffsyncer_test.go b/commonspace/headsync/diffsyncer_test.go index ae40e5a7..e6558b61 100644 --- a/commonspace/headsync/diffsyncer_test.go +++ b/commonspace/headsync/diffsyncer_test.go @@ -12,7 +12,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/acl/liststorage/mock_liststorage" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" mock_treestorage "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage/mock_treestorage" - "github.com/anytypeio/any-sync/commonspace/object/treegetter/mock_treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager/mock_treemanager" "github.com/anytypeio/any-sync/commonspace/peermanager/mock_peermanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate/mock_settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage/mock_spacestorage" @@ -109,7 +109,7 @@ func TestDiffSyncer_Sync(t *testing.T) { diffMock := mock_ldiff.NewMockDiff(ctrl) peerManagerMock := mock_peermanager.NewMockPeerManager(ctrl) - cacheMock := mock_treegetter.NewMockTreeGetter(ctrl) + cacheMock := mock_treemanager.NewMockTreeManager(ctrl) stMock := mock_spacestorage.NewMockSpaceStorage(ctrl) clientMock := mock_spacesyncproto.NewMockDRPCSpaceSyncClient(ctrl) factory := spacesyncproto.ClientFactoryFunc(func(cc drpc.Conn) spacesyncproto.DRPCSpaceSyncClient { diff --git a/commonspace/headsync/headsync.go b/commonspace/headsync/headsync.go index 08d6d9e3..80fe77b8 100644 --- a/commonspace/headsync/headsync.go +++ b/commonspace/headsync/headsync.go @@ -6,7 +6,7 @@ import ( "github.com/anytypeio/any-sync/app/ldiff" "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/commonspace/credentialprovider" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" @@ -59,7 +59,7 @@ func NewHeadSync( configuration nodeconf.Configuration, storage spacestorage.SpaceStorage, peerManager peermanager.PeerManager, - cache treegetter.TreeGetter, + cache treemanager.TreeManager, syncStatus syncstatus.StatusUpdater, credentialProvider credentialprovider.CredentialProvider, log logger.CtxLogger) HeadSync { diff --git a/commonspace/object/tree/objecttree/changebuilder.go b/commonspace/object/tree/objecttree/changebuilder.go index 2f10fca9..b3dc65a1 100644 --- a/commonspace/object/tree/objecttree/changebuilder.go +++ b/commonspace/object/tree/objecttree/changebuilder.go @@ -84,7 +84,7 @@ func NewEmptyDataBuilder(keys crypto.KeyStorage, rootChange *treechangeproto.Raw rootChange: rootChange, keys: keys, newChange: func(id string, identity crypto.PubKey, ch *treechangeproto.TreeChange, signature []byte) *Change { - c := NewChange(id, identity, ch, signature) + c := NewChange(id, identity, ch, nil) c.Data = nil return c }, diff --git a/commonspace/object/tree/objecttree/objecttreefactory.go b/commonspace/object/tree/objecttree/objecttreefactory.go index 5b9196e4..a913d183 100644 --- a/commonspace/object/tree/objecttree/objecttreefactory.go +++ b/commonspace/object/tree/objecttree/objecttreefactory.go @@ -33,6 +33,8 @@ type objectTreeDeps struct { aclList list.AclList } +type BuildObjectTreeFunc = func(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) + var defaultObjectTreeDeps = verifiableTreeDeps func verifiableTreeDeps( diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index a1374b48..00b91846 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -54,7 +54,6 @@ type syncTree struct { var log = logger.NewNamed("common.commonspace.synctree") -var buildObjectTree = objecttree.BuildObjectTree var createSyncClient = newSyncClient type ResponsiblePeersGetter interface { @@ -73,6 +72,7 @@ type BuildDeps struct { OnClose func(id string) SyncStatus syncstatus.StatusUpdater PeerGetter ResponsiblePeersGetter + BuildObjectTree objecttree.BuildObjectTreeFunc WaitTreeRemoteSync bool } @@ -94,7 +94,7 @@ func PutSyncTree(ctx context.Context, payload treestorage.TreeStorageCreatePaylo } func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t SyncTree, err error) { - objTree, err := buildObjectTree(deps.TreeStorage, deps.AclList) + objTree, err := deps.BuildObjectTree(deps.TreeStorage, deps.AclList) if err != nil { return } diff --git a/commonspace/object/tree/synctree/synctreehandler.go b/commonspace/object/tree/synctree/synctreehandler.go index a41a2bad..63b708d9 100644 --- a/commonspace/object/tree/synctree/synctreehandler.go +++ b/commonspace/object/tree/synctree/synctreehandler.go @@ -82,7 +82,6 @@ func (s *syncTreeHandler) handleHeadUpdate( isEmptyUpdate = len(update.Changes) == 0 objTree = s.objTree ) - log := log.With( zap.Strings("update heads", update.Heads), zap.String("treeId", objTree.Id()), diff --git a/commonspace/object/tree/synctree/synctreehandler_test.go b/commonspace/object/tree/synctree/synctreehandler_test.go index 052479d2..f655b34a 100644 --- a/commonspace/object/tree/synctree/synctreehandler_test.go +++ b/commonspace/object/tree/synctree/synctreehandler_test.go @@ -89,6 +89,13 @@ func (fx *syncHandlerFixture) stop() { func TestSyncHandler_HandleHeadUpdate(t *testing.T) { ctx := context.Background() log = logger.CtxLogger{Logger: zap.NewNop()} + fullRequest := &treechangeproto.TreeSyncMessage{ + Content: &treechangeproto.TreeSyncContentValue{ + Value: &treechangeproto.TreeSyncContentValue_FullSyncRequest{ + FullSyncRequest: &treechangeproto.TreeFullSyncRequest{}, + }, + }, + } t.Run("head update non empty all heads added", func(t *testing.T) { fx := newSyncHandlerFixture(t) @@ -113,7 +120,6 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { RawChanges: []*treechangeproto.RawTreeChangeWithId{chWithId}, })). Return(objecttree.AddResult{}, nil) - fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2", "h1"}) fx.objectTreeMock.EXPECT().HasChanges(gomock.Eq([]string{"h1"})).Return(true) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) @@ -133,7 +139,6 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") - fullRequest := &treechangeproto.TreeSyncMessage{} fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2"}).AnyTimes() @@ -188,7 +193,6 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") - fullRequest := &treechangeproto.TreeSyncMessage{} fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2"}).AnyTimes() @@ -226,6 +230,13 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { ctx := context.Background() log = logger.CtxLogger{Logger: zap.NewNop()} + fullResponse := &treechangeproto.TreeSyncMessage{ + Content: &treechangeproto.TreeSyncContentValue{ + Value: &treechangeproto.TreeSyncContentValue_FullSyncResponse{ + FullSyncResponse: &treechangeproto.TreeFullSyncResponse{}, + }, + }, + } t.Run("full sync request with change", func(t *testing.T) { fx := newSyncHandlerFixture(t) @@ -240,7 +251,6 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") - fullResponse := &treechangeproto.TreeSyncMessage{} fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Header().Return(nil) @@ -274,7 +284,6 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") - fullResponse := &treechangeproto.TreeSyncMessage{} fx.objectTreeMock.EXPECT(). Id().AnyTimes().Return(treeId) @@ -305,7 +314,6 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") objectMsg.RequestId = replyId - fullResponse := &treechangeproto.TreeSyncMessage{} fx.objectTreeMock.EXPECT(). Id().AnyTimes().Return(treeId) diff --git a/commonspace/object/treegetter/mock_treegetter/mock_treegetter.go b/commonspace/object/treegetter/mock_treegetter/mock_treegetter.go deleted file mode 100644 index 6aee2208..00000000 --- a/commonspace/object/treegetter/mock_treegetter/mock_treegetter.go +++ /dev/null @@ -1,136 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/anytypeio/any-sync/commonspace/object/treegetter (interfaces: TreeGetter) - -// Package mock_treegetter is a generated GoMock package. -package mock_treegetter - -import ( - context "context" - reflect "reflect" - - app "github.com/anytypeio/any-sync/app" - objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" - gomock "github.com/golang/mock/gomock" -) - -// MockTreeGetter is a mock of TreeGetter interface. -type MockTreeGetter struct { - ctrl *gomock.Controller - recorder *MockTreeGetterMockRecorder -} - -// MockTreeGetterMockRecorder is the mock recorder for MockTreeGetter. -type MockTreeGetterMockRecorder struct { - mock *MockTreeGetter -} - -// NewMockTreeGetter creates a new mock instance. -func NewMockTreeGetter(ctrl *gomock.Controller) *MockTreeGetter { - mock := &MockTreeGetter{ctrl: ctrl} - mock.recorder = &MockTreeGetterMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockTreeGetter) EXPECT() *MockTreeGetterMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MockTreeGetter) Close(arg0 context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MockTreeGetterMockRecorder) Close(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTreeGetter)(nil).Close), arg0) -} - -// DeleteSpace mocks base method. -func (m *MockTreeGetter) DeleteSpace(arg0 context.Context, arg1 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteSpace", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteSpace indicates an expected call of DeleteSpace. -func (mr *MockTreeGetterMockRecorder) DeleteSpace(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSpace", reflect.TypeOf((*MockTreeGetter)(nil).DeleteSpace), arg0, arg1) -} - -// DeleteTree mocks base method. -func (m *MockTreeGetter) DeleteTree(arg0 context.Context, arg1, arg2 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteTree", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteTree indicates an expected call of DeleteTree. -func (mr *MockTreeGetterMockRecorder) DeleteTree(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteTree", reflect.TypeOf((*MockTreeGetter)(nil).DeleteTree), arg0, arg1, arg2) -} - -// GetTree mocks base method. -func (m *MockTreeGetter) GetTree(arg0 context.Context, arg1, arg2 string) (objecttree.ObjectTree, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTree", arg0, arg1, arg2) - ret0, _ := ret[0].(objecttree.ObjectTree) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTree indicates an expected call of GetTree. -func (mr *MockTreeGetterMockRecorder) GetTree(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTree", reflect.TypeOf((*MockTreeGetter)(nil).GetTree), arg0, arg1, arg2) -} - -// Init mocks base method. -func (m *MockTreeGetter) Init(arg0 *app.App) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Init", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Init indicates an expected call of Init. -func (mr *MockTreeGetterMockRecorder) Init(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockTreeGetter)(nil).Init), arg0) -} - -// Name mocks base method. -func (m *MockTreeGetter) Name() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Name") - ret0, _ := ret[0].(string) - return ret0 -} - -// Name indicates an expected call of Name. -func (mr *MockTreeGetterMockRecorder) Name() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockTreeGetter)(nil).Name)) -} - -// Run mocks base method. -func (m *MockTreeGetter) Run(arg0 context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Run", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Run indicates an expected call of Run. -func (mr *MockTreeGetterMockRecorder) Run(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockTreeGetter)(nil).Run), arg0) -} diff --git a/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go b/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go new file mode 100644 index 00000000..f378bc43 --- /dev/null +++ b/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go @@ -0,0 +1,152 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/anytypeio/any-sync/commonspace/object/treemanager (interfaces: TreeManager) + +// Package mock_treemanager is a generated GoMock package. +package mock_treemanager + +import ( + context "context" + reflect "reflect" + + app "github.com/anytypeio/any-sync/app" + list "github.com/anytypeio/any-sync/commonspace/object/acl/list" + objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + treestorage "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + gomock "github.com/golang/mock/gomock" +) + +// MockTreeManager is a mock of TreeManager interface. +type MockTreeManager struct { + ctrl *gomock.Controller + recorder *MockTreeManagerMockRecorder +} + +// MockTreeManagerMockRecorder is the mock recorder for MockTreeManager. +type MockTreeManagerMockRecorder struct { + mock *MockTreeManager +} + +// NewMockTreeManager creates a new mock instance. +func NewMockTreeManager(ctrl *gomock.Controller) *MockTreeManager { + mock := &MockTreeManager{ctrl: ctrl} + mock.recorder = &MockTreeManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockTreeManager) EXPECT() *MockTreeManagerMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockTreeManager) Close(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockTreeManagerMockRecorder) Close(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTreeManager)(nil).Close), arg0) +} + +// DeleteSpace mocks base method. +func (m *MockTreeManager) DeleteSpace(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteSpace", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteSpace indicates an expected call of DeleteSpace. +func (mr *MockTreeManagerMockRecorder) DeleteSpace(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSpace", reflect.TypeOf((*MockTreeManager)(nil).DeleteSpace), arg0, arg1) +} + +// DeleteTree mocks base method. +func (m *MockTreeManager) DeleteTree(arg0 context.Context, arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteTree", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteTree indicates an expected call of DeleteTree. +func (mr *MockTreeManagerMockRecorder) DeleteTree(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteTree", reflect.TypeOf((*MockTreeManager)(nil).DeleteTree), arg0, arg1, arg2) +} + +// GetTree mocks base method. +func (m *MockTreeManager) GetTree(arg0 context.Context, arg1, arg2 string) (objecttree.ObjectTree, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTree", arg0, arg1, arg2) + ret0, _ := ret[0].(objecttree.ObjectTree) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTree indicates an expected call of GetTree. +func (mr *MockTreeManagerMockRecorder) GetTree(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTree", reflect.TypeOf((*MockTreeManager)(nil).GetTree), arg0, arg1, arg2) +} + +// Init mocks base method. +func (m *MockTreeManager) Init(arg0 *app.App) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Init", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Init indicates an expected call of Init. +func (mr *MockTreeManagerMockRecorder) Init(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockTreeManager)(nil).Init), arg0) +} + +// Name mocks base method. +func (m *MockTreeManager) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockTreeManagerMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockTreeManager)(nil).Name)) +} + +// ObjectTreeBuilder mocks base method. +func (m *MockTreeManager) ObjectTreeBuilder() func(treestorage.TreeStorage, list.AclList) (objecttree.ObjectTree, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ObjectTreeBuilder") + ret0, _ := ret[0].(func(treestorage.TreeStorage, list.AclList) (objecttree.ObjectTree, error)) + return ret0 +} + +// ObjectTreeBuilder indicates an expected call of ObjectTreeBuilder. +func (mr *MockTreeManagerMockRecorder) ObjectTreeBuilder() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ObjectTreeBuilder", reflect.TypeOf((*MockTreeManager)(nil).ObjectTreeBuilder)) +} + +// Run mocks base method. +func (m *MockTreeManager) Run(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Run", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Run indicates an expected call of Run. +func (mr *MockTreeManagerMockRecorder) Run(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockTreeManager)(nil).Run), arg0) +} diff --git a/commonspace/object/treegetter/treegetter.go b/commonspace/object/treemanager/treemanager.go similarity index 55% rename from commonspace/object/treegetter/treegetter.go rename to commonspace/object/treemanager/treemanager.go index 4e604d10..00685ff5 100644 --- a/commonspace/object/treegetter/treegetter.go +++ b/commonspace/object/treemanager/treemanager.go @@ -1,5 +1,5 @@ -//go:generate mockgen -destination mock_treegetter/mock_treegetter.go github.com/anytypeio/any-sync/commonspace/object/treegetter TreeGetter -package treegetter +//go:generate mockgen -destination mock_treemanager/mock_treemanager.go github.com/anytypeio/any-sync/commonspace/object/treemanager TreeManager +package treemanager import ( "context" @@ -7,11 +7,12 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" ) -const CName = "common.object.treegetter" +const CName = "common.object.treemanager" -type TreeGetter interface { +type TreeManager interface { app.ComponentRunnable GetTree(ctx context.Context, spaceId, treeId string) (objecttree.ObjectTree, error) DeleteTree(ctx context.Context, spaceId, treeId string) error DeleteSpace(ctx context.Context, spaceId string) error + ObjectTreeBuilder() objecttree.BuildObjectTreeFunc } diff --git a/commonspace/settings/deleter.go b/commonspace/settings/deleter.go index 55f026c1..c50da268 100644 --- a/commonspace/settings/deleter.go +++ b/commonspace/settings/deleter.go @@ -2,7 +2,7 @@ package settings import ( "context" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" "go.uber.org/zap" @@ -15,10 +15,10 @@ type Deleter interface { type deleter struct { st spacestorage.SpaceStorage state settingsstate.ObjectDeletionState - getter treegetter.TreeGetter + getter treemanager.TreeManager } -func newDeleter(st spacestorage.SpaceStorage, state settingsstate.ObjectDeletionState, getter treegetter.TreeGetter) Deleter { +func newDeleter(st spacestorage.SpaceStorage, state settingsstate.ObjectDeletionState, getter treemanager.TreeManager) Deleter { return &deleter{st, state, getter} } diff --git a/commonspace/settings/deleter_test.go b/commonspace/settings/deleter_test.go index 3606225b..fe6f7c51 100644 --- a/commonspace/settings/deleter_test.go +++ b/commonspace/settings/deleter_test.go @@ -2,7 +2,7 @@ package settings import ( "fmt" - "github.com/anytypeio/any-sync/commonspace/object/treegetter/mock_treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager/mock_treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate/mock_settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/commonspace/spacestorage/mock_spacestorage" @@ -12,18 +12,18 @@ import ( func TestDeleter_Delete(t *testing.T) { ctrl := gomock.NewController(t) - treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) + treeManager := mock_treemanager.NewMockTreeManager(ctrl) st := mock_spacestorage.NewMockSpaceStorage(ctrl) delState := mock_settingsstate.NewMockObjectDeletionState(ctrl) - deleter := newDeleter(st, delState, treeGetter) + deleter := newDeleter(st, delState, treeManager) t.Run("deleter delete queued", func(t *testing.T) { id := "id" spaceId := "spaceId" delState.EXPECT().GetQueued().Return([]string{id}) st.EXPECT().Id().Return(spaceId) - treeGetter.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(nil) + treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(nil) delState.EXPECT().Delete(id).Return(nil) deleter.Delete() @@ -34,7 +34,7 @@ func TestDeleter_Delete(t *testing.T) { spaceId := "spaceId" delState.EXPECT().GetQueued().Return([]string{id}) st.EXPECT().Id().Return(spaceId) - treeGetter.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(spacestorage.ErrTreeStorageAlreadyDeleted) + treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(spacestorage.ErrTreeStorageAlreadyDeleted) delState.EXPECT().Delete(id).Return(nil) deleter.Delete() @@ -45,7 +45,7 @@ func TestDeleter_Delete(t *testing.T) { spaceId := "spaceId" delState.EXPECT().GetQueued().Return([]string{id}) st.EXPECT().Id().Return(spaceId) - treeGetter.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(fmt.Errorf("some error")) + treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(fmt.Errorf("some error")) deleter.Delete() }) diff --git a/commonspace/settings/deletionmanager.go b/commonspace/settings/deletionmanager.go index 8bfc15e3..366c2686 100644 --- a/commonspace/settings/deletionmanager.go +++ b/commonspace/settings/deletionmanager.go @@ -2,7 +2,7 @@ package settings import ( "context" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/util/slice" "go.uber.org/zap" @@ -20,12 +20,12 @@ func newDeletionManager( spaceId string, settingsId string, isResponsible bool, - treeGetter treegetter.TreeGetter, + treeManager treemanager.TreeManager, deletionState settingsstate.ObjectDeletionState, provider SpaceIdsProvider, onSpaceDelete func()) DeletionManager { return &deletionManager{ - treeGetter: treeGetter, + treeManager: treeManager, isResponsible: isResponsible, spaceId: spaceId, settingsId: settingsId, @@ -38,7 +38,7 @@ func newDeletionManager( type deletionManager struct { deletionState settingsstate.ObjectDeletionState provider SpaceIdsProvider - treeGetter treegetter.TreeGetter + treeManager treemanager.TreeManager spaceId string settingsId string isResponsible bool @@ -55,7 +55,7 @@ func (d *deletionManager) UpdateState(ctx context.Context, state *settingsstate. return nil } log.Debug("deleting space") - err = d.treeGetter.DeleteSpace(ctx, d.spaceId) + err = d.treeManager.DeleteSpace(ctx, d.spaceId) if err != nil { log.Debug("failed to notify on space deletion", zap.Error(err)) } diff --git a/commonspace/settings/deletionmanager_test.go b/commonspace/settings/deletionmanager_test.go index a4992903..990efc07 100644 --- a/commonspace/settings/deletionmanager_test.go +++ b/commonspace/settings/deletionmanager_test.go @@ -2,7 +2,7 @@ package settings import ( "context" - "github.com/anytypeio/any-sync/commonspace/object/treegetter/mock_treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager/mock_treemanager" "github.com/anytypeio/any-sync/commonspace/settings/mock_settings" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate/mock_settingsstate" @@ -27,15 +27,15 @@ func TestDeletionManager_UpdateState_NotResponsible(t *testing.T) { deleted = true } delState := mock_settingsstate.NewMockObjectDeletionState(ctrl) - treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) + treeManager := mock_treemanager.NewMockTreeManager(ctrl) delState.EXPECT().Add(state.DeletedIds).Return(nil) - treeGetter.EXPECT().DeleteSpace(ctx, spaceId).Return(nil) + treeManager.EXPECT().DeleteSpace(ctx, spaceId).Return(nil) delManager := newDeletionManager(spaceId, settingsId, false, - treeGetter, + treeManager, delState, nil, onDeleted) @@ -60,17 +60,17 @@ func TestDeletionManager_UpdateState_Responsible(t *testing.T) { deleted = true } delState := mock_settingsstate.NewMockObjectDeletionState(ctrl) - treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) + treeManager := mock_treemanager.NewMockTreeManager(ctrl) provider := mock_settings.NewMockSpaceIdsProvider(ctrl) delState.EXPECT().Add(state.DeletedIds).Return(nil) - treeGetter.EXPECT().DeleteSpace(ctx, spaceId).Return(nil) + treeManager.EXPECT().DeleteSpace(ctx, spaceId).Return(nil) provider.EXPECT().AllIds().Return([]string{"id", "otherId", settingsId}) delState.EXPECT().Add([]string{"id", "otherId"}).Return(nil) delManager := newDeletionManager(spaceId, settingsId, true, - treeGetter, + treeManager, delState, provider, onDeleted) diff --git a/commonspace/settings/settings.go b/commonspace/settings/settings.go index 1455faff..14430e0f 100644 --- a/commonspace/settings/settings.go +++ b/commonspace/settings/settings.go @@ -13,7 +13,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" @@ -47,7 +47,7 @@ type BuildTreeFunc func(ctx context.Context, id string, listener updatelistener. type Deps struct { BuildFunc BuildTreeFunc Account accountservice.Service - TreeGetter treegetter.TreeGetter + TreeManager treemanager.TreeManager Store spacestorage.SpaceStorage Configuration nodeconf.Configuration DeletionState settingsstate.ObjectDeletionState @@ -62,13 +62,13 @@ type Deps struct { type settingsObject struct { synctree.SyncTree - account accountservice.Service - spaceId string - treeGetter treegetter.TreeGetter - store spacestorage.SpaceStorage - builder settingsstate.StateBuilder - buildFunc BuildTreeFunc - loop *deleteLoop + account accountservice.Service + spaceId string + treeManager treemanager.TreeManager + store spacestorage.SpaceStorage + builder settingsstate.StateBuilder + buildFunc BuildTreeFunc + loop *deleteLoop state *settingsstate.State deletionState settingsstate.ObjectDeletionState @@ -84,7 +84,7 @@ func NewSettingsObject(deps Deps, spaceId string) (obj SettingsObject) { changeFactory settingsstate.ChangeFactory ) if deps.del == nil { - deleter = newDeleter(deps.Store, deps.DeletionState, deps.TreeGetter) + deleter = newDeleter(deps.Store, deps.DeletionState, deps.TreeManager) } else { deleter = deps.del } @@ -93,7 +93,7 @@ func NewSettingsObject(deps Deps, spaceId string) (obj SettingsObject) { spaceId, deps.Store.SpaceSettingsId(), deps.Configuration.IsResponsible(spaceId), - deps.TreeGetter, + deps.TreeManager, deps.DeletionState, deps.Provider, deps.OnSpaceDelete) @@ -123,7 +123,7 @@ func NewSettingsObject(deps Deps, spaceId string) (obj SettingsObject) { spaceId: spaceId, account: deps.Account, deletionState: deps.DeletionState, - treeGetter: deps.TreeGetter, + treeManager: deps.TreeManager, store: deps.Store, buildFunc: deps.BuildFunc, builder: builder, diff --git a/commonspace/settings/settings_test.go b/commonspace/settings/settings_test.go index 166cf284..b4879f53 100644 --- a/commonspace/settings/settings_test.go +++ b/commonspace/settings/settings_test.go @@ -9,7 +9,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/mock_synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/object/treegetter/mock_treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager/mock_treemanager" "github.com/anytypeio/any-sync/commonspace/settings/mock_settings" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate/mock_settingsstate" @@ -45,7 +45,7 @@ type settingsFixture struct { docId string doc *settingsObject ctrl *gomock.Controller - treeGetter *mock_treegetter.MockTreeGetter + treeManager *mock_treemanager.MockTreeManager spaceStorage *mock_spacestorage.MockSpaceStorage stateBuilder *mock_settingsstate.MockStateBuilder deletionManager *mock_settings.MockDeletionManager @@ -62,7 +62,7 @@ func newSettingsFixture(t *testing.T) *settingsFixture { ctrl := gomock.NewController(t) acc := mock_accountservice.NewMockService(ctrl) - treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) + treeManager := mock_treemanager.NewMockTreeManager(ctrl) st := mock_spacestorage.NewMockSpaceStorage(ctrl) delState := mock_settingsstate.NewMockObjectDeletionState(ctrl) delManager := mock_settings.NewMockDeletionManager(ctrl) @@ -81,7 +81,7 @@ func newSettingsFixture(t *testing.T) *settingsFixture { deps := Deps{ BuildFunc: buildFunc, Account: acc, - TreeGetter: treeGetter, + TreeManager: treeManager, Store: st, DeletionState: delState, delManager: delManager, @@ -95,7 +95,7 @@ func newSettingsFixture(t *testing.T) *settingsFixture { docId: objectId, doc: doc, ctrl: ctrl, - treeGetter: treeGetter, + treeManager: treeManager, spaceStorage: st, stateBuilder: stateBuilder, changeFactory: changeFactory, diff --git a/commonspace/space.go b/commonspace/space.go index 8b4375fb..c68ff8e3 100644 --- a/commonspace/space.go +++ b/commonspace/space.go @@ -118,7 +118,7 @@ type space struct { headSync headsync.HeadSync syncStatus syncstatus.StatusUpdater storage spacestorage.SpaceStorage - cache *commonGetter + treeManager *commonGetter account accountservice.Service aclList *syncacl.SyncAcl configuration nodeconf.Configuration @@ -179,7 +179,7 @@ func (s *space) Init(ctx context.Context) (err error) { return } s.aclList = syncacl.NewSyncAcl(aclList, s.objectSync.MessagePool()) - s.cache.AddObject(s.aclList) + s.treeManager.AddObject(s.aclList) deletionState := settingsstate.NewObjectDeletionState(s.storage) deps := settings.Deps{ @@ -196,7 +196,7 @@ func (s *space) Init(ctx context.Context) (err error) { return }, Account: s.account, - TreeGetter: s.cache, + TreeManager: s.treeManager, Store: s.storage, DeletionState: deletionState, Provider: s.headSync, @@ -210,7 +210,7 @@ func (s *space) Init(ctx context.Context) (err error) { if err != nil { return } - s.cache.AddObject(s.settingsObject) + s.treeManager.AddObject(s.settingsObject) s.syncStatus.Run() s.handleQueue = multiqueue.New[HandleMessage](s.handleMessage, 100) return nil @@ -283,16 +283,17 @@ func (s *space) PutTree(ctx context.Context, payload treestorage.TreeStorageCrea return } deps := synctree.BuildDeps{ - SpaceId: s.id, - ObjectSync: s.objectSync, - Configuration: s.configuration, - HeadNotifiable: s.headSync, - Listener: listener, - AclList: s.aclList, - SpaceStorage: s.storage, - OnClose: s.onObjectClose, - SyncStatus: s.syncStatus, - PeerGetter: s.peerManager, + SpaceId: s.id, + ObjectSync: s.objectSync, + Configuration: s.configuration, + HeadNotifiable: s.headSync, + Listener: listener, + AclList: s.aclList, + SpaceStorage: s.storage, + OnClose: s.onObjectClose, + SyncStatus: s.syncStatus, + PeerGetter: s.peerManager, + BuildObjectTree: s.treeManager.ObjectTreeBuilder(), } t, err = synctree.PutSyncTree(ctx, payload, deps) if err != nil { @@ -331,6 +332,7 @@ func (s *space) BuildTree(ctx context.Context, id string, opts BuildTreeOpts) (t SyncStatus: s.syncStatus, WaitTreeRemoteSync: opts.WaitTreeRemoteSync, PeerGetter: s.peerManager, + BuildObjectTree: s.treeManager.ObjectTreeBuilder(), } if t, err = synctree.BuildSyncTreeOrGetRemote(ctx, id, deps); err != nil { return nil, err diff --git a/commonspace/spaceservice.go b/commonspace/spaceservice.go index cf4a2710..cfe33692 100644 --- a/commonspace/spaceservice.go +++ b/commonspace/spaceservice.go @@ -9,7 +9,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/headsync" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/objectsync" "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/spacestorage" @@ -49,7 +49,7 @@ type spaceService struct { storageProvider spacestorage.SpaceStorageProvider peermanagerProvider peermanager.PeerManagerProvider credentialProvider credentialprovider.CredentialProvider - treeGetter treegetter.TreeGetter + treeManager treemanager.TreeManager pool pool.Pool } @@ -58,7 +58,7 @@ func (s *spaceService) Init(a *app.App) (err error) { s.account = a.MustComponent(accountservice.CName).(accountservice.Service) s.storageProvider = a.MustComponent(spacestorage.CName).(spacestorage.SpaceStorageProvider) s.configurationService = a.MustComponent(nodeconf.CName).(nodeconf.Service) - s.treeGetter = a.MustComponent(treegetter.CName).(treegetter.TreeGetter) + s.treeManager = a.MustComponent(treemanager.CName).(treemanager.TreeManager) s.peermanagerProvider = a.MustComponent(peermanager.CName).(peermanager.PeerManagerProvider) credProvider := a.Component(credentialprovider.CName) if credProvider != nil { @@ -145,7 +145,7 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) { return nil, err } spaceIsDeleted.Swap(isDeleted) - getter := newCommonGetter(st.Id(), s.treeGetter, spaceIsClosed) + getter := newCommonGetter(st.Id(), s.treeManager, spaceIsClosed) syncStatus := syncstatus.NewNoOpSyncStatus() // this will work only for clients, not the best solution, but... if !lastConfiguration.IsResponsible(st.Id()) { @@ -165,7 +165,7 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) { objectSync: objectSync, headSync: headSync, syncStatus: syncStatus, - cache: getter, + treeManager: getter, account: s.account, configuration: lastConfiguration, peerManager: peerManager, From 99bad9ce9d5e0ad53e511ece48b59bc1d26026ee Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 18 Apr 2023 12:09:25 +0200 Subject: [PATCH 12/27] Send error response on sync (wip) --- .../object/tree/synctree/syncclient.go | 2 +- commonspace/object/tree/synctree/synctree.go | 2 +- .../object/tree/synctree/treeremotegetter.go | 10 +- .../object/tree/synctree/utils_test.go | 4 +- .../treechangeproto/protos/treechange.proto | 2 +- .../tree/treechangeproto/treechange.pb.go | 130 ++++++++---------- commonspace/objectsync/objectsync.go | 51 ++++--- commonspace/space.go | 1 - 8 files changed, 94 insertions(+), 108 deletions(-) diff --git a/commonspace/object/tree/synctree/syncclient.go b/commonspace/object/tree/synctree/syncclient.go index f6e6b990..5967441b 100644 --- a/commonspace/object/tree/synctree/syncclient.go +++ b/commonspace/object/tree/synctree/syncclient.go @@ -20,7 +20,7 @@ type syncClient struct { spaceId string } -func newSyncClient( +func NewSyncClient( spaceId string, peerManager peermanager.PeerManager, factory RequestFactory) SyncClient { diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index 00b91846..89787597 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -54,7 +54,7 @@ type syncTree struct { var log = logger.NewNamed("common.commonspace.synctree") -var createSyncClient = newSyncClient +var createSyncClient = NewSyncClient type ResponsiblePeersGetter interface { GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) diff --git a/commonspace/object/tree/synctree/treeremotegetter.go b/commonspace/object/tree/synctree/treeremotegetter.go index 561d850c..9fd0677a 100644 --- a/commonspace/object/tree/synctree/treeremotegetter.go +++ b/commonspace/object/tree/synctree/treeremotegetter.go @@ -8,6 +8,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/net/peer" + "github.com/anytypeio/any-sync/net/rpc/rpcerr" "github.com/gogo/protobuf/proto" "go.uber.org/zap" "time" @@ -117,9 +118,16 @@ func (t treeRemoteGetter) getTree(ctx context.Context) (treeStorage treestorage. if err != nil { return } - if resp.GetContent().GetFullSyncResponse() == nil { + switch { + case resp.GetContent().GetErrorResponse() != nil: + errResp := resp.GetContent().GetErrorResponse() + err = rpcerr.Err(errResp.ErrCode) + return + case resp.GetContent().GetFullSyncResponse() == nil: err = fmt.Errorf("expected to get full sync response, but got something else") return + default: + break } fullSyncResp := resp.GetContent().GetFullSyncResponse() diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index ff2f5c06..524ab1d2 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -244,7 +244,7 @@ func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.Ra func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *processSyncHandler { factory := GetRequestFactory() - syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) + syncClient := NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) netTree := &broadcastTree{ ObjectTree: objTree, SyncClient: syncClient, @@ -255,7 +255,7 @@ func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, lo func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *processSyncHandler { factory := GetRequestFactory() - syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) + syncClient := NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) batcher := mb.New[processMsg](0) return &processSyncHandler{ diff --git a/commonspace/object/tree/treechangeproto/protos/treechange.proto b/commonspace/object/tree/treechangeproto/protos/treechange.proto index dfc74a42..1ecd9bd6 100644 --- a/commonspace/object/tree/treechangeproto/protos/treechange.proto +++ b/commonspace/object/tree/treechangeproto/protos/treechange.proto @@ -94,7 +94,7 @@ message TreeFullSyncResponse { // TreeErrorResponse is an error sent as a response for a full sync request message TreeErrorResponse { - string error = 1; + uint64 errCode = 1; } // TreeChangeInfo is used internally in Tree implementation for ease of marshalling diff --git a/commonspace/object/tree/treechangeproto/treechange.pb.go b/commonspace/object/tree/treechangeproto/treechange.pb.go index 60eb619b..14815df1 100644 --- a/commonspace/object/tree/treechangeproto/treechange.pb.go +++ b/commonspace/object/tree/treechangeproto/treechange.pb.go @@ -691,7 +691,7 @@ func (m *TreeFullSyncResponse) GetSnapshotPath() []string { // TreeErrorResponse is an error sent as a response for a full sync request type TreeErrorResponse struct { - Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` + ErrCode uint64 `protobuf:"varint,1,opt,name=errCode,proto3" json:"errCode,omitempty"` } func (m *TreeErrorResponse) Reset() { *m = TreeErrorResponse{} } @@ -727,11 +727,11 @@ func (m *TreeErrorResponse) XXX_DiscardUnknown() { var xxx_messageInfo_TreeErrorResponse proto.InternalMessageInfo -func (m *TreeErrorResponse) GetError() string { +func (m *TreeErrorResponse) GetErrCode() uint64 { if m != nil { - return m.Error + return m.ErrCode } - return "" + return 0 } // TreeChangeInfo is used internally in Tree implementation for ease of marshalling @@ -806,50 +806,50 @@ func init() { } var fileDescriptor_5033f0301ef9b772 = []byte{ - // 677 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0xcf, 0x4f, 0xd4, 0x40, - 0x14, 0xee, 0x74, 0x81, 0xb2, 0x8f, 0x05, 0x74, 0xe0, 0xd0, 0x10, 0xad, 0x4d, 0x63, 0x74, 0xbd, - 0x40, 0x82, 0x27, 0x8d, 0x09, 0x11, 0x04, 0x77, 0x43, 0x34, 0x64, 0x40, 0x4c, 0xbc, 0x0d, 0xed, - 0xc0, 0xd6, 0xec, 0x76, 0x6a, 0x67, 0x56, 0xb2, 0x7f, 0x80, 0x17, 0x4d, 0x88, 0xff, 0x92, 0x37, - 0x8f, 0x1c, 0x39, 0x1a, 0xf6, 0x1f, 0x31, 0x9d, 0x69, 0xb7, 0x3f, 0x76, 0x0f, 0xdc, 0xb8, 0x74, - 0xf7, 0x7d, 0x7d, 0xef, 0x7b, 0xdf, 0xfb, 0xe6, 0x47, 0x61, 0xc7, 0xe7, 0x83, 0x01, 0x8f, 0x44, - 0x4c, 0x7d, 0xb6, 0xc5, 0xcf, 0xbe, 0x32, 0x5f, 0x6e, 0xc9, 0x84, 0x31, 0xf5, 0xf0, 0x7b, 0x34, - 0xba, 0x60, 0x71, 0xc2, 0x25, 0xdf, 0x52, 0x4f, 0x51, 0x82, 0x37, 0x15, 0x82, 0xa1, 0x40, 0xbc, - 0x1b, 0x04, 0x40, 0x38, 0x97, 0x7b, 0x2a, 0xc4, 0x8f, 0xa0, 0x49, 0xfd, 0x7e, 0x87, 0xd1, 0xa0, - 0x1b, 0xd8, 0xc8, 0x45, 0xed, 0x26, 0x29, 0x00, 0x6c, 0x83, 0xa5, 0xba, 0x76, 0x03, 0xdb, 0x54, - 0xef, 0xf2, 0x10, 0x3b, 0x00, 0x9a, 0xf0, 0x64, 0x14, 0x33, 0xbb, 0xa1, 0x5e, 0x96, 0x90, 0x94, - 0x57, 0x86, 0x03, 0x26, 0x24, 0x1d, 0xc4, 0xf6, 0x9c, 0x8b, 0xda, 0x0d, 0x52, 0x00, 0x18, 0xc3, - 0x9c, 0x60, 0x2c, 0xb0, 0xe7, 0x5d, 0xd4, 0x6e, 0x11, 0xf5, 0x1f, 0x6f, 0xc0, 0x62, 0x18, 0xb0, - 0x48, 0x86, 0x72, 0x64, 0x2f, 0x28, 0x7c, 0x12, 0xe3, 0xa7, 0xb0, 0xac, 0xb9, 0x8f, 0xe8, 0xa8, - 0xcf, 0x69, 0x60, 0x5b, 0x2a, 0xa1, 0x0a, 0x7a, 0x57, 0x26, 0xc0, 0x49, 0xc2, 0x58, 0x36, 0x9a, - 0x0b, 0x4b, 0xe9, 0xdc, 0x7a, 0x14, 0x61, 0x23, 0xb7, 0xd1, 0x6e, 0x92, 0x32, 0x54, 0x1d, 0xde, - 0xac, 0x0f, 0xff, 0x0c, 0x56, 0x44, 0x44, 0x63, 0xd1, 0xe3, 0x72, 0x97, 0x8a, 0xd4, 0x03, 0x3d, - 0x66, 0x0d, 0x4d, 0xfb, 0x68, 0x1d, 0xe2, 0x1d, 0x95, 0x54, 0x0d, 0xdb, 0x22, 0x65, 0x28, 0xed, - 0x93, 0x30, 0x1a, 0x1c, 0xb2, 0x51, 0x57, 0xcf, 0xdc, 0x24, 0x05, 0x50, 0xb5, 0x6a, 0xa1, 0x6e, - 0x55, 0xd9, 0x16, 0xab, 0x66, 0x8b, 0x03, 0x10, 0x8a, 0xe3, 0x4c, 0x8d, 0xbd, 0xe8, 0xa2, 0xf6, - 0x22, 0x29, 0x21, 0xde, 0x7b, 0x58, 0x26, 0xf4, 0xb2, 0x64, 0x89, 0x0d, 0x56, 0x9c, 0x39, 0x88, - 0x14, 0x57, 0x1e, 0xa6, 0x22, 0x44, 0x78, 0x11, 0x51, 0x39, 0x4c, 0x98, 0xb2, 0xa2, 0x45, 0x0a, - 0xc0, 0xdb, 0x83, 0xb5, 0x0a, 0xd1, 0xe7, 0x50, 0xf6, 0xb4, 0xf2, 0x84, 0x5e, 0x6a, 0x28, 0x23, - 0x2c, 0x00, 0xbc, 0x02, 0x66, 0x98, 0xdb, 0x6a, 0x86, 0x81, 0x77, 0x85, 0x60, 0x35, 0xa5, 0x38, - 0x1e, 0x45, 0xfe, 0x07, 0x26, 0x04, 0xbd, 0x60, 0xf8, 0x35, 0x58, 0x3e, 0x8f, 0x24, 0x8b, 0xa4, - 0xaa, 0x5f, 0xda, 0x76, 0x37, 0x4b, 0xbb, 0x37, 0xcf, 0xde, 0xd3, 0x29, 0xa7, 0xb4, 0x3f, 0x64, - 0x24, 0x2f, 0xc0, 0x3b, 0x00, 0xc9, 0x64, 0x23, 0xab, 0x3e, 0x4b, 0xdb, 0x4f, 0xca, 0xe5, 0x33, - 0x24, 0x93, 0x52, 0x89, 0xf7, 0xc7, 0x84, 0xf5, 0x59, 0x2d, 0xf0, 0x1b, 0x80, 0x1e, 0xa3, 0xc1, - 0xa7, 0x38, 0xa0, 0x92, 0x65, 0xc2, 0x36, 0xea, 0xc2, 0x3a, 0x93, 0x8c, 0x8e, 0x41, 0x4a, 0xf9, - 0xf8, 0x10, 0x56, 0xcf, 0x87, 0xfd, 0x7e, 0xca, 0x4a, 0xd8, 0xb7, 0x21, 0x13, 0x72, 0x96, 0xb8, - 0x94, 0xe2, 0xa0, 0x9a, 0xd6, 0x31, 0x48, 0xbd, 0x12, 0x7f, 0x84, 0x07, 0x05, 0x24, 0x62, 0x1e, - 0x09, 0x7d, 0xda, 0x66, 0x38, 0x75, 0x50, 0xcb, 0xeb, 0x18, 0x64, 0xaa, 0x16, 0xef, 0xc3, 0x32, - 0x4b, 0x12, 0x9e, 0x4c, 0xc8, 0xe6, 0x14, 0xd9, 0xe3, 0x3a, 0xd9, 0x7e, 0x39, 0xa9, 0x63, 0x90, - 0x6a, 0xd5, 0xae, 0x05, 0xf3, 0xdf, 0x53, 0xab, 0xbc, 0x1f, 0x08, 0x56, 0xaa, 0x6e, 0xe0, 0x75, - 0x98, 0x4f, 0xdd, 0xc8, 0x4f, 0x9c, 0x0e, 0xf0, 0x2b, 0xb0, 0xb2, 0x23, 0x61, 0x9b, 0x6e, 0xe3, - 0x2e, 0x4b, 0x95, 0xe7, 0x63, 0x0f, 0x5a, 0xf9, 0x91, 0x3b, 0xa2, 0xb2, 0x67, 0x37, 0x14, 0x6f, - 0x05, 0xf3, 0x7e, 0x22, 0x58, 0x9b, 0x61, 0xe9, 0xfd, 0x88, 0xf9, 0x85, 0xf4, 0xc6, 0xaa, 0xaf, - 0xc8, 0xfd, 0xa8, 0x79, 0x01, 0x0f, 0xa7, 0x56, 0x34, 0x55, 0xa2, 0x56, 0x34, 0xbb, 0xf3, 0x75, - 0xe0, 0x9d, 0xea, 0xc5, 0xd4, 0xbd, 0xba, 0xd1, 0x39, 0xaf, 0xdd, 0xf3, 0x68, 0xea, 0x9e, 0x9f, - 0xba, 0x99, 0xcd, 0x19, 0x37, 0xf3, 0xee, 0xdb, 0xbf, 0xb7, 0x0e, 0xba, 0xbe, 0x75, 0xd0, 0xbf, - 0x5b, 0x07, 0xfd, 0x1e, 0x3b, 0xc6, 0xf5, 0xd8, 0x31, 0x6e, 0xc6, 0x8e, 0xf1, 0xe5, 0xf9, 0x1d, - 0xbf, 0x6d, 0x67, 0x0b, 0xea, 0xe7, 0xe5, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0xbc, 0xc0, 0xf7, - 0x30, 0x0d, 0x07, 0x00, 0x00, + // 681 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0x4d, 0x6f, 0xd3, 0x40, + 0x10, 0xf5, 0x3a, 0x69, 0xdd, 0x4c, 0xd3, 0x16, 0xb6, 0x3d, 0x58, 0x15, 0x18, 0xcb, 0x42, 0x90, + 0x0b, 0xad, 0x54, 0x4e, 0x20, 0xa4, 0x8a, 0x86, 0x96, 0x44, 0x15, 0xa8, 0xda, 0x96, 0x22, 0x71, + 0xdb, 0xda, 0xdb, 0xc6, 0x28, 0xf1, 0x1a, 0xef, 0x86, 0x2a, 0x3f, 0x80, 0x0b, 0x48, 0x15, 0x7f, + 0x89, 0x1b, 0xc7, 0x1e, 0x7b, 0x44, 0xcd, 0x1f, 0x41, 0xde, 0x8d, 0xe3, 0x8f, 0xe4, 0xd0, 0x5b, + 0x2f, 0x4e, 0xe6, 0x79, 0xe6, 0xcd, 0x9b, 0xb7, 0x1f, 0x86, 0x5d, 0x9f, 0x0f, 0x06, 0x3c, 0x12, + 0x31, 0xf5, 0xd9, 0x36, 0x3f, 0xfb, 0xca, 0x7c, 0xb9, 0x2d, 0x13, 0xc6, 0xd4, 0xc3, 0xef, 0xd1, + 0xe8, 0x82, 0xc5, 0x09, 0x97, 0x7c, 0x5b, 0x3d, 0x45, 0x01, 0xde, 0x52, 0x08, 0x86, 0x1c, 0xf1, + 0x6e, 0x10, 0x00, 0xe1, 0x5c, 0xb6, 0x55, 0x88, 0x1f, 0x41, 0x83, 0xfa, 0xfd, 0x0e, 0xa3, 0x41, + 0x37, 0xb0, 0x91, 0x8b, 0x5a, 0x0d, 0x92, 0x03, 0xd8, 0x06, 0x4b, 0x75, 0xed, 0x06, 0xb6, 0xa9, + 0xde, 0x65, 0x21, 0x76, 0x00, 0x34, 0xe1, 0xc9, 0x28, 0x66, 0x76, 0x4d, 0xbd, 0x2c, 0x20, 0x29, + 0xaf, 0x0c, 0x07, 0x4c, 0x48, 0x3a, 0x88, 0xed, 0xba, 0x8b, 0x5a, 0x35, 0x92, 0x03, 0x18, 0x43, + 0x5d, 0x30, 0x16, 0xd8, 0x0b, 0x2e, 0x6a, 0x35, 0x89, 0xfa, 0x8f, 0x37, 0x61, 0x29, 0x0c, 0x58, + 0x24, 0x43, 0x39, 0xb2, 0x17, 0x15, 0x3e, 0x8d, 0xf1, 0x53, 0x58, 0xd1, 0xdc, 0x47, 0x74, 0xd4, + 0xe7, 0x34, 0xb0, 0x2d, 0x95, 0x50, 0x06, 0xbd, 0x2b, 0x13, 0xe0, 0x24, 0x61, 0x6c, 0x32, 0x9a, + 0x0b, 0xcb, 0xe9, 0xdc, 0x7a, 0x14, 0x61, 0x23, 0xb7, 0xd6, 0x6a, 0x90, 0x22, 0x54, 0x1e, 0xde, + 0xac, 0x0e, 0xff, 0x0c, 0x56, 0x45, 0x44, 0x63, 0xd1, 0xe3, 0x72, 0x8f, 0x8a, 0xd4, 0x03, 0x3d, + 0x66, 0x05, 0x4d, 0xfb, 0x68, 0x1d, 0xe2, 0x1d, 0x95, 0x54, 0x0d, 0xdb, 0x24, 0x45, 0x28, 0xed, + 0x93, 0x30, 0x1a, 0x1c, 0xb2, 0x51, 0x57, 0xcf, 0xdc, 0x20, 0x39, 0x50, 0xb6, 0x6a, 0xb1, 0x6a, + 0x55, 0xd1, 0x16, 0xab, 0x62, 0x8b, 0x03, 0x10, 0x8a, 0xe3, 0x89, 0x1a, 0x7b, 0xc9, 0x45, 0xad, + 0x25, 0x52, 0x40, 0xbc, 0xf7, 0xb0, 0x42, 0xe8, 0x65, 0xc1, 0x12, 0x1b, 0xac, 0x78, 0xe2, 0x20, + 0x52, 0x5c, 0x59, 0x98, 0x8a, 0x10, 0xe1, 0x45, 0x44, 0xe5, 0x30, 0x61, 0xca, 0x8a, 0x26, 0xc9, + 0x01, 0xaf, 0x0d, 0xeb, 0x25, 0xa2, 0xcf, 0xa1, 0xec, 0x69, 0xe5, 0x09, 0xbd, 0xd4, 0xd0, 0x84, + 0x30, 0x07, 0xf0, 0x2a, 0x98, 0x61, 0x66, 0xab, 0x19, 0x06, 0xde, 0x15, 0x82, 0xb5, 0x94, 0xe2, + 0x78, 0x14, 0xf9, 0x1f, 0x98, 0x10, 0xf4, 0x82, 0xe1, 0xd7, 0x60, 0xf9, 0x3c, 0x92, 0x2c, 0x92, + 0xaa, 0x7e, 0x79, 0xc7, 0xdd, 0x2a, 0xec, 0xde, 0x2c, 0xbb, 0xad, 0x53, 0x4e, 0x69, 0x7f, 0xc8, + 0x48, 0x56, 0x80, 0x77, 0x01, 0x92, 0xe9, 0x46, 0x56, 0x7d, 0x96, 0x77, 0x9e, 0x14, 0xcb, 0xe7, + 0x48, 0x26, 0x85, 0x12, 0xef, 0x8f, 0x09, 0x1b, 0xf3, 0x5a, 0xe0, 0x37, 0x00, 0x3d, 0x46, 0x83, + 0x4f, 0x71, 0x40, 0x25, 0x9b, 0x08, 0xdb, 0xac, 0x0a, 0xeb, 0x4c, 0x33, 0x3a, 0x06, 0x29, 0xe4, + 0xe3, 0x43, 0x58, 0x3b, 0x1f, 0xf6, 0xfb, 0x29, 0x2b, 0x61, 0xdf, 0x86, 0x4c, 0xc8, 0x79, 0xe2, + 0x52, 0x8a, 0x83, 0x72, 0x5a, 0xc7, 0x20, 0xd5, 0x4a, 0xfc, 0x11, 0x1e, 0xe4, 0x90, 0x88, 0x79, + 0x24, 0xf4, 0x69, 0x9b, 0xe3, 0xd4, 0x41, 0x25, 0xaf, 0x63, 0x90, 0x99, 0x5a, 0xbc, 0x0f, 0x2b, + 0x2c, 0x49, 0x78, 0x32, 0x25, 0xab, 0x2b, 0xb2, 0xc7, 0x55, 0xb2, 0xfd, 0x62, 0x52, 0xc7, 0x20, + 0xe5, 0xaa, 0x3d, 0x0b, 0x16, 0xbe, 0xa7, 0x56, 0x79, 0x3f, 0x10, 0xac, 0x96, 0xdd, 0xc0, 0x1b, + 0xb0, 0x90, 0xba, 0x91, 0x9d, 0x38, 0x1d, 0xe0, 0x57, 0x60, 0x4d, 0x8e, 0x84, 0x6d, 0xba, 0xb5, + 0xbb, 0x2c, 0x55, 0x96, 0x8f, 0x3d, 0x68, 0x66, 0x47, 0xee, 0x88, 0xca, 0x9e, 0x5d, 0x53, 0xbc, + 0x25, 0xcc, 0xfb, 0x89, 0x60, 0x7d, 0x8e, 0xa5, 0xf7, 0x23, 0xe6, 0x17, 0xd2, 0x1b, 0xab, 0xba, + 0x22, 0xf7, 0xa3, 0xe6, 0x05, 0x3c, 0x9c, 0x59, 0xd1, 0xf4, 0x26, 0x60, 0x49, 0xd2, 0xe6, 0x81, + 0xde, 0xdf, 0x75, 0x92, 0x85, 0xde, 0xa9, 0x5e, 0x50, 0xdd, 0xaf, 0x1b, 0x9d, 0xf3, 0xca, 0x5d, + 0x8f, 0x66, 0xee, 0xfa, 0x99, 0xdb, 0xd9, 0x9c, 0x73, 0x3b, 0xef, 0xbd, 0xfd, 0x7b, 0xeb, 0xa0, + 0xeb, 0x5b, 0x07, 0xfd, 0xbb, 0x75, 0xd0, 0xef, 0xb1, 0x63, 0x5c, 0x8f, 0x1d, 0xe3, 0x66, 0xec, + 0x18, 0x5f, 0x9e, 0xdf, 0xf1, 0xfb, 0x76, 0xb6, 0xa8, 0x7e, 0x5e, 0xfe, 0x0f, 0x00, 0x00, 0xff, + 0xff, 0x28, 0x34, 0x11, 0x90, 0x11, 0x07, 0x00, 0x00, } func (m *RootChange) Marshal() (dAtA []byte, err error) { @@ -1426,12 +1426,10 @@ func (m *TreeErrorResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Error) > 0 { - i -= len(m.Error) - copy(dAtA[i:], m.Error) - i = encodeVarintTreechange(dAtA, i, uint64(len(m.Error))) + if m.ErrCode != 0 { + i = encodeVarintTreechange(dAtA, i, uint64(m.ErrCode)) i-- - dAtA[i] = 0xa + dAtA[i] = 0x8 } return len(dAtA) - i, nil } @@ -1759,9 +1757,8 @@ func (m *TreeErrorResponse) Size() (n int) { } var l int _ = l - l = len(m.Error) - if l > 0 { - n += 1 + l + sovTreechange(uint64(l)) + if m.ErrCode != 0 { + n += 1 + sovTreechange(uint64(m.ErrCode)) } return n } @@ -3361,10 +3358,10 @@ func (m *TreeErrorResponse) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ErrCode", wireType) } - var stringLen uint64 + m.ErrCode = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowTreechange @@ -3374,24 +3371,11 @@ func (m *TreeErrorResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + m.ErrCode |= uint64(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthTreechange - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthTreechange - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Error = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipTreechange(dAtA[iNdEx:]) diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index 74b3f7fa..450a1ce5 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -2,6 +2,9 @@ package objectsync import ( "context" + "github.com/anytypeio/any-sync/commonspace/object/tree/synctree" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/gogo/protobuf/proto" "sync/atomic" "time" @@ -23,7 +26,6 @@ type ObjectSync interface { synchandler.SyncHandler MessagePool() MessagePool - Init() Close() (err error) } @@ -31,6 +33,7 @@ type objectSync struct { spaceId string messagePool MessagePool + syncClient synctree.SyncClient objectGetter syncobjectgetter.SyncObjectGetter configuration nodeconf.Configuration spaceStorage spacestorage.SpaceStorage @@ -48,40 +51,18 @@ func NewObjectSync( objectGetter syncobjectgetter.SyncObjectGetter, storage spacestorage.SpaceStorage) ObjectSync { syncCtx, cancel := context.WithCancel(context.Background()) - os := newObjectSync( - spaceId, - spaceIsDeleted, - configuration, - objectGetter, - storage, - syncCtx, - cancel) - msgPool := newMessagePool(peerManager, os.handleMessage) - os.messagePool = msgPool - return os -} - -func newObjectSync( - spaceId string, - spaceIsDeleted *atomic.Bool, - configuration nodeconf.Configuration, - objectGetter syncobjectgetter.SyncObjectGetter, - spaceStorage spacestorage.SpaceStorage, - syncCtx context.Context, - cancel context.CancelFunc, -) *objectSync { - return &objectSync{ + os := &objectSync{ objectGetter: objectGetter, - spaceStorage: spaceStorage, + spaceStorage: storage, spaceId: spaceId, syncCtx: syncCtx, cancelSync: cancel, spaceIsDeleted: spaceIsDeleted, configuration: configuration, + syncClient: synctree.NewSyncClient(spaceId, peerManager, synctree.GetRequestFactory()), } -} - -func (s *objectSync) Init() { + os.messagePool = newMessagePool(peerManager, os.handleMessage) + return os } func (s *objectSync) Close() (err error) { @@ -109,6 +90,10 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp log.With(zap.String("objectId", msg.ObjectId), zap.String("replyId", msg.ReplyId)).DebugCtx(ctx, "handling message") obj, err := s.objectGetter.GetObject(ctx, msg.ObjectId) if err != nil { + respErr := s.sendErrorResponse(ctx, msg, senderId, err) + if respErr != nil { + log.Debug("failed to send error response", zap.Error(respErr)) + } return } return obj.HandleMessage(ctx, senderId, msg) @@ -117,3 +102,13 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp func (s *objectSync) MessagePool() MessagePool { return s.messagePool } + +func (s *objectSync) sendErrorResponse(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, senderId string, respErr error) (err error) { + unmarshalled := &treechangeproto.TreeSyncMessage{} + err = proto.Unmarshal(msg.Payload, unmarshalled) + if err != nil { + return + } + resp := treechangeproto.WrapError(respErr, unmarshalled.RootChange) + return s.syncClient.SendWithReply(ctx, senderId, resp, msg.ReplyId) +} diff --git a/commonspace/space.go b/commonspace/space.go index c68ff8e3..f69b4aaa 100644 --- a/commonspace/space.go +++ b/commonspace/space.go @@ -204,7 +204,6 @@ func (s *space) Init(ctx context.Context) (err error) { OnSpaceDelete: s.onSpaceDelete, } s.settingsObject = settings.NewSettingsObject(deps, s.id) - s.objectSync.Init() s.headSync.Init(initialIds, deletionState) err = s.settingsObject.Init(ctx) if err != nil { From 58f3c4ef8c482c0c4482dddd93e9363f69eb9204 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 18 Apr 2023 22:51:12 +0200 Subject: [PATCH 13/27] Fix objectsync etc --- commonspace/config.go | 5 +- .../synctree/mock_synctree/mock_synctree.go | 111 +------------- commonspace/object/tree/synctree/synctree.go | 7 +- .../object/tree/synctree/synctree_test.go | 10 +- .../object/tree/synctree/synctreehandler.go | 7 +- .../tree/synctree/synctreehandler_test.go | 29 ++-- .../object/tree/synctree/treeremotegetter.go | 5 +- .../object/tree/synctree/utils_test.go | 25 ++-- .../object/tree/treechangeproto/errors.go | 14 ++ .../treechangeproto/protos/treechange.proto | 7 + .../object/tree/treechangeproto/treechange.go | 4 +- .../tree/treechangeproto/treechange.pb.go | 123 ++++++++++------ .../mock_treemanager/mock_treemanager.go | 30 ---- commonspace/object/treemanager/treemanager.go | 2 - .../mock_objectsync/mock_objectsync.go | 138 ++++++++++++++++++ commonspace/objectsync/objectsync.go | 12 +- .../synctree => objectsync}/requestfactory.go | 2 +- .../synctree => objectsync}/syncclient.go | 24 +-- commonspace/settings/deletionmanager.go | 4 - commonspace/settings/deletionmanager_test.go | 2 - commonspace/space.go | 5 +- commonspace/spaceservice.go | 8 + 22 files changed, 320 insertions(+), 254 deletions(-) create mode 100644 commonspace/object/tree/treechangeproto/errors.go create mode 100644 commonspace/objectsync/mock_objectsync/mock_objectsync.go rename commonspace/{object/tree/synctree => objectsync}/requestfactory.go (99%) rename commonspace/{object/tree/synctree => objectsync}/syncclient.go (70%) diff --git a/commonspace/config.go b/commonspace/config.go index befa3ac8..f7444fdd 100644 --- a/commonspace/config.go +++ b/commonspace/config.go @@ -5,6 +5,7 @@ type ConfigGetter interface { } type Config struct { - GCTTL int `yaml:"gcTTL"` - SyncPeriod int `yaml:"syncPeriod"` + GCTTL int `yaml:"gcTTL"` + SyncPeriod int `yaml:"syncPeriod"` + TreeInMemoryData bool `yaml:"treeInMemoryData"` } diff --git a/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go b/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go index 3ab6cf1b..07eb8878 100644 --- a/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go +++ b/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/anytypeio/any-sync/commonspace/object/tree/synctree (interfaces: SyncClient,SyncTree,ReceiveQueue,HeadNotifiable) +// Source: github.com/anytypeio/any-sync/commonspace/object/tree/synctree (interfaces: SyncTree,ReceiveQueue,HeadNotifiable) // Package mock_synctree is a generated GoMock package. package mock_synctree @@ -18,115 +18,6 @@ import ( gomock "github.com/golang/mock/gomock" ) -// MockSyncClient is a mock of SyncClient interface. -type MockSyncClient struct { - ctrl *gomock.Controller - recorder *MockSyncClientMockRecorder -} - -// MockSyncClientMockRecorder is the mock recorder for MockSyncClient. -type MockSyncClientMockRecorder struct { - mock *MockSyncClient -} - -// NewMockSyncClient creates a new mock instance. -func NewMockSyncClient(ctrl *gomock.Controller) *MockSyncClient { - mock := &MockSyncClient{ctrl: ctrl} - mock.recorder = &MockSyncClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockSyncClient) EXPECT() *MockSyncClientMockRecorder { - return m.recorder -} - -// Broadcast mocks base method. -func (m *MockSyncClient) Broadcast(arg0 context.Context, arg1 *treechangeproto.TreeSyncMessage) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Broadcast", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// Broadcast indicates an expected call of Broadcast. -func (mr *MockSyncClientMockRecorder) Broadcast(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Broadcast", reflect.TypeOf((*MockSyncClient)(nil).Broadcast), arg0, arg1) -} - -// CreateFullSyncRequest mocks base method. -func (m *MockSyncClient) CreateFullSyncRequest(arg0 objecttree.ObjectTree, arg1, arg2 []string) (*treechangeproto.TreeSyncMessage, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateFullSyncRequest", arg0, arg1, arg2) - ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateFullSyncRequest indicates an expected call of CreateFullSyncRequest. -func (mr *MockSyncClientMockRecorder) CreateFullSyncRequest(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFullSyncRequest", reflect.TypeOf((*MockSyncClient)(nil).CreateFullSyncRequest), arg0, arg1, arg2) -} - -// CreateFullSyncResponse mocks base method. -func (m *MockSyncClient) CreateFullSyncResponse(arg0 objecttree.ObjectTree, arg1, arg2 []string) (*treechangeproto.TreeSyncMessage, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateFullSyncResponse", arg0, arg1, arg2) - ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateFullSyncResponse indicates an expected call of CreateFullSyncResponse. -func (mr *MockSyncClientMockRecorder) CreateFullSyncResponse(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFullSyncResponse", reflect.TypeOf((*MockSyncClient)(nil).CreateFullSyncResponse), arg0, arg1, arg2) -} - -// CreateHeadUpdate mocks base method. -func (m *MockSyncClient) CreateHeadUpdate(arg0 objecttree.ObjectTree, arg1 []*treechangeproto.RawTreeChangeWithId) *treechangeproto.TreeSyncMessage { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateHeadUpdate", arg0, arg1) - ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) - return ret0 -} - -// CreateHeadUpdate indicates an expected call of CreateHeadUpdate. -func (mr *MockSyncClientMockRecorder) CreateHeadUpdate(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateHeadUpdate", reflect.TypeOf((*MockSyncClient)(nil).CreateHeadUpdate), arg0, arg1) -} - -// CreateNewTreeRequest mocks base method. -func (m *MockSyncClient) CreateNewTreeRequest() *treechangeproto.TreeSyncMessage { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateNewTreeRequest") - ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) - return ret0 -} - -// CreateNewTreeRequest indicates an expected call of CreateNewTreeRequest. -func (mr *MockSyncClientMockRecorder) CreateNewTreeRequest() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateNewTreeRequest", reflect.TypeOf((*MockSyncClient)(nil).CreateNewTreeRequest)) -} - -// SendWithReply mocks base method. -func (m *MockSyncClient) SendWithReply(arg0 context.Context, arg1 string, arg2 *treechangeproto.TreeSyncMessage, arg3 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendWithReply", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(error) - return ret0 -} - -// SendWithReply indicates an expected call of SendWithReply. -func (mr *MockSyncClientMockRecorder) SendWithReply(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendWithReply", reflect.TypeOf((*MockSyncClient)(nil).SendWithReply), arg0, arg1, arg2, arg3) -} - // MockSyncTree is a mock of SyncTree interface. type MockSyncTree struct { ctrl *gomock.Controller diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index 89787597..683079f5 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -1,3 +1,4 @@ +//go:generate mockgen -destination mock_synctree/mock_synctree.go github.com/anytypeio/any-sync/commonspace/object/tree/synctree SyncTree,ReceiveQueue,HeadNotifiable package synctree import ( @@ -43,7 +44,7 @@ type SyncTree interface { type syncTree struct { objecttree.ObjectTree synchandler.SyncHandler - syncClient SyncClient + syncClient objectsync.SyncClient syncStatus syncstatus.StatusUpdater notifiable HeadNotifiable listener updatelistener.UpdateListener @@ -54,7 +55,7 @@ type syncTree struct { var log = logger.NewNamed("common.commonspace.synctree") -var createSyncClient = NewSyncClient +var createSyncClient = objectsync.NewSyncClient type ResponsiblePeersGetter interface { GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) @@ -101,7 +102,7 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t Sy syncClient := createSyncClient( deps.SpaceId, deps.ObjectSync.MessagePool(), - sharedFactory) + objectsync.GetRequestFactory()) syncTree := &syncTree{ ObjectTree: objTree, syncClient: syncClient, diff --git a/commonspace/object/tree/synctree/synctree_test.go b/commonspace/object/tree/synctree/synctree_test.go index 8854a702..cfd56a8d 100644 --- a/commonspace/object/tree/synctree/synctree_test.go +++ b/commonspace/object/tree/synctree/synctree_test.go @@ -4,11 +4,11 @@ import ( "context" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree/mock_objecttree" - "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/mock_synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener/mock_updatelistener" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/objectsync" + "github.com/anytypeio/any-sync/commonspace/objectsync/mock_objectsync" "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/anytypeio/any-sync/nodeconf" "github.com/golang/mock/gomock" @@ -18,7 +18,7 @@ import ( type syncTreeMatcher struct { objTree objecttree.ObjectTree - client SyncClient + client objectsync.SyncClient listener updatelistener.UpdateListener } @@ -34,8 +34,8 @@ func (s syncTreeMatcher) String() string { return "" } -func syncClientFuncCreator(client SyncClient) func(spaceId string, factory RequestFactory, objectSync objectsync.ObjectSync, configuration nodeconf.Configuration) SyncClient { - return func(spaceId string, factory RequestFactory, objectSync objectsync.ObjectSync, configuration nodeconf.Configuration) SyncClient { +func syncClientFuncCreator(client objectsync.SyncClient) func(spaceId string, factory objectsync.RequestFactory, objectSync objectsync.ObjectSync, configuration nodeconf.Configuration) objectsync.SyncClient { + return func(spaceId string, factory objectsync.RequestFactory, objectSync objectsync.ObjectSync, configuration nodeconf.Configuration) objectsync.SyncClient { return client } } @@ -46,7 +46,7 @@ func Test_BuildSyncTree(t *testing.T) { defer ctrl.Finish() updateListenerMock := mock_updatelistener.NewMockUpdateListener(ctrl) - syncClientMock := mock_synctree.NewMockSyncClient(ctrl) + syncClientMock := mock_objectsync.NewMockSyncClient(ctrl) objTreeMock := newTestObjMock(mock_objecttree.NewMockObjectTree(ctrl)) tr := &syncTree{ ObjectTree: objTreeMock, diff --git a/commonspace/object/tree/synctree/synctreehandler.go b/commonspace/object/tree/synctree/synctreehandler.go index 63b708d9..7865f2a7 100644 --- a/commonspace/object/tree/synctree/synctreehandler.go +++ b/commonspace/object/tree/synctree/synctreehandler.go @@ -4,6 +4,7 @@ import ( "context" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/commonspace/objectsync" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/commonspace/syncstatus" @@ -15,7 +16,7 @@ import ( type syncTreeHandler struct { objTree objecttree.ObjectTree - syncClient SyncClient + syncClient objectsync.SyncClient syncStatus syncstatus.StatusUpdater handlerLock sync.Mutex spaceId string @@ -24,7 +25,7 @@ type syncTreeHandler struct { const maxQueueSize = 5 -func newSyncTreeHandler(spaceId string, objTree objecttree.ObjectTree, syncClient SyncClient, syncStatus syncstatus.StatusUpdater) synchandler.SyncHandler { +func newSyncTreeHandler(spaceId string, objTree objecttree.ObjectTree, syncClient objectsync.SyncClient, syncStatus syncstatus.StatusUpdater) synchandler.SyncHandler { return &syncTreeHandler{ objTree: objTree, syncClient: syncClient, @@ -166,7 +167,7 @@ func (s *syncTreeHandler) handleFullSyncRequest( defer func() { if err != nil { log.With(zap.Error(err)).DebugCtx(ctx, "full sync request finished with error") - s.syncClient.SendWithReply(ctx, senderId, treechangeproto.WrapError(err, header), replyId) + s.syncClient.SendWithReply(ctx, senderId, treechangeproto.WrapError(treechangeproto.ErrorCodes_FullSyncRequestError, header), replyId) return } else if fullResponse != nil { cnt := fullResponse.Content.GetFullSyncResponse() diff --git a/commonspace/object/tree/synctree/synctreehandler_test.go b/commonspace/object/tree/synctree/synctreehandler_test.go index f655b34a..b95ffc82 100644 --- a/commonspace/object/tree/synctree/synctreehandler_test.go +++ b/commonspace/object/tree/synctree/synctreehandler_test.go @@ -3,13 +3,14 @@ package synctree import ( "context" "fmt" + "github.com/anytypeio/any-sync/commonspace/objectsync" + "github.com/anytypeio/any-sync/commonspace/objectsync/mock_objectsync" "sync" "testing" "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree/mock_objecttree" - "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/mock_synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/golang/mock/gomock" @@ -54,7 +55,7 @@ func (t *testObjTreeMock) TryRLock() bool { type syncHandlerFixture struct { ctrl *gomock.Controller - syncClientMock *mock_synctree.MockSyncClient + syncClientMock *mock_objectsync.MockSyncClient objectTreeMock *testObjTreeMock receiveQueueMock ReceiveQueue @@ -63,7 +64,7 @@ type syncHandlerFixture struct { func newSyncHandlerFixture(t *testing.T) *syncHandlerFixture { ctrl := gomock.NewController(t) - syncClientMock := mock_synctree.NewMockSyncClient(ctrl) + syncClientMock := mock_objectsync.NewMockSyncClient(ctrl) objectTreeMock := newTestObjMock(mock_objecttree.NewMockObjectTree(ctrl)) receiveQueue := newReceiveQueue(5) @@ -109,7 +110,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2"}).Times(2) @@ -138,7 +139,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2"}).AnyTimes() @@ -171,7 +172,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h1"}).AnyTimes() @@ -192,7 +193,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2"}).AnyTimes() @@ -217,7 +218,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h1"}).AnyTimes() @@ -250,7 +251,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Header().Return(nil) @@ -283,7 +284,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT(). Id().AnyTimes().Return(treeId) @@ -312,7 +313,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") objectMsg.RequestId = replyId fx.objectTreeMock.EXPECT(). @@ -339,7 +340,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT(). Id().AnyTimes().Return(treeId) @@ -380,7 +381,7 @@ func TestSyncHandler_HandleFullSyncResponse(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullResponse(fullSyncResponse, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, replyId) + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, replyId) fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT(). @@ -413,7 +414,7 @@ func TestSyncHandler_HandleFullSyncResponse(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullResponse(fullSyncResponse, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, replyId) + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, replyId) fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT(). diff --git a/commonspace/object/tree/synctree/treeremotegetter.go b/commonspace/object/tree/synctree/treeremotegetter.go index 9fd0677a..9f819c41 100644 --- a/commonspace/object/tree/synctree/treeremotegetter.go +++ b/commonspace/object/tree/synctree/treeremotegetter.go @@ -6,6 +6,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/commonspace/objectsync" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/net/peer" "github.com/anytypeio/any-sync/net/rpc/rpcerr" @@ -46,8 +47,8 @@ func (t treeRemoteGetter) getPeers(ctx context.Context) (peerIds []string, err e } func (t treeRemoteGetter) treeRequest(ctx context.Context, peerId string) (msg *treechangeproto.TreeSyncMessage, err error) { - newTreeRequest := GetRequestFactory().CreateNewTreeRequest() - objMsg, err := marshallTreeMessage(newTreeRequest, t.deps.SpaceId, t.treeId, "") + newTreeRequest := objectsync.GetRequestFactory().CreateNewTreeRequest() + objMsg, err := objectsync.MarshallTreeMessage(newTreeRequest, t.deps.SpaceId, t.treeId, "") if err != nil { return } diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index 524ab1d2..8e2ffbe9 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -7,6 +7,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/commonspace/objectsync" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/commonspace/syncstatus" @@ -84,7 +85,7 @@ type processSyncHandler struct { peerId string aclList list.AclList log *messageLog - syncClient SyncClient + syncClient objectsync.SyncClient } func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { @@ -97,9 +98,9 @@ func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, return } if unmarshalled.Content.GetFullSyncResponse() == nil { - newTreeRequest := GetRequestFactory().CreateNewTreeRequest() + newTreeRequest := objectsync.GetRequestFactory().CreateNewTreeRequest() var objMsg *spacesyncproto.ObjectSyncMessage - objMsg, err = marshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") + objMsg, err = objectsync.MarshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") if err != nil { return } @@ -124,8 +125,8 @@ func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, } p.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, p.syncClient, syncstatus.NewNoOpSyncStatus()) var objMsg *spacesyncproto.ObjectSyncMessage - newTreeRequest := GetRequestFactory().CreateHeadUpdate(netTree, res.Added) - objMsg, err = marshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") + newTreeRequest := objectsync.GetRequestFactory().CreateHeadUpdate(netTree, res.Added) + objMsg, err = objectsync.MarshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") if err != nil { return } @@ -143,9 +144,9 @@ func newProcessSyncHandler(peerId string, syncHandler synchandler.SyncHandler) * func (p *processSyncHandler) manager() *processPeerManager { if p.SyncHandler != nil { - return p.SyncHandler.(*syncTreeHandler).syncClient.(*syncClient).PeerManager.(*processPeerManager) + return p.SyncHandler.(*syncTreeHandler).syncClient.PeerManager().(*processPeerManager) } - return p.syncClient.(*syncClient).PeerManager.(*processPeerManager) + return p.syncClient.PeerManager().(*processPeerManager) } func (p *processSyncHandler) tree() *broadcastTree { @@ -229,7 +230,7 @@ func (m *processPeerManager) GetResponsiblePeers(ctx context.Context) (peers []p type broadcastTree struct { objecttree.ObjectTree - SyncClient + objectsync.SyncClient } func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) (objecttree.AddResult, error) { @@ -243,8 +244,8 @@ func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.Ra } func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *processSyncHandler { - factory := GetRequestFactory() - syncClient := NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) + factory := objectsync.GetRequestFactory() + syncClient := objectsync.NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) netTree := &broadcastTree{ ObjectTree: objTree, SyncClient: syncClient, @@ -254,8 +255,8 @@ func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, lo } func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *processSyncHandler { - factory := GetRequestFactory() - syncClient := NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) + factory := objectsync.GetRequestFactory() + syncClient := objectsync.NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) batcher := mb.New[processMsg](0) return &processSyncHandler{ diff --git a/commonspace/object/tree/treechangeproto/errors.go b/commonspace/object/tree/treechangeproto/errors.go new file mode 100644 index 00000000..375ebedb --- /dev/null +++ b/commonspace/object/tree/treechangeproto/errors.go @@ -0,0 +1,14 @@ +package treechangeproto + +import ( + "errors" + "github.com/anytypeio/any-sync/net/rpc/rpcerr" +) + +var ( + errGroup = rpcerr.ErrGroup(ErrorCodes_ErrorOffset) + + ErrUnexpected = errGroup.Register(errors.New("unexpected error"), uint64(ErrorCodes_Unexpected)) + ErrGetTree = errGroup.Register(errors.New("tree not found"), uint64(ErrorCodes_GetTreeError)) + ErrFullSync = errGroup.Register(errors.New("full sync request error"), uint64(ErrorCodes_FullSyncRequestError)) +) diff --git a/commonspace/object/tree/treechangeproto/protos/treechange.proto b/commonspace/object/tree/treechangeproto/protos/treechange.proto index 1ecd9bd6..ae7cebd5 100644 --- a/commonspace/object/tree/treechangeproto/protos/treechange.proto +++ b/commonspace/object/tree/treechangeproto/protos/treechange.proto @@ -56,6 +56,13 @@ message RawTreeChangeWithId { string id = 2; } +enum ErrorCodes { + Unexpected = 0; + GetTreeError = 1; + FullSyncRequestError = 2; + ErrorOffset = 400; +} + message TreeSyncMessage { TreeSyncContentValue content = 1; RawTreeChangeWithId rootChange = 2; diff --git a/commonspace/object/tree/treechangeproto/treechange.go b/commonspace/object/tree/treechangeproto/treechange.go index 3aba1fb0..0dc5d668 100644 --- a/commonspace/object/tree/treechangeproto/treechange.go +++ b/commonspace/object/tree/treechangeproto/treechange.go @@ -27,10 +27,10 @@ func WrapFullResponse(response *TreeFullSyncResponse, rootChange *RawTreeChangeW } } -func WrapError(err error, rootChange *RawTreeChangeWithId) *TreeSyncMessage { +func WrapError(code ErrorCodes, rootChange *RawTreeChangeWithId) *TreeSyncMessage { return &TreeSyncMessage{ Content: &TreeSyncContentValue{ - Value: &TreeSyncContentValue_ErrorResponse{ErrorResponse: &TreeErrorResponse{Error: err.Error()}}, + Value: &TreeSyncContentValue_ErrorResponse{ErrorResponse: &TreeErrorResponse{ErrCode: uint64(code)}}, }, RootChange: rootChange, } diff --git a/commonspace/object/tree/treechangeproto/treechange.pb.go b/commonspace/object/tree/treechangeproto/treechange.pb.go index 14815df1..f4098fc6 100644 --- a/commonspace/object/tree/treechangeproto/treechange.pb.go +++ b/commonspace/object/tree/treechangeproto/treechange.pb.go @@ -22,6 +22,37 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package +type ErrorCodes int32 + +const ( + ErrorCodes_Unexpected ErrorCodes = 0 + ErrorCodes_GetTreeError ErrorCodes = 1 + ErrorCodes_FullSyncRequestError ErrorCodes = 2 + ErrorCodes_ErrorOffset ErrorCodes = 400 +) + +var ErrorCodes_name = map[int32]string{ + 0: "Unexpected", + 1: "GetTreeError", + 2: "FullSyncRequestError", + 400: "ErrorOffset", +} + +var ErrorCodes_value = map[string]int32{ + "Unexpected": 0, + "GetTreeError": 1, + "FullSyncRequestError": 2, + "ErrorOffset": 400, +} + +func (x ErrorCodes) String() string { + return proto.EnumName(ErrorCodes_name, int32(x)) +} + +func (ErrorCodes) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_5033f0301ef9b772, []int{0} +} + // RootChange is a root of a tree type RootChange struct { // AclHeadId is a cid of latest acl record at the time of tree creation @@ -788,6 +819,7 @@ func (m *TreeChangeInfo) GetChangePayload() []byte { } func init() { + proto.RegisterEnum("treechange.ErrorCodes", ErrorCodes_name, ErrorCodes_value) proto.RegisterType((*RootChange)(nil), "treechange.RootChange") proto.RegisterType((*TreeChange)(nil), "treechange.TreeChange") proto.RegisterType((*RawTreeChange)(nil), "treechange.RawTreeChange") @@ -806,50 +838,53 @@ func init() { } var fileDescriptor_5033f0301ef9b772 = []byte{ - // 681 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0x4d, 0x6f, 0xd3, 0x40, - 0x10, 0xf5, 0x3a, 0x69, 0xdd, 0x4c, 0xd3, 0x16, 0xb6, 0x3d, 0x58, 0x15, 0x18, 0xcb, 0x42, 0x90, - 0x0b, 0xad, 0x54, 0x4e, 0x20, 0xa4, 0x8a, 0x86, 0x96, 0x44, 0x15, 0xa8, 0xda, 0x96, 0x22, 0x71, - 0xdb, 0xda, 0xdb, 0xc6, 0x28, 0xf1, 0x1a, 0xef, 0x86, 0x2a, 0x3f, 0x80, 0x0b, 0x48, 0x15, 0x7f, - 0x89, 0x1b, 0xc7, 0x1e, 0x7b, 0x44, 0xcd, 0x1f, 0x41, 0xde, 0x8d, 0xe3, 0x8f, 0xe4, 0xd0, 0x5b, - 0x2f, 0x4e, 0xe6, 0x79, 0xe6, 0xcd, 0x9b, 0xb7, 0x1f, 0x86, 0x5d, 0x9f, 0x0f, 0x06, 0x3c, 0x12, - 0x31, 0xf5, 0xd9, 0x36, 0x3f, 0xfb, 0xca, 0x7c, 0xb9, 0x2d, 0x13, 0xc6, 0xd4, 0xc3, 0xef, 0xd1, - 0xe8, 0x82, 0xc5, 0x09, 0x97, 0x7c, 0x5b, 0x3d, 0x45, 0x01, 0xde, 0x52, 0x08, 0x86, 0x1c, 0xf1, - 0x6e, 0x10, 0x00, 0xe1, 0x5c, 0xb6, 0x55, 0x88, 0x1f, 0x41, 0x83, 0xfa, 0xfd, 0x0e, 0xa3, 0x41, - 0x37, 0xb0, 0x91, 0x8b, 0x5a, 0x0d, 0x92, 0x03, 0xd8, 0x06, 0x4b, 0x75, 0xed, 0x06, 0xb6, 0xa9, - 0xde, 0x65, 0x21, 0x76, 0x00, 0x34, 0xe1, 0xc9, 0x28, 0x66, 0x76, 0x4d, 0xbd, 0x2c, 0x20, 0x29, - 0xaf, 0x0c, 0x07, 0x4c, 0x48, 0x3a, 0x88, 0xed, 0xba, 0x8b, 0x5a, 0x35, 0x92, 0x03, 0x18, 0x43, - 0x5d, 0x30, 0x16, 0xd8, 0x0b, 0x2e, 0x6a, 0x35, 0x89, 0xfa, 0x8f, 0x37, 0x61, 0x29, 0x0c, 0x58, - 0x24, 0x43, 0x39, 0xb2, 0x17, 0x15, 0x3e, 0x8d, 0xf1, 0x53, 0x58, 0xd1, 0xdc, 0x47, 0x74, 0xd4, - 0xe7, 0x34, 0xb0, 0x2d, 0x95, 0x50, 0x06, 0xbd, 0x2b, 0x13, 0xe0, 0x24, 0x61, 0x6c, 0x32, 0x9a, - 0x0b, 0xcb, 0xe9, 0xdc, 0x7a, 0x14, 0x61, 0x23, 0xb7, 0xd6, 0x6a, 0x90, 0x22, 0x54, 0x1e, 0xde, - 0xac, 0x0e, 0xff, 0x0c, 0x56, 0x45, 0x44, 0x63, 0xd1, 0xe3, 0x72, 0x8f, 0x8a, 0xd4, 0x03, 0x3d, - 0x66, 0x05, 0x4d, 0xfb, 0x68, 0x1d, 0xe2, 0x1d, 0x95, 0x54, 0x0d, 0xdb, 0x24, 0x45, 0x28, 0xed, - 0x93, 0x30, 0x1a, 0x1c, 0xb2, 0x51, 0x57, 0xcf, 0xdc, 0x20, 0x39, 0x50, 0xb6, 0x6a, 0xb1, 0x6a, - 0x55, 0xd1, 0x16, 0xab, 0x62, 0x8b, 0x03, 0x10, 0x8a, 0xe3, 0x89, 0x1a, 0x7b, 0xc9, 0x45, 0xad, - 0x25, 0x52, 0x40, 0xbc, 0xf7, 0xb0, 0x42, 0xe8, 0x65, 0xc1, 0x12, 0x1b, 0xac, 0x78, 0xe2, 0x20, - 0x52, 0x5c, 0x59, 0x98, 0x8a, 0x10, 0xe1, 0x45, 0x44, 0xe5, 0x30, 0x61, 0xca, 0x8a, 0x26, 0xc9, - 0x01, 0xaf, 0x0d, 0xeb, 0x25, 0xa2, 0xcf, 0xa1, 0xec, 0x69, 0xe5, 0x09, 0xbd, 0xd4, 0xd0, 0x84, - 0x30, 0x07, 0xf0, 0x2a, 0x98, 0x61, 0x66, 0xab, 0x19, 0x06, 0xde, 0x15, 0x82, 0xb5, 0x94, 0xe2, - 0x78, 0x14, 0xf9, 0x1f, 0x98, 0x10, 0xf4, 0x82, 0xe1, 0xd7, 0x60, 0xf9, 0x3c, 0x92, 0x2c, 0x92, - 0xaa, 0x7e, 0x79, 0xc7, 0xdd, 0x2a, 0xec, 0xde, 0x2c, 0xbb, 0xad, 0x53, 0x4e, 0x69, 0x7f, 0xc8, - 0x48, 0x56, 0x80, 0x77, 0x01, 0x92, 0xe9, 0x46, 0x56, 0x7d, 0x96, 0x77, 0x9e, 0x14, 0xcb, 0xe7, - 0x48, 0x26, 0x85, 0x12, 0xef, 0x8f, 0x09, 0x1b, 0xf3, 0x5a, 0xe0, 0x37, 0x00, 0x3d, 0x46, 0x83, - 0x4f, 0x71, 0x40, 0x25, 0x9b, 0x08, 0xdb, 0xac, 0x0a, 0xeb, 0x4c, 0x33, 0x3a, 0x06, 0x29, 0xe4, - 0xe3, 0x43, 0x58, 0x3b, 0x1f, 0xf6, 0xfb, 0x29, 0x2b, 0x61, 0xdf, 0x86, 0x4c, 0xc8, 0x79, 0xe2, - 0x52, 0x8a, 0x83, 0x72, 0x5a, 0xc7, 0x20, 0xd5, 0x4a, 0xfc, 0x11, 0x1e, 0xe4, 0x90, 0x88, 0x79, - 0x24, 0xf4, 0x69, 0x9b, 0xe3, 0xd4, 0x41, 0x25, 0xaf, 0x63, 0x90, 0x99, 0x5a, 0xbc, 0x0f, 0x2b, - 0x2c, 0x49, 0x78, 0x32, 0x25, 0xab, 0x2b, 0xb2, 0xc7, 0x55, 0xb2, 0xfd, 0x62, 0x52, 0xc7, 0x20, - 0xe5, 0xaa, 0x3d, 0x0b, 0x16, 0xbe, 0xa7, 0x56, 0x79, 0x3f, 0x10, 0xac, 0x96, 0xdd, 0xc0, 0x1b, - 0xb0, 0x90, 0xba, 0x91, 0x9d, 0x38, 0x1d, 0xe0, 0x57, 0x60, 0x4d, 0x8e, 0x84, 0x6d, 0xba, 0xb5, - 0xbb, 0x2c, 0x55, 0x96, 0x8f, 0x3d, 0x68, 0x66, 0x47, 0xee, 0x88, 0xca, 0x9e, 0x5d, 0x53, 0xbc, - 0x25, 0xcc, 0xfb, 0x89, 0x60, 0x7d, 0x8e, 0xa5, 0xf7, 0x23, 0xe6, 0x17, 0xd2, 0x1b, 0xab, 0xba, - 0x22, 0xf7, 0xa3, 0xe6, 0x05, 0x3c, 0x9c, 0x59, 0xd1, 0xf4, 0x26, 0x60, 0x49, 0xd2, 0xe6, 0x81, - 0xde, 0xdf, 0x75, 0x92, 0x85, 0xde, 0xa9, 0x5e, 0x50, 0xdd, 0xaf, 0x1b, 0x9d, 0xf3, 0xca, 0x5d, - 0x8f, 0x66, 0xee, 0xfa, 0x99, 0xdb, 0xd9, 0x9c, 0x73, 0x3b, 0xef, 0xbd, 0xfd, 0x7b, 0xeb, 0xa0, - 0xeb, 0x5b, 0x07, 0xfd, 0xbb, 0x75, 0xd0, 0xef, 0xb1, 0x63, 0x5c, 0x8f, 0x1d, 0xe3, 0x66, 0xec, - 0x18, 0x5f, 0x9e, 0xdf, 0xf1, 0xfb, 0x76, 0xb6, 0xa8, 0x7e, 0x5e, 0xfe, 0x0f, 0x00, 0x00, 0xff, - 0xff, 0x28, 0x34, 0x11, 0x90, 0x11, 0x07, 0x00, 0x00, + // 732 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0x4f, 0x4f, 0xfb, 0x46, + 0x10, 0xf5, 0x3a, 0x81, 0x90, 0x49, 0x08, 0xe9, 0xc2, 0xc1, 0x42, 0xad, 0x6b, 0x59, 0x55, 0x1b, + 0x55, 0x2a, 0x48, 0xf4, 0xd4, 0xaa, 0x12, 0x2a, 0x29, 0x90, 0x08, 0xb5, 0x45, 0xcb, 0x9f, 0x4a, + 0xdc, 0x16, 0x7b, 0x42, 0x5c, 0x25, 0xb6, 0xeb, 0xdd, 0x94, 0xe6, 0x03, 0xf4, 0xd2, 0x4a, 0x88, + 0xaf, 0xd4, 0x5b, 0x8f, 0x1c, 0x39, 0xfe, 0x04, 0x5f, 0xe4, 0x27, 0xef, 0xc6, 0x89, 0xed, 0xe4, + 0xc0, 0x8d, 0x8b, 0xe3, 0x79, 0x9e, 0x79, 0xf3, 0xe6, 0xed, 0x9f, 0xc0, 0xa1, 0x17, 0x8d, 0xc7, + 0x51, 0x28, 0x62, 0xee, 0xe1, 0x7e, 0x74, 0xfb, 0x3b, 0x7a, 0x72, 0x5f, 0x26, 0x88, 0xea, 0xe1, + 0x0d, 0x79, 0x78, 0x87, 0x71, 0x12, 0xc9, 0x68, 0x5f, 0x3d, 0x45, 0x0e, 0xde, 0x53, 0x08, 0x85, + 0x05, 0xe2, 0x3e, 0x13, 0x00, 0x16, 0x45, 0xb2, 0xab, 0x42, 0xfa, 0x29, 0xd4, 0xb9, 0x37, 0xea, + 0x21, 0xf7, 0xfb, 0xbe, 0x45, 0x1c, 0xd2, 0xa9, 0xb3, 0x05, 0x40, 0x2d, 0xa8, 0xa9, 0xae, 0x7d, + 0xdf, 0x32, 0xd5, 0xb7, 0x2c, 0xa4, 0x36, 0x80, 0x26, 0xbc, 0x9c, 0xc6, 0x68, 0x55, 0xd4, 0xc7, + 0x1c, 0x92, 0xf2, 0xca, 0x60, 0x8c, 0x42, 0xf2, 0x71, 0x6c, 0x55, 0x1d, 0xd2, 0xa9, 0xb0, 0x05, + 0x40, 0x29, 0x54, 0x05, 0xa2, 0x6f, 0xad, 0x39, 0xa4, 0xd3, 0x64, 0xea, 0x9d, 0xee, 0xc2, 0x46, + 0xe0, 0x63, 0x28, 0x03, 0x39, 0xb5, 0xd6, 0x15, 0x3e, 0x8f, 0xe9, 0x17, 0xb0, 0xa9, 0xb9, 0xcf, + 0xf9, 0x74, 0x14, 0x71, 0xdf, 0xaa, 0xa9, 0x84, 0x22, 0xe8, 0x3e, 0x98, 0x00, 0x97, 0x09, 0xe2, + 0x6c, 0x34, 0x07, 0x1a, 0xe9, 0xdc, 0x7a, 0x14, 0x61, 0x11, 0xa7, 0xd2, 0xa9, 0xb3, 0x3c, 0x54, + 0x1c, 0xde, 0x2c, 0x0f, 0xff, 0x25, 0xb4, 0x44, 0xc8, 0x63, 0x31, 0x8c, 0xe4, 0x11, 0x17, 0xa9, + 0x07, 0x7a, 0xcc, 0x12, 0x9a, 0xf6, 0xd1, 0x3a, 0xc4, 0x4f, 0x5c, 0x72, 0x35, 0x6c, 0x93, 0xe5, + 0xa1, 0xb4, 0x4f, 0x82, 0xdc, 0x3f, 0xc3, 0x69, 0x5f, 0xcf, 0x5c, 0x67, 0x0b, 0xa0, 0x68, 0xd5, + 0x7a, 0xd9, 0xaa, 0xbc, 0x2d, 0xb5, 0x92, 0x2d, 0x36, 0x40, 0x20, 0x2e, 0x66, 0x6a, 0xac, 0x0d, + 0x87, 0x74, 0x36, 0x58, 0x0e, 0x71, 0x4f, 0x61, 0x93, 0xf1, 0xfb, 0x9c, 0x25, 0x16, 0xd4, 0xe2, + 0x99, 0x83, 0x44, 0x71, 0x65, 0x61, 0x2a, 0x42, 0x04, 0x77, 0x21, 0x97, 0x93, 0x04, 0x95, 0x15, + 0x4d, 0xb6, 0x00, 0xdc, 0x2e, 0x6c, 0x17, 0x88, 0x7e, 0x0b, 0xe4, 0x50, 0x2b, 0x4f, 0xf8, 0xbd, + 0x86, 0x66, 0x84, 0x0b, 0x80, 0xb6, 0xc0, 0x0c, 0x32, 0x5b, 0xcd, 0xc0, 0x77, 0x1f, 0x08, 0x6c, + 0xa5, 0x14, 0x17, 0xd3, 0xd0, 0xfb, 0x19, 0x85, 0xe0, 0x77, 0x48, 0xbf, 0x87, 0x9a, 0x17, 0x85, + 0x12, 0x43, 0xa9, 0xea, 0x1b, 0x07, 0xce, 0x5e, 0x6e, 0xf7, 0x66, 0xd9, 0x5d, 0x9d, 0x72, 0xcd, + 0x47, 0x13, 0x64, 0x59, 0x01, 0x3d, 0x04, 0x48, 0xe6, 0x1b, 0x59, 0xf5, 0x69, 0x1c, 0x7c, 0x9e, + 0x2f, 0x5f, 0x21, 0x99, 0xe5, 0x4a, 0xdc, 0xff, 0x4c, 0xd8, 0x59, 0xd5, 0x82, 0xfe, 0x00, 0x30, + 0x44, 0xee, 0x5f, 0xc5, 0x3e, 0x97, 0x38, 0x13, 0xb6, 0x5b, 0x16, 0xd6, 0x9b, 0x67, 0xf4, 0x0c, + 0x96, 0xcb, 0xa7, 0x67, 0xb0, 0x35, 0x98, 0x8c, 0x46, 0x29, 0x2b, 0xc3, 0x3f, 0x26, 0x28, 0xe4, + 0x2a, 0x71, 0x29, 0xc5, 0x49, 0x31, 0xad, 0x67, 0xb0, 0x72, 0x25, 0xfd, 0x05, 0xda, 0x0b, 0x48, + 0xc4, 0x51, 0x28, 0xf4, 0x69, 0x5b, 0xe1, 0xd4, 0x49, 0x29, 0xaf, 0x67, 0xb0, 0xa5, 0x5a, 0x7a, + 0x0c, 0x9b, 0x98, 0x24, 0x51, 0x32, 0x27, 0xab, 0x2a, 0xb2, 0xcf, 0xca, 0x64, 0xc7, 0xf9, 0xa4, + 0x9e, 0xc1, 0x8a, 0x55, 0x47, 0x35, 0x58, 0xfb, 0x33, 0xb5, 0xca, 0xfd, 0x9b, 0x40, 0xab, 0xe8, + 0x06, 0xdd, 0x81, 0xb5, 0xd4, 0x8d, 0xec, 0xc4, 0xe9, 0x80, 0x7e, 0x07, 0xb5, 0xd9, 0x91, 0xb0, + 0x4c, 0xa7, 0xf2, 0x96, 0xa5, 0xca, 0xf2, 0xa9, 0x0b, 0xcd, 0xec, 0xc8, 0x9d, 0x73, 0x39, 0xb4, + 0x2a, 0x8a, 0xb7, 0x80, 0xb9, 0xff, 0x10, 0xd8, 0x5e, 0x61, 0xe9, 0xfb, 0x88, 0xf9, 0x97, 0xe8, + 0x8d, 0x55, 0x5e, 0x91, 0xf7, 0x51, 0xf3, 0x0d, 0x7c, 0xb2, 0xb4, 0xa2, 0xe9, 0x4d, 0x80, 0x49, + 0xd2, 0x8d, 0x7c, 0xbd, 0xbf, 0xab, 0x2c, 0x0b, 0xdd, 0x6b, 0xbd, 0xa0, 0xba, 0x5f, 0x3f, 0x1c, + 0x44, 0xa5, 0xbb, 0x9e, 0x2c, 0xdd, 0xf5, 0x4b, 0xb7, 0xb3, 0xb9, 0xe2, 0x76, 0xfe, 0xfa, 0x06, + 0x40, 0x49, 0x48, 0x9b, 0x08, 0xda, 0x02, 0xb8, 0x0a, 0xf1, 0xaf, 0x18, 0x3d, 0x89, 0x7e, 0xdb, + 0xa0, 0x6d, 0x68, 0x9e, 0xa2, 0x9c, 0xeb, 0x6c, 0x13, 0x6a, 0xc1, 0x4e, 0x69, 0x31, 0xf5, 0x17, + 0x93, 0xb6, 0xa1, 0xa1, 0x5e, 0x7f, 0x1d, 0x0c, 0x04, 0xca, 0xf6, 0x63, 0xe5, 0xe8, 0xc7, 0xff, + 0x5f, 0x6c, 0xf2, 0xf4, 0x62, 0x93, 0x0f, 0x2f, 0x36, 0x79, 0x7c, 0xb5, 0x8d, 0xa7, 0x57, 0xdb, + 0x78, 0x7e, 0xb5, 0x8d, 0x9b, 0xaf, 0xde, 0xf8, 0xdf, 0x79, 0xbb, 0xae, 0x7e, 0xbe, 0xfd, 0x18, + 0x00, 0x00, 0xff, 0xff, 0xb5, 0x68, 0xf7, 0x3f, 0x6d, 0x07, 0x00, 0x00, } func (m *RootChange) Marshal() (dAtA []byte, err error) { diff --git a/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go b/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go index f378bc43..68f1d579 100644 --- a/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go +++ b/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go @@ -9,9 +9,7 @@ import ( reflect "reflect" app "github.com/anytypeio/any-sync/app" - list "github.com/anytypeio/any-sync/commonspace/object/acl/list" objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" - treestorage "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" gomock "github.com/golang/mock/gomock" ) @@ -52,20 +50,6 @@ func (mr *MockTreeManagerMockRecorder) Close(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTreeManager)(nil).Close), arg0) } -// DeleteSpace mocks base method. -func (m *MockTreeManager) DeleteSpace(arg0 context.Context, arg1 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteSpace", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteSpace indicates an expected call of DeleteSpace. -func (mr *MockTreeManagerMockRecorder) DeleteSpace(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSpace", reflect.TypeOf((*MockTreeManager)(nil).DeleteSpace), arg0, arg1) -} - // DeleteTree mocks base method. func (m *MockTreeManager) DeleteTree(arg0 context.Context, arg1, arg2 string) error { m.ctrl.T.Helper() @@ -123,20 +107,6 @@ func (mr *MockTreeManagerMockRecorder) Name() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockTreeManager)(nil).Name)) } -// ObjectTreeBuilder mocks base method. -func (m *MockTreeManager) ObjectTreeBuilder() func(treestorage.TreeStorage, list.AclList) (objecttree.ObjectTree, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ObjectTreeBuilder") - ret0, _ := ret[0].(func(treestorage.TreeStorage, list.AclList) (objecttree.ObjectTree, error)) - return ret0 -} - -// ObjectTreeBuilder indicates an expected call of ObjectTreeBuilder. -func (mr *MockTreeManagerMockRecorder) ObjectTreeBuilder() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ObjectTreeBuilder", reflect.TypeOf((*MockTreeManager)(nil).ObjectTreeBuilder)) -} - // Run mocks base method. func (m *MockTreeManager) Run(arg0 context.Context) error { m.ctrl.T.Helper() diff --git a/commonspace/object/treemanager/treemanager.go b/commonspace/object/treemanager/treemanager.go index 00685ff5..0573471e 100644 --- a/commonspace/object/treemanager/treemanager.go +++ b/commonspace/object/treemanager/treemanager.go @@ -13,6 +13,4 @@ type TreeManager interface { app.ComponentRunnable GetTree(ctx context.Context, spaceId, treeId string) (objecttree.ObjectTree, error) DeleteTree(ctx context.Context, spaceId, treeId string) error - DeleteSpace(ctx context.Context, spaceId string) error - ObjectTreeBuilder() objecttree.BuildObjectTreeFunc } diff --git a/commonspace/objectsync/mock_objectsync/mock_objectsync.go b/commonspace/objectsync/mock_objectsync/mock_objectsync.go new file mode 100644 index 00000000..b82360db --- /dev/null +++ b/commonspace/objectsync/mock_objectsync/mock_objectsync.go @@ -0,0 +1,138 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/anytypeio/any-sync/commonspace/objectsync (interfaces: SyncClient) + +// Package mock_objectsync is a generated GoMock package. +package mock_objectsync + +import ( + context "context" + reflect "reflect" + + objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + treechangeproto "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + peermanager "github.com/anytypeio/any-sync/commonspace/peermanager" + gomock "github.com/golang/mock/gomock" +) + +// MockSyncClient is a mock of SyncClient interface. +type MockSyncClient struct { + ctrl *gomock.Controller + recorder *MockSyncClientMockRecorder +} + +// MockSyncClientMockRecorder is the mock recorder for MockSyncClient. +type MockSyncClientMockRecorder struct { + mock *MockSyncClient +} + +// NewMockSyncClient creates a new mock instance. +func NewMockSyncClient(ctrl *gomock.Controller) *MockSyncClient { + mock := &MockSyncClient{ctrl: ctrl} + mock.recorder = &MockSyncClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSyncClient) EXPECT() *MockSyncClientMockRecorder { + return m.recorder +} + +// Broadcast mocks base method. +func (m *MockSyncClient) Broadcast(arg0 context.Context, arg1 *treechangeproto.TreeSyncMessage) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Broadcast", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Broadcast indicates an expected call of Broadcast. +func (mr *MockSyncClientMockRecorder) Broadcast(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Broadcast", reflect.TypeOf((*MockSyncClient)(nil).Broadcast), arg0, arg1) +} + +// CreateFullSyncRequest mocks base method. +func (m *MockSyncClient) CreateFullSyncRequest(arg0 objecttree.ObjectTree, arg1, arg2 []string) (*treechangeproto.TreeSyncMessage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateFullSyncRequest", arg0, arg1, arg2) + ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateFullSyncRequest indicates an expected call of CreateFullSyncRequest. +func (mr *MockSyncClientMockRecorder) CreateFullSyncRequest(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFullSyncRequest", reflect.TypeOf((*MockSyncClient)(nil).CreateFullSyncRequest), arg0, arg1, arg2) +} + +// CreateFullSyncResponse mocks base method. +func (m *MockSyncClient) CreateFullSyncResponse(arg0 objecttree.ObjectTree, arg1, arg2 []string) (*treechangeproto.TreeSyncMessage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateFullSyncResponse", arg0, arg1, arg2) + ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateFullSyncResponse indicates an expected call of CreateFullSyncResponse. +func (mr *MockSyncClientMockRecorder) CreateFullSyncResponse(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFullSyncResponse", reflect.TypeOf((*MockSyncClient)(nil).CreateFullSyncResponse), arg0, arg1, arg2) +} + +// CreateHeadUpdate mocks base method. +func (m *MockSyncClient) CreateHeadUpdate(arg0 objecttree.ObjectTree, arg1 []*treechangeproto.RawTreeChangeWithId) *treechangeproto.TreeSyncMessage { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateHeadUpdate", arg0, arg1) + ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) + return ret0 +} + +// CreateHeadUpdate indicates an expected call of CreateHeadUpdate. +func (mr *MockSyncClientMockRecorder) CreateHeadUpdate(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateHeadUpdate", reflect.TypeOf((*MockSyncClient)(nil).CreateHeadUpdate), arg0, arg1) +} + +// CreateNewTreeRequest mocks base method. +func (m *MockSyncClient) CreateNewTreeRequest() *treechangeproto.TreeSyncMessage { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateNewTreeRequest") + ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) + return ret0 +} + +// CreateNewTreeRequest indicates an expected call of CreateNewTreeRequest. +func (mr *MockSyncClientMockRecorder) CreateNewTreeRequest() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateNewTreeRequest", reflect.TypeOf((*MockSyncClient)(nil).CreateNewTreeRequest)) +} + +// PeerManager mocks base method. +func (m *MockSyncClient) PeerManager() peermanager.PeerManager { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PeerManager") + ret0, _ := ret[0].(peermanager.PeerManager) + return ret0 +} + +// PeerManager indicates an expected call of PeerManager. +func (mr *MockSyncClientMockRecorder) PeerManager() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PeerManager", reflect.TypeOf((*MockSyncClient)(nil).PeerManager)) +} + +// SendWithReply mocks base method. +func (m *MockSyncClient) SendWithReply(arg0 context.Context, arg1 string, arg2 *treechangeproto.TreeSyncMessage, arg3 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendWithReply", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendWithReply indicates an expected call of SendWithReply. +func (mr *MockSyncClientMockRecorder) SendWithReply(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendWithReply", reflect.TypeOf((*MockSyncClient)(nil).SendWithReply), arg0, arg1, arg2, arg3) +} diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index 450a1ce5..6cd39695 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -1,8 +1,8 @@ +//go:generate mockgen -destination mock_objectsync/mock_objectsync.go github.com/anytypeio/any-sync/commonspace/objectsync SyncClient package objectsync import ( "context" - "github.com/anytypeio/any-sync/commonspace/object/tree/synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/gogo/protobuf/proto" "sync/atomic" @@ -33,7 +33,7 @@ type objectSync struct { spaceId string messagePool MessagePool - syncClient synctree.SyncClient + syncClient SyncClient objectGetter syncobjectgetter.SyncObjectGetter configuration nodeconf.Configuration spaceStorage spacestorage.SpaceStorage @@ -59,7 +59,7 @@ func NewObjectSync( cancelSync: cancel, spaceIsDeleted: spaceIsDeleted, configuration: configuration, - syncClient: synctree.NewSyncClient(spaceId, peerManager, synctree.GetRequestFactory()), + syncClient: NewSyncClient(spaceId, peerManager, GetRequestFactory()), } os.messagePool = newMessagePool(peerManager, os.handleMessage) return os @@ -90,7 +90,7 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp log.With(zap.String("objectId", msg.ObjectId), zap.String("replyId", msg.ReplyId)).DebugCtx(ctx, "handling message") obj, err := s.objectGetter.GetObject(ctx, msg.ObjectId) if err != nil { - respErr := s.sendErrorResponse(ctx, msg, senderId, err) + respErr := s.sendErrorResponse(ctx, msg, senderId) if respErr != nil { log.Debug("failed to send error response", zap.Error(respErr)) } @@ -103,12 +103,12 @@ func (s *objectSync) MessagePool() MessagePool { return s.messagePool } -func (s *objectSync) sendErrorResponse(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, senderId string, respErr error) (err error) { +func (s *objectSync) sendErrorResponse(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, senderId string) (err error) { unmarshalled := &treechangeproto.TreeSyncMessage{} err = proto.Unmarshal(msg.Payload, unmarshalled) if err != nil { return } - resp := treechangeproto.WrapError(respErr, unmarshalled.RootChange) + resp := treechangeproto.WrapError(treechangeproto.ErrorCodes_GetTreeError, unmarshalled.RootChange) return s.syncClient.SendWithReply(ctx, senderId, resp, msg.ReplyId) } diff --git a/commonspace/object/tree/synctree/requestfactory.go b/commonspace/objectsync/requestfactory.go similarity index 99% rename from commonspace/object/tree/synctree/requestfactory.go rename to commonspace/objectsync/requestfactory.go index 6fc5202c..0608ad33 100644 --- a/commonspace/object/tree/synctree/requestfactory.go +++ b/commonspace/objectsync/requestfactory.go @@ -1,4 +1,4 @@ -package synctree +package objectsync import ( "fmt" diff --git a/commonspace/object/tree/synctree/syncclient.go b/commonspace/objectsync/syncclient.go similarity index 70% rename from commonspace/object/tree/synctree/syncclient.go rename to commonspace/objectsync/syncclient.go index 5967441b..6ef203c0 100644 --- a/commonspace/object/tree/synctree/syncclient.go +++ b/commonspace/objectsync/syncclient.go @@ -1,5 +1,4 @@ -//go:generate mockgen -destination mock_synctree/mock_synctree.go github.com/anytypeio/any-sync/commonspace/object/tree/synctree SyncClient,SyncTree,ReceiveQueue,HeadNotifiable -package synctree +package objectsync import ( "context" @@ -12,12 +11,13 @@ type SyncClient interface { RequestFactory Broadcast(ctx context.Context, msg *treechangeproto.TreeSyncMessage) (err error) SendWithReply(ctx context.Context, peerId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) + PeerManager() peermanager.PeerManager } type syncClient struct { - peermanager.PeerManager RequestFactory - spaceId string + spaceId string + peerManager peermanager.PeerManager } func NewSyncClient( @@ -25,29 +25,33 @@ func NewSyncClient( peerManager peermanager.PeerManager, factory RequestFactory) SyncClient { return &syncClient{ - PeerManager: peerManager, + peerManager: peerManager, RequestFactory: factory, spaceId: spaceId, } } func (s *syncClient) Broadcast(ctx context.Context, msg *treechangeproto.TreeSyncMessage) (err error) { - objMsg, err := marshallTreeMessage(msg, s.spaceId, msg.RootChange.Id, "") + objMsg, err := MarshallTreeMessage(msg, s.spaceId, msg.RootChange.Id, "") if err != nil { return } - return s.PeerManager.Broadcast(ctx, objMsg) + return s.peerManager.Broadcast(ctx, objMsg) } func (s *syncClient) SendWithReply(ctx context.Context, peerId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) { - objMsg, err := marshallTreeMessage(msg, s.spaceId, msg.RootChange.Id, replyId) + objMsg, err := MarshallTreeMessage(msg, s.spaceId, msg.RootChange.Id, replyId) if err != nil { return } - return s.PeerManager.SendPeer(ctx, peerId, objMsg) + return s.peerManager.SendPeer(ctx, peerId, objMsg) } -func marshallTreeMessage(message *treechangeproto.TreeSyncMessage, spaceId, objectId, replyId string) (objMsg *spacesyncproto.ObjectSyncMessage, err error) { +func (s *syncClient) PeerManager() peermanager.PeerManager { + return s.peerManager +} + +func MarshallTreeMessage(message *treechangeproto.TreeSyncMessage, spaceId, objectId, replyId string) (objMsg *spacesyncproto.ObjectSyncMessage, err error) { payload, err := message.Marshal() if err != nil { return diff --git a/commonspace/settings/deletionmanager.go b/commonspace/settings/deletionmanager.go index 366c2686..1e357cdf 100644 --- a/commonspace/settings/deletionmanager.go +++ b/commonspace/settings/deletionmanager.go @@ -55,10 +55,6 @@ func (d *deletionManager) UpdateState(ctx context.Context, state *settingsstate. return nil } log.Debug("deleting space") - err = d.treeManager.DeleteSpace(ctx, d.spaceId) - if err != nil { - log.Debug("failed to notify on space deletion", zap.Error(err)) - } if d.isResponsible { allIds := slice.DiscardFromSlice(d.provider.AllIds(), func(id string) bool { return id == d.settingsId diff --git a/commonspace/settings/deletionmanager_test.go b/commonspace/settings/deletionmanager_test.go index 990efc07..31aff516 100644 --- a/commonspace/settings/deletionmanager_test.go +++ b/commonspace/settings/deletionmanager_test.go @@ -30,7 +30,6 @@ func TestDeletionManager_UpdateState_NotResponsible(t *testing.T) { treeManager := mock_treemanager.NewMockTreeManager(ctrl) delState.EXPECT().Add(state.DeletedIds).Return(nil) - treeManager.EXPECT().DeleteSpace(ctx, spaceId).Return(nil) delManager := newDeletionManager(spaceId, settingsId, @@ -64,7 +63,6 @@ func TestDeletionManager_UpdateState_Responsible(t *testing.T) { provider := mock_settings.NewMockSpaceIdsProvider(ctrl) delState.EXPECT().Add(state.DeletedIds).Return(nil) - treeManager.EXPECT().DeleteSpace(ctx, spaceId).Return(nil) provider.EXPECT().AllIds().Return([]string{"id", "otherId", settingsId}) delState.EXPECT().Add([]string{"id", "otherId"}).Return(nil) delManager := newDeletionManager(spaceId, diff --git a/commonspace/space.go b/commonspace/space.go index f69b4aaa..de8c8f76 100644 --- a/commonspace/space.go +++ b/commonspace/space.go @@ -124,6 +124,7 @@ type space struct { configuration nodeconf.Configuration settingsObject settings.SettingsObject peerManager peermanager.PeerManager + treeBuilder objecttree.BuildObjectTreeFunc handleQueue multiqueue.MultiQueue[HandleMessage] @@ -292,7 +293,7 @@ func (s *space) PutTree(ctx context.Context, payload treestorage.TreeStorageCrea OnClose: s.onObjectClose, SyncStatus: s.syncStatus, PeerGetter: s.peerManager, - BuildObjectTree: s.treeManager.ObjectTreeBuilder(), + BuildObjectTree: s.treeBuilder, } t, err = synctree.PutSyncTree(ctx, payload, deps) if err != nil { @@ -331,7 +332,7 @@ func (s *space) BuildTree(ctx context.Context, id string, opts BuildTreeOpts) (t SyncStatus: s.syncStatus, WaitTreeRemoteSync: opts.WaitTreeRemoteSync, PeerGetter: s.peerManager, - BuildObjectTree: s.treeManager.ObjectTreeBuilder(), + BuildObjectTree: s.treeBuilder, } if t, err = synctree.BuildSyncTreeOrGetRemote(ctx, id, deps); err != nil { return nil, err diff --git a/commonspace/spaceservice.go b/commonspace/spaceservice.go index cfe33692..69daef67 100644 --- a/commonspace/spaceservice.go +++ b/commonspace/spaceservice.go @@ -8,6 +8,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/credentialprovider" "github.com/anytypeio/any-sync/commonspace/headsync" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/objectsync" @@ -152,6 +153,12 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) { // TODO: move it to the client package and add possibility to inject StatusProvider from the client syncStatus = syncstatus.NewSyncStatusProvider(st.Id(), syncstatus.DefaultDeps(lastConfiguration, st)) } + var builder objecttree.BuildObjectTreeFunc + if s.config.TreeInMemoryData { + builder = objecttree.BuildEmptyDataObjectTree + } else { + builder = objecttree.BuildObjectTree + } peerManager, err := s.peermanagerProvider.NewPeerManager(ctx, id) if err != nil { @@ -171,6 +178,7 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) { peerManager: peerManager, storage: st, treesUsed: &atomic.Int32{}, + treeBuilder: builder, isClosed: spaceIsClosed, isDeleted: spaceIsDeleted, } From 9acec773faafc572d018153dc84a6762c9d07f4a Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 19 Apr 2023 11:49:04 +0200 Subject: [PATCH 14/27] Add comments to test --- .../object/tree/synctree/syncprotocol_test.go | 4 +- .../object/tree/synctree/utils_test.go | 180 ++++++++++-------- 2 files changed, 99 insertions(+), 85 deletions(-) diff --git a/commonspace/object/tree/synctree/syncprotocol_test.go b/commonspace/object/tree/synctree/syncprotocol_test.go index 765e3235..24a3e535 100644 --- a/commonspace/object/tree/synctree/syncprotocol_test.go +++ b/commonspace/object/tree/synctree/syncprotocol_test.go @@ -32,7 +32,7 @@ func TestEmptyClientGetsFullHistory(t *testing.T) { }, emptyTrees: []string{"peer2"}, } - fx := newProcessFixture(t, spaceId, deps) + fx := newProtocolFixture(t, spaceId, deps) fx.run(t) fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ NewHeads: nil, @@ -111,7 +111,7 @@ func testTreeMerge(t *testing.T, levels, perlevel int, isSnapshot func() bool) { }, emptyTrees: []string{"peer2", "node1"}, } - fx := newProcessFixture(t, spaceId, deps) + fx := newProtocolFixture(t, spaceId, deps) fx.run(t) fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ NewHeads: initialRes.heads, diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index 8e2ffbe9..9d2c4f68 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -22,13 +22,15 @@ import ( "time" ) -type processMsg struct { +// protocolMsg is a message used in sync protocol tests +type protocolMsg struct { msg *spacesyncproto.ObjectSyncMessage senderId string receiverId string userMsg *objecttree.RawChangesPayload } +// msgDescription is a representation of message used for checking the results of the test type msgDescription struct { name string from string @@ -37,7 +39,7 @@ type msgDescription struct { changes []*treechangeproto.RawTreeChangeWithId } -func (p *processMsg) description() (descr msgDescription) { +func (p *protocolMsg) description() (descr msgDescription) { unmarshalled := &treechangeproto.TreeSyncMessage{} err := proto.Unmarshal(p.msg.Payload, unmarshalled) if err != nil { @@ -67,30 +69,68 @@ func (p *processMsg) description() (descr msgDescription) { return } +// messageLog saves all messages that were sent during sync test type messageLog struct { - batcher *mb.MB[processMsg] + batcher *mb.MB[protocolMsg] } func newMessageLog() *messageLog { - return &messageLog{batcher: mb.New[processMsg](0)} + return &messageLog{batcher: mb.New[protocolMsg](0)} } -func (m *messageLog) addMessage(msg processMsg) { +func (m *messageLog) addMessage(msg protocolMsg) { m.batcher.Add(context.Background(), msg) } -type processSyncHandler struct { +// testSyncHandler is the wrapper around individual tree to test sync protocol +type testSyncHandler struct { synchandler.SyncHandler - batcher *mb.MB[processMsg] + batcher *mb.MB[protocolMsg] peerId string aclList list.AclList log *messageLog syncClient objectsync.SyncClient } -func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { - if p.SyncHandler != nil { - return p.SyncHandler.HandleMessage(ctx, senderId, request) +// createSyncHandler creates a sync handler when a tree is already created +func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *testSyncHandler { + factory := objectsync.GetRequestFactory() + syncClient := objectsync.NewSyncClient(spaceId, newTestPeerManager(peerId, log), factory) + netTree := &broadcastTree{ + ObjectTree: objTree, + SyncClient: syncClient, + } + handler := newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) + return newTestSyncHandler(peerId, handler) +} + +// createEmptySyncHandler creates a sync handler when the tree will be provided later (this emulates the situation when we have no tree) +func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *testSyncHandler { + factory := objectsync.GetRequestFactory() + syncClient := objectsync.NewSyncClient(spaceId, newTestPeerManager(peerId, log), factory) + + batcher := mb.New[protocolMsg](0) + return &testSyncHandler{ + batcher: batcher, + peerId: peerId, + aclList: aclList, + log: log, + syncClient: syncClient, + } +} + +func newTestSyncHandler(peerId string, syncHandler synchandler.SyncHandler) *testSyncHandler { + batcher := mb.New[protocolMsg](0) + return &testSyncHandler{ + SyncHandler: syncHandler, + batcher: batcher, + peerId: peerId, + } +} + +func (h *testSyncHandler) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { + if h.SyncHandler != nil { + return h.SyncHandler.HandleMessage(ctx, senderId, request) } unmarshalled := &treechangeproto.TreeSyncMessage{} err = proto.Unmarshal(request.Payload, unmarshalled) @@ -104,17 +144,17 @@ func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, if err != nil { return } - return p.manager().SendPeer(context.Background(), senderId, objMsg) + return h.manager().SendPeer(context.Background(), senderId, objMsg) } fullSyncResponse := unmarshalled.Content.GetFullSyncResponse() treeStorage, _ := treestorage.NewInMemoryTreeStorage(unmarshalled.RootChange, []string{unmarshalled.RootChange.Id}, nil) - tree, err := createTestTree(p.aclList, treeStorage) + tree, err := createTestTree(h.aclList, treeStorage) if err != nil { return } netTree := &broadcastTree{ ObjectTree: tree, - SyncClient: p.syncClient, + SyncClient: h.syncClient, } res, err := netTree.AddRawChanges(context.Background(), objecttree.RawChangesPayload{ NewHeads: fullSyncResponse.Heads, @@ -123,62 +163,53 @@ func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, if err != nil { return } - p.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, p.syncClient, syncstatus.NewNoOpSyncStatus()) + h.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, h.syncClient, syncstatus.NewNoOpSyncStatus()) var objMsg *spacesyncproto.ObjectSyncMessage newTreeRequest := objectsync.GetRequestFactory().CreateHeadUpdate(netTree, res.Added) objMsg, err = objectsync.MarshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") if err != nil { return } - return p.manager().Broadcast(context.Background(), objMsg) + return h.manager().Broadcast(context.Background(), objMsg) } -func newProcessSyncHandler(peerId string, syncHandler synchandler.SyncHandler) *processSyncHandler { - batcher := mb.New[processMsg](0) - return &processSyncHandler{ - SyncHandler: syncHandler, - batcher: batcher, - peerId: peerId, +func (h *testSyncHandler) manager() *testPeerManager { + if h.SyncHandler != nil { + return h.SyncHandler.(*syncTreeHandler).syncClient.PeerManager().(*testPeerManager) } + return h.syncClient.PeerManager().(*testPeerManager) } -func (p *processSyncHandler) manager() *processPeerManager { - if p.SyncHandler != nil { - return p.SyncHandler.(*syncTreeHandler).syncClient.PeerManager().(*processPeerManager) - } - return p.syncClient.PeerManager().(*processPeerManager) +func (h *testSyncHandler) tree() *broadcastTree { + return h.SyncHandler.(*syncTreeHandler).objTree.(*broadcastTree) } -func (p *processSyncHandler) tree() *broadcastTree { - return p.SyncHandler.(*syncTreeHandler).objTree.(*broadcastTree) +func (h *testSyncHandler) send(ctx context.Context, msg protocolMsg) (err error) { + return h.batcher.Add(ctx, msg) } -func (p *processSyncHandler) send(ctx context.Context, msg processMsg) (err error) { - return p.batcher.Add(ctx, msg) +func (h *testSyncHandler) sendRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) { + h.batcher.Add(ctx, protocolMsg{userMsg: &changes}) } -func (p *processSyncHandler) sendRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) { - p.batcher.Add(ctx, processMsg{userMsg: &changes}) -} - -func (p *processSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.WaitGroup) { +func (h *testSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.WaitGroup) { wg.Add(1) go func() { defer wg.Done() for { - res, err := p.batcher.WaitOne(ctx) + res, err := h.batcher.WaitOne(ctx) if err != nil { return } if res.userMsg != nil { - p.tree().Lock() - userRes, err := p.tree().AddRawChanges(ctx, *res.userMsg) + h.tree().Lock() + userRes, err := h.tree().AddRawChanges(ctx, *res.userMsg) require.NoError(t, err) fmt.Println("user add result", userRes.Heads) - p.tree().Unlock() + h.tree().Unlock() continue } - err = p.HandleMessage(ctx, res.senderId, res.msg) + err = h.HandleMessage(ctx, res.senderId, res.msg) if err != nil { fmt.Println("error handling message", err.Error()) continue @@ -187,22 +218,23 @@ func (p *processSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.Wai }() } -type processPeerManager struct { +// testPeerManager captures all other handlers and sends messages to them +type testPeerManager struct { peerId string - handlers map[string]*processSyncHandler + handlers map[string]*testSyncHandler log *messageLog } -func newProcessPeerManager(peerId string, log *messageLog) *processPeerManager { - return &processPeerManager{handlers: map[string]*processSyncHandler{}, peerId: peerId, log: log} +func newTestPeerManager(peerId string, log *messageLog) *testPeerManager { + return &testPeerManager{handlers: map[string]*testSyncHandler{}, peerId: peerId, log: log} } -func (m *processPeerManager) addHandler(peerId string, handler *processSyncHandler) { +func (m *testPeerManager) addHandler(peerId string, handler *testSyncHandler) { m.handlers[peerId] = handler } -func (m *processPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { - pMsg := processMsg{ +func (m *testPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { + pMsg := protocolMsg{ msg: msg, senderId: m.peerId, receiverId: peerId, @@ -211,9 +243,9 @@ func (m *processPeerManager) SendPeer(ctx context.Context, peerId string, msg *s return m.handlers[peerId].send(context.Background(), pMsg) } -func (m *processPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { +func (m *testPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { for _, handler := range m.handlers { - pMsg := processMsg{ + pMsg := protocolMsg{ msg: msg, senderId: m.peerId, receiverId: handler.peerId, @@ -224,10 +256,12 @@ func (m *processPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto. return } -func (m *processPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { +func (m *testPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { panic("should not be called") } +// broadcastTree is the tree that broadcasts changes to everyone when changes are added +// it is a simplified version of SyncTree which is easier to use in the test environment type broadcastTree struct { objecttree.ObjectTree objectsync.SyncClient @@ -243,31 +277,6 @@ func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.Ra return res, nil } -func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *processSyncHandler { - factory := objectsync.GetRequestFactory() - syncClient := objectsync.NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) - netTree := &broadcastTree{ - ObjectTree: objTree, - SyncClient: syncClient, - } - handler := newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) - return newProcessSyncHandler(peerId, handler) -} - -func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *processSyncHandler { - factory := objectsync.GetRequestFactory() - syncClient := objectsync.NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) - - batcher := mb.New[processMsg](0) - return &processSyncHandler{ - batcher: batcher, - peerId: peerId, - aclList: aclList, - log: log, - syncClient: syncClient, - } -} - func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage { changeCreator := objecttree.NewMockChangeCreator() st := changeCreator.CreateNewTreeStorage(treeId, aclList.Head().Id) @@ -285,24 +294,25 @@ type fixtureDeps struct { emptyTrees []string } -type processFixture struct { - handlers map[string]*processSyncHandler +// protocolFixture is the test environment for sync protocol tests +type protocolFixture struct { + handlers map[string]*testSyncHandler log *messageLog wg *sync.WaitGroup ctx context.Context cancel context.CancelFunc } -func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processFixture { +func newProtocolFixture(t *testing.T, spaceId string, deps fixtureDeps) *protocolFixture { var ( - handlers = map[string]*processSyncHandler{} + handlers = map[string]*testSyncHandler{} log = newMessageLog() wg = sync.WaitGroup{} ctx, cancel = context.WithCancel(context.Background()) ) for peerId := range deps.connectionMap { - var handler *processSyncHandler + var handler *testSyncHandler if slices.Contains(deps.emptyTrees, peerId) { handler = createEmptySyncHandler(peerId, spaceId, deps.aclList, log) } else { @@ -320,7 +330,7 @@ func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processF manager.addHandler(connectionId, handlers[connectionId]) } } - return &processFixture{ + return &protocolFixture{ handlers: handlers, log: log, wg: &wg, @@ -329,18 +339,20 @@ func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processF } } -func (p *processFixture) run(t *testing.T) { +func (p *protocolFixture) run(t *testing.T) { for _, handler := range p.handlers { handler.run(p.ctx, t, p.wg) } } -func (p *processFixture) stop() { +func (p *protocolFixture) stop() { p.cancel() p.wg.Wait() } +// genParams is the parameters for genChanges type genParams struct { + // prefix is the prefix which is added to change id prefix string aclId string startIdx int @@ -351,12 +363,14 @@ type genParams struct { isSnapshot func() bool } +// genResult is the result of genChanges type genResult struct { changes []*treechangeproto.RawTreeChangeWithId heads []string snapshotId string } +// genChanges generates several levels of tree changes where each level is connected with only previous one func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res genResult) { src := rand.NewSource(time.Now().Unix()) rnd := rand.New(src) From 34865d0e686e182fdb1d8ca62ed32f5104e9b17a Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 19 Apr 2023 11:52:11 +0200 Subject: [PATCH 15/27] Further test comments and changes --- commonspace/object/tree/synctree/syncprotocol_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/commonspace/object/tree/synctree/syncprotocol_test.go b/commonspace/object/tree/synctree/syncprotocol_test.go index 24a3e535..cc7ebec4 100644 --- a/commonspace/object/tree/synctree/syncprotocol_test.go +++ b/commonspace/object/tree/synctree/syncprotocol_test.go @@ -44,7 +44,7 @@ func TestEmptyClientGetsFullHistory(t *testing.T) { fx.stop() firstHeads := fx.handlers["peer1"].tree().Heads() secondHeads := fx.handlers["peer2"].tree().Heads() - require.True(t, slice.SortedEquals(firstHeads, secondHeads)) + require.True(t, slice.UnsortedEquals(firstHeads, secondHeads)) require.Equal(t, []string{"1"}, firstHeads) logMsgs := fx.log.batcher.GetAll() @@ -98,6 +98,7 @@ func testTreeMerge(t *testing.T, levels, perlevel int, isSnapshot func() bool) { prevHeads: []string{treeId}, isSnapshot: isSnapshot, } + // generating initial tree initialRes := genChanges(changeCreator, params) err = storage.TransactionAdd(initialRes.changes, initialRes.heads) require.NoError(t, err) @@ -132,6 +133,7 @@ func testTreeMerge(t *testing.T, levels, perlevel int, isSnapshot func() bool) { prevHeads: initialRes.heads, isSnapshot: isSnapshot, } + // generating different additions to the tree for different peers peer1Res := genChanges(changeCreator, params) params.prefix = "peer2" peer2Res := genChanges(changeCreator, params) From aeaad8ab3596f53444d132d60401e7897f158cdf Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 19 Apr 2023 19:34:31 +0200 Subject: [PATCH 16/27] Add logs if failed to get object --- commonspace/objectsync/objectsync.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index ec931eaa..a6d52d73 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -87,12 +87,13 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp return spacesyncproto.ErrSpaceIsDeleted } } - log.With(zap.String("objectId", msg.ObjectId), zap.String("replyId", msg.ReplyId)).DebugCtx(ctx, "handling message") + log.DebugCtx(ctx, "handling message") obj, err := s.objectGetter.GetObject(ctx, msg.ObjectId) if err != nil { + log.DebugCtx(ctx, "failed to get object") respErr := s.sendErrorResponse(ctx, msg, senderId) if respErr != nil { - log.Debug("failed to send error response", zap.Error(respErr)) + log.DebugCtx(ctx, "failed to send error response", zap.Error(respErr)) } return } From 3edda33db6044a221aca8e10cffb930e11bd2a1b Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 19 Apr 2023 20:42:47 +0200 Subject: [PATCH 17/27] Update object sync error handling logic --- .../object/tree/synctree/synctreehandler.go | 2 +- .../object/tree/treechangeproto/treechange.go | 6 ++- commonspace/objectsync/objectsync.go | 38 ++++++++++++++----- .../mock_spacestorage/mock_spacestorage.go | 15 ++++++++ commonspace/spacestorage/spacestorage.go | 1 + net/rpc/rpcerr/registry.go | 4 ++ 6 files changed, 54 insertions(+), 12 deletions(-) diff --git a/commonspace/object/tree/synctree/synctreehandler.go b/commonspace/object/tree/synctree/synctreehandler.go index 7865f2a7..7b4cb2a8 100644 --- a/commonspace/object/tree/synctree/synctreehandler.go +++ b/commonspace/object/tree/synctree/synctreehandler.go @@ -167,7 +167,7 @@ func (s *syncTreeHandler) handleFullSyncRequest( defer func() { if err != nil { log.With(zap.Error(err)).DebugCtx(ctx, "full sync request finished with error") - s.syncClient.SendWithReply(ctx, senderId, treechangeproto.WrapError(treechangeproto.ErrorCodes_FullSyncRequestError, header), replyId) + s.syncClient.SendWithReply(ctx, senderId, treechangeproto.WrapError(treechangeproto.ErrFullSync, header), replyId) return } else if fullResponse != nil { cnt := fullResponse.Content.GetFullSyncResponse() diff --git a/commonspace/object/tree/treechangeproto/treechange.go b/commonspace/object/tree/treechangeproto/treechange.go index 0dc5d668..92ef3f16 100644 --- a/commonspace/object/tree/treechangeproto/treechange.go +++ b/commonspace/object/tree/treechangeproto/treechange.go @@ -1,5 +1,7 @@ package treechangeproto +import "github.com/anytypeio/any-sync/net/rpc/rpcerr" + func WrapHeadUpdate(update *TreeHeadUpdate, rootChange *RawTreeChangeWithId) *TreeSyncMessage { return &TreeSyncMessage{ Content: &TreeSyncContentValue{ @@ -27,10 +29,10 @@ func WrapFullResponse(response *TreeFullSyncResponse, rootChange *RawTreeChangeW } } -func WrapError(code ErrorCodes, rootChange *RawTreeChangeWithId) *TreeSyncMessage { +func WrapError(err error, rootChange *RawTreeChangeWithId) *TreeSyncMessage { return &TreeSyncMessage{ Content: &TreeSyncContentValue{ - Value: &TreeSyncContentValue_ErrorResponse{ErrorResponse: &TreeErrorResponse{ErrCode: uint64(code)}}, + Value: &TreeSyncContentValue_ErrorResponse{ErrorResponse: &TreeErrorResponse{ErrCode: rpcerr.Code(err)}}, }, RootChange: rootChange, } diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index a6d52d73..f6e311ac 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -84,18 +84,30 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp log = log.With(zap.Bool("isDeleted", true)) // preventing sync with other clients if they are not just syncing the settings tree if !slices.Contains(s.configuration.NodeIds(s.spaceId), senderId) && msg.ObjectId != s.spaceStorage.SpaceSettingsId() { - return spacesyncproto.ErrSpaceIsDeleted + return s.unmarshallSendError(ctx, msg, spacesyncproto.ErrSpaceIsDeleted, senderId) } } log.DebugCtx(ctx, "handling message") + hasTree, err := s.spaceStorage.HasTree(msg.ObjectId) + if err != nil { + return s.unmarshallSendError(ctx, msg, spacesyncproto.ErrUnexpected, senderId) + } + // in this case we will try to get it from remote, unless the sender also sent us the same request :-) + if !hasTree { + treeMsg := &treechangeproto.TreeSyncMessage{} + err = proto.Unmarshal(msg.Payload, treeMsg) + if err != nil { + return s.sendError(ctx, nil, spacesyncproto.ErrUnexpected, senderId, msg.ReplyId) + } + // this means that we don't have the tree locally and therefore can't return it + if s.isEmptyFullSyncRequest(treeMsg) { + return s.sendError(ctx, treeMsg.RootChange, treechangeproto.ErrGetTree, senderId, msg.ReplyId) + } + } obj, err := s.objectGetter.GetObject(ctx, msg.ObjectId) if err != nil { log.DebugCtx(ctx, "failed to get object") - respErr := s.sendErrorResponse(ctx, msg, senderId) - if respErr != nil { - log.DebugCtx(ctx, "failed to send error response", zap.Error(respErr)) - } - return + return s.unmarshallSendError(ctx, msg, err, senderId) } return obj.HandleMessage(ctx, senderId, msg) } @@ -104,12 +116,20 @@ func (s *objectSync) MessagePool() MessagePool { return s.messagePool } -func (s *objectSync) sendErrorResponse(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, senderId string) (err error) { +func (s *objectSync) unmarshallSendError(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, respErr error, senderId string) (err error) { unmarshalled := &treechangeproto.TreeSyncMessage{} err = proto.Unmarshal(msg.Payload, unmarshalled) if err != nil { return } - resp := treechangeproto.WrapError(treechangeproto.ErrorCodes_GetTreeError, unmarshalled.RootChange) - return s.syncClient.SendWithReply(ctx, senderId, resp, msg.ReplyId) + return s.sendError(ctx, unmarshalled.RootChange, respErr, senderId, msg.ReplyId) +} + +func (s *objectSync) sendError(ctx context.Context, root *treechangeproto.RawTreeChangeWithId, respErr error, senderId, replyId string) (err error) { + resp := treechangeproto.WrapError(respErr, root) + return s.syncClient.SendWithReply(ctx, senderId, resp, replyId) +} + +func (s *objectSync) isEmptyFullSyncRequest(msg *treechangeproto.TreeSyncMessage) bool { + return len(msg.GetContent().GetFullSyncRequest().GetHeads()) == 0 } diff --git a/commonspace/spacestorage/mock_spacestorage/mock_spacestorage.go b/commonspace/spacestorage/mock_spacestorage/mock_spacestorage.go index 82a14be2..93baa877 100644 --- a/commonspace/spacestorage/mock_spacestorage/mock_spacestorage.go +++ b/commonspace/spacestorage/mock_spacestorage/mock_spacestorage.go @@ -81,6 +81,21 @@ func (mr *MockSpaceStorageMockRecorder) CreateTreeStorage(arg0 interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTreeStorage", reflect.TypeOf((*MockSpaceStorage)(nil).CreateTreeStorage), arg0) } +// HasTree mocks base method. +func (m *MockSpaceStorage) HasTree(arg0 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HasTree", arg0) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// HasTree indicates an expected call of HasTree. +func (mr *MockSpaceStorageMockRecorder) HasTree(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasTree", reflect.TypeOf((*MockSpaceStorage)(nil).HasTree), arg0) +} + // Id mocks base method. func (m *MockSpaceStorage) Id() string { m.ctrl.T.Helper() diff --git a/commonspace/spacestorage/spacestorage.go b/commonspace/spacestorage/spacestorage.go index ebd4b68d..3f21c22e 100644 --- a/commonspace/spacestorage/spacestorage.go +++ b/commonspace/spacestorage/spacestorage.go @@ -47,6 +47,7 @@ type SpaceStorage interface { StoredIds() ([]string, error) TreeRoot(id string) (*treechangeproto.RawTreeChangeWithId, error) TreeStorage(id string) (treestorage.TreeStorage, error) + HasTree(id string) (bool, error) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (treestorage.TreeStorage, error) WriteSpaceHash(hash string) error ReadSpaceHash() (hash string, err error) diff --git a/net/rpc/rpcerr/registry.go b/net/rpc/rpcerr/registry.go index 5160fec3..e8a749e6 100644 --- a/net/rpc/rpcerr/registry.go +++ b/net/rpc/rpcerr/registry.go @@ -24,6 +24,10 @@ func RegisterErr(err error, code uint64) error { return errWithCode } +func Code(err error) uint64 { + return drpcerr.Code(err) +} + func Err(code uint64) error { err, ok := errsMap[code] if !ok { From ed16abcedeb3a9ff943287dd11198850f8125b9c Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 19 Apr 2023 21:16:32 +0200 Subject: [PATCH 18/27] Fix sending reply with empty rootchange --- commonspace/object/tree/synctree/synctree.go | 2 +- .../object/tree/synctree/synctreehandler.go | 17 ++++++++++------- .../tree/synctree/synctreehandler_test.go | 12 ++++++------ .../mock_objectsync/mock_objectsync.go | 8 ++++---- commonspace/objectsync/objectsync.go | 18 +++++++++--------- commonspace/objectsync/syncclient.go | 6 +++--- 6 files changed, 33 insertions(+), 30 deletions(-) diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index 8b3295df..906ba5cf 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -243,7 +243,7 @@ func (s *syncTree) SyncWithPeer(ctx context.Context, peerId string) (err error) s.Lock() defer s.Unlock() headUpdate := s.syncClient.CreateHeadUpdate(s, nil) - return s.syncClient.SendWithReply(ctx, peerId, headUpdate, "") + return s.syncClient.SendWithReply(ctx, peerId, headUpdate.RootChange.Id, headUpdate, "") } func (s *syncTree) afterBuild() { diff --git a/commonspace/object/tree/synctree/synctreehandler.go b/commonspace/object/tree/synctree/synctreehandler.go index 7b4cb2a8..dab63e92 100644 --- a/commonspace/object/tree/synctree/synctreehandler.go +++ b/commonspace/object/tree/synctree/synctreehandler.go @@ -82,10 +82,11 @@ func (s *syncTreeHandler) handleHeadUpdate( fullRequest *treechangeproto.TreeSyncMessage isEmptyUpdate = len(update.Changes) == 0 objTree = s.objTree + treeId = objTree.Id() ) log := log.With( zap.Strings("update heads", update.Heads), - zap.String("treeId", objTree.Id()), + zap.String("treeId", treeId), zap.String("spaceId", s.spaceId), zap.Int("len(update changes)", len(update.Changes))) log.DebugCtx(ctx, "received head update message") @@ -118,7 +119,7 @@ func (s *syncTreeHandler) handleHeadUpdate( return } - return s.syncClient.SendWithReply(ctx, senderId, fullRequest, replyId) + return s.syncClient.SendWithReply(ctx, senderId, treeId, fullRequest, replyId) } if s.alreadyHasHeads(objTree, update.Heads) { @@ -142,7 +143,7 @@ func (s *syncTreeHandler) handleHeadUpdate( return } - return s.syncClient.SendWithReply(ctx, senderId, fullRequest, replyId) + return s.syncClient.SendWithReply(ctx, senderId, treeId, fullRequest, replyId) } func (s *syncTreeHandler) handleFullSyncRequest( @@ -154,11 +155,12 @@ func (s *syncTreeHandler) handleFullSyncRequest( fullResponse *treechangeproto.TreeSyncMessage header = s.objTree.Header() objTree = s.objTree + treeId = s.objTree.Id() ) log := log.With(zap.String("senderId", senderId), zap.Strings("request heads", request.Heads), - zap.String("treeId", s.objTree.Id()), + zap.String("treeId", treeId), zap.String("replyId", replyId), zap.String("spaceId", s.spaceId), zap.Int("len(request changes)", len(request.Changes))) @@ -167,7 +169,7 @@ func (s *syncTreeHandler) handleFullSyncRequest( defer func() { if err != nil { log.With(zap.Error(err)).DebugCtx(ctx, "full sync request finished with error") - s.syncClient.SendWithReply(ctx, senderId, treechangeproto.WrapError(treechangeproto.ErrFullSync, header), replyId) + s.syncClient.SendWithReply(ctx, senderId, treeId, treechangeproto.WrapError(treechangeproto.ErrFullSync, header), replyId) return } else if fullResponse != nil { cnt := fullResponse.Content.GetFullSyncResponse() @@ -190,7 +192,7 @@ func (s *syncTreeHandler) handleFullSyncRequest( return } - return s.syncClient.SendWithReply(ctx, senderId, fullResponse, replyId) + return s.syncClient.SendWithReply(ctx, senderId, treeId, fullResponse, replyId) } func (s *syncTreeHandler) handleFullSyncResponse( @@ -199,10 +201,11 @@ func (s *syncTreeHandler) handleFullSyncResponse( response *treechangeproto.TreeFullSyncResponse) (err error) { var ( objTree = s.objTree + treeId = s.objTree.Id() ) log := log.With( zap.Strings("heads", response.Heads), - zap.String("treeId", s.objTree.Id()), + zap.String("treeId", treeId), zap.String("spaceId", s.spaceId), zap.Int("len(changes)", len(response.Changes))) log.DebugCtx(ctx, "received full sync response message") diff --git a/commonspace/object/tree/synctree/synctreehandler_test.go b/commonspace/object/tree/synctree/synctreehandler_test.go index b95ffc82..401f9989 100644 --- a/commonspace/object/tree/synctree/synctreehandler_test.go +++ b/commonspace/object/tree/synctree/synctreehandler_test.go @@ -154,7 +154,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { fx.syncClientMock.EXPECT(). CreateFullSyncRequest(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullRequest, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullRequest), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullRequest), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -200,7 +200,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { fx.syncClientMock.EXPECT(). CreateFullSyncRequest(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullRequest, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullRequest), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullRequest), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -266,7 +266,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { fx.syncClientMock.EXPECT(). CreateFullSyncResponse(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullResponse, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullResponse), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullResponse), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -295,7 +295,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { fx.syncClientMock.EXPECT(). CreateFullSyncResponse(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullResponse, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullResponse), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullResponse), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -322,7 +322,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { fx.syncClientMock.EXPECT(). CreateFullSyncResponse(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullResponse, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullResponse), gomock.Eq(replyId)) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullResponse), gomock.Eq(replyId)) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -357,7 +357,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { RawChanges: []*treechangeproto.RawTreeChangeWithId{chWithId}, })). Return(objecttree.AddResult{}, fmt.Errorf("")) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Any(), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Any(), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.Error(t, err) diff --git a/commonspace/objectsync/mock_objectsync/mock_objectsync.go b/commonspace/objectsync/mock_objectsync/mock_objectsync.go index b82360db..aaca46a5 100644 --- a/commonspace/objectsync/mock_objectsync/mock_objectsync.go +++ b/commonspace/objectsync/mock_objectsync/mock_objectsync.go @@ -124,15 +124,15 @@ func (mr *MockSyncClientMockRecorder) PeerManager() *gomock.Call { } // SendWithReply mocks base method. -func (m *MockSyncClient) SendWithReply(arg0 context.Context, arg1 string, arg2 *treechangeproto.TreeSyncMessage, arg3 string) error { +func (m *MockSyncClient) SendWithReply(arg0 context.Context, arg1, arg2 string, arg3 *treechangeproto.TreeSyncMessage, arg4 string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendWithReply", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "SendWithReply", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(error) return ret0 } // SendWithReply indicates an expected call of SendWithReply. -func (mr *MockSyncClientMockRecorder) SendWithReply(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +func (mr *MockSyncClientMockRecorder) SendWithReply(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendWithReply", reflect.TypeOf((*MockSyncClient)(nil).SendWithReply), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendWithReply", reflect.TypeOf((*MockSyncClient)(nil).SendWithReply), arg0, arg1, arg2, arg3, arg4) } diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index f6e311ac..8bb9b5f2 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -84,30 +84,30 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp log = log.With(zap.Bool("isDeleted", true)) // preventing sync with other clients if they are not just syncing the settings tree if !slices.Contains(s.configuration.NodeIds(s.spaceId), senderId) && msg.ObjectId != s.spaceStorage.SpaceSettingsId() { - return s.unmarshallSendError(ctx, msg, spacesyncproto.ErrSpaceIsDeleted, senderId) + return s.unmarshallSendError(ctx, msg, spacesyncproto.ErrSpaceIsDeleted, senderId, msg.ObjectId) } } log.DebugCtx(ctx, "handling message") hasTree, err := s.spaceStorage.HasTree(msg.ObjectId) if err != nil { - return s.unmarshallSendError(ctx, msg, spacesyncproto.ErrUnexpected, senderId) + return s.unmarshallSendError(ctx, msg, spacesyncproto.ErrUnexpected, senderId, msg.ObjectId) } // in this case we will try to get it from remote, unless the sender also sent us the same request :-) if !hasTree { treeMsg := &treechangeproto.TreeSyncMessage{} err = proto.Unmarshal(msg.Payload, treeMsg) if err != nil { - return s.sendError(ctx, nil, spacesyncproto.ErrUnexpected, senderId, msg.ReplyId) + return s.sendError(ctx, nil, spacesyncproto.ErrUnexpected, senderId, msg.ObjectId, msg.ReplyId) } // this means that we don't have the tree locally and therefore can't return it if s.isEmptyFullSyncRequest(treeMsg) { - return s.sendError(ctx, treeMsg.RootChange, treechangeproto.ErrGetTree, senderId, msg.ReplyId) + return s.sendError(ctx, nil, treechangeproto.ErrGetTree, senderId, msg.ObjectId, msg.ReplyId) } } obj, err := s.objectGetter.GetObject(ctx, msg.ObjectId) if err != nil { log.DebugCtx(ctx, "failed to get object") - return s.unmarshallSendError(ctx, msg, err, senderId) + return s.unmarshallSendError(ctx, msg, err, msg.ObjectId, senderId) } return obj.HandleMessage(ctx, senderId, msg) } @@ -116,18 +116,18 @@ func (s *objectSync) MessagePool() MessagePool { return s.messagePool } -func (s *objectSync) unmarshallSendError(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, respErr error, senderId string) (err error) { +func (s *objectSync) unmarshallSendError(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, respErr error, senderId, objectId string) (err error) { unmarshalled := &treechangeproto.TreeSyncMessage{} err = proto.Unmarshal(msg.Payload, unmarshalled) if err != nil { return } - return s.sendError(ctx, unmarshalled.RootChange, respErr, senderId, msg.ReplyId) + return s.sendError(ctx, unmarshalled.RootChange, respErr, senderId, objectId, msg.ReplyId) } -func (s *objectSync) sendError(ctx context.Context, root *treechangeproto.RawTreeChangeWithId, respErr error, senderId, replyId string) (err error) { +func (s *objectSync) sendError(ctx context.Context, root *treechangeproto.RawTreeChangeWithId, respErr error, senderId, objectId, replyId string) (err error) { resp := treechangeproto.WrapError(respErr, root) - return s.syncClient.SendWithReply(ctx, senderId, resp, replyId) + return s.syncClient.SendWithReply(ctx, senderId, objectId, resp, replyId) } func (s *objectSync) isEmptyFullSyncRequest(msg *treechangeproto.TreeSyncMessage) bool { diff --git a/commonspace/objectsync/syncclient.go b/commonspace/objectsync/syncclient.go index 6ef203c0..4a95cae1 100644 --- a/commonspace/objectsync/syncclient.go +++ b/commonspace/objectsync/syncclient.go @@ -10,7 +10,7 @@ import ( type SyncClient interface { RequestFactory Broadcast(ctx context.Context, msg *treechangeproto.TreeSyncMessage) (err error) - SendWithReply(ctx context.Context, peerId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) + SendWithReply(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) PeerManager() peermanager.PeerManager } @@ -39,8 +39,8 @@ func (s *syncClient) Broadcast(ctx context.Context, msg *treechangeproto.TreeSyn return s.peerManager.Broadcast(ctx, objMsg) } -func (s *syncClient) SendWithReply(ctx context.Context, peerId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) { - objMsg, err := MarshallTreeMessage(msg, s.spaceId, msg.RootChange.Id, replyId) +func (s *syncClient) SendWithReply(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) { + objMsg, err := MarshallTreeMessage(msg, s.spaceId, objectId, replyId) if err != nil { return } From e79d319925eef63368811783670024777d108198 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 19 Apr 2023 22:43:46 +0200 Subject: [PATCH 19/27] Change sync client message pool interactions and request-reply --- commonspace/object/tree/synctree/synctree.go | 9 +---- .../object/tree/synctree/treeremotegetter.go | 7 +--- .../object/tree/synctree/utils_test.go | 38 ++++++++++++------- .../mock_objectsync/mock_objectsync.go | 32 ++++++++++++---- commonspace/objectsync/objectsync.go | 19 ++++++---- commonspace/objectsync/syncclient.go | 26 ++++++++----- commonspace/space.go | 6 +-- 7 files changed, 83 insertions(+), 54 deletions(-) diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index 906ba5cf..dc87a3c4 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -55,15 +55,13 @@ type syncTree struct { var log = logger.NewNamed("common.commonspace.synctree") -var createSyncClient = objectsync.NewSyncClient - type ResponsiblePeersGetter interface { GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) } type BuildDeps struct { SpaceId string - ObjectSync objectsync.ObjectSync + SyncClient objectsync.SyncClient Configuration nodeconf.NodeConf HeadNotifiable HeadNotifiable Listener updatelistener.UpdateListener @@ -99,10 +97,7 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t Sy if err != nil { return } - syncClient := createSyncClient( - deps.SpaceId, - deps.ObjectSync.MessagePool(), - objectsync.GetRequestFactory()) + syncClient := deps.SyncClient syncTree := &syncTree{ ObjectTree: objTree, syncClient: syncClient, diff --git a/commonspace/object/tree/synctree/treeremotegetter.go b/commonspace/object/tree/synctree/treeremotegetter.go index 9f819c41..bcf547b0 100644 --- a/commonspace/object/tree/synctree/treeremotegetter.go +++ b/commonspace/object/tree/synctree/treeremotegetter.go @@ -48,12 +48,7 @@ func (t treeRemoteGetter) getPeers(ctx context.Context) (peerIds []string, err e func (t treeRemoteGetter) treeRequest(ctx context.Context, peerId string) (msg *treechangeproto.TreeSyncMessage, err error) { newTreeRequest := objectsync.GetRequestFactory().CreateNewTreeRequest() - objMsg, err := objectsync.MarshallTreeMessage(newTreeRequest, t.deps.SpaceId, t.treeId, "") - if err != nil { - return - } - - resp, err := t.deps.ObjectSync.MessagePool().SendSync(ctx, peerId, objMsg) + resp, err := t.deps.SyncClient.SendSync(ctx, peerId, t.treeId, newTreeRequest) if err != nil { return } diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index 9d2c4f68..451f7536 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -95,7 +95,7 @@ type testSyncHandler struct { // createSyncHandler creates a sync handler when a tree is already created func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *testSyncHandler { factory := objectsync.GetRequestFactory() - syncClient := objectsync.NewSyncClient(spaceId, newTestPeerManager(peerId, log), factory) + syncClient := objectsync.NewSyncClient(spaceId, newTestMessagePool(peerId, log), factory) netTree := &broadcastTree{ ObjectTree: objTree, SyncClient: syncClient, @@ -107,7 +107,7 @@ func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, lo // createEmptySyncHandler creates a sync handler when the tree will be provided later (this emulates the situation when we have no tree) func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *testSyncHandler { factory := objectsync.GetRequestFactory() - syncClient := objectsync.NewSyncClient(spaceId, newTestPeerManager(peerId, log), factory) + syncClient := objectsync.NewSyncClient(spaceId, newTestMessagePool(peerId, log), factory) batcher := mb.New[protocolMsg](0) return &testSyncHandler{ @@ -173,11 +173,11 @@ func (h *testSyncHandler) HandleMessage(ctx context.Context, senderId string, re return h.manager().Broadcast(context.Background(), objMsg) } -func (h *testSyncHandler) manager() *testPeerManager { +func (h *testSyncHandler) manager() *testMessagePool { if h.SyncHandler != nil { - return h.SyncHandler.(*syncTreeHandler).syncClient.PeerManager().(*testPeerManager) + return h.SyncHandler.(*syncTreeHandler).syncClient.MessagePool().(*testMessagePool) } - return h.syncClient.PeerManager().(*testPeerManager) + return h.syncClient.MessagePool().(*testMessagePool) } func (h *testSyncHandler) tree() *broadcastTree { @@ -218,22 +218,22 @@ func (h *testSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.WaitGr }() } -// testPeerManager captures all other handlers and sends messages to them -type testPeerManager struct { +// testMessagePool captures all other handlers and sends messages to them +type testMessagePool struct { peerId string handlers map[string]*testSyncHandler log *messageLog } -func newTestPeerManager(peerId string, log *messageLog) *testPeerManager { - return &testPeerManager{handlers: map[string]*testSyncHandler{}, peerId: peerId, log: log} +func newTestMessagePool(peerId string, log *messageLog) *testMessagePool { + return &testMessagePool{handlers: map[string]*testSyncHandler{}, peerId: peerId, log: log} } -func (m *testPeerManager) addHandler(peerId string, handler *testSyncHandler) { +func (m *testMessagePool) addHandler(peerId string, handler *testSyncHandler) { m.handlers[peerId] = handler } -func (m *testPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { +func (m *testMessagePool) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { pMsg := protocolMsg{ msg: msg, senderId: m.peerId, @@ -243,7 +243,7 @@ func (m *testPeerManager) SendPeer(ctx context.Context, peerId string, msg *spac return m.handlers[peerId].send(context.Background(), pMsg) } -func (m *testPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { +func (m *testMessagePool) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { for _, handler := range m.handlers { pMsg := protocolMsg{ msg: msg, @@ -256,7 +256,19 @@ func (m *testPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.Obj return } -func (m *testPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { +func (m *testMessagePool) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { + panic("should not be called") +} + +func (m *testMessagePool) LastUsage() time.Time { + panic("should not be called") +} + +func (m *testMessagePool) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { + panic("should not be called") +} + +func (m *testMessagePool) SendSync(ctx context.Context, peerId string, message *spacesyncproto.ObjectSyncMessage) (reply *spacesyncproto.ObjectSyncMessage, err error) { panic("should not be called") } diff --git a/commonspace/objectsync/mock_objectsync/mock_objectsync.go b/commonspace/objectsync/mock_objectsync/mock_objectsync.go index aaca46a5..29ea2717 100644 --- a/commonspace/objectsync/mock_objectsync/mock_objectsync.go +++ b/commonspace/objectsync/mock_objectsync/mock_objectsync.go @@ -10,7 +10,8 @@ import ( objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" treechangeproto "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - peermanager "github.com/anytypeio/any-sync/commonspace/peermanager" + objectsync "github.com/anytypeio/any-sync/commonspace/objectsync" + spacesyncproto "github.com/anytypeio/any-sync/commonspace/spacesyncproto" gomock "github.com/golang/mock/gomock" ) @@ -109,18 +110,33 @@ func (mr *MockSyncClientMockRecorder) CreateNewTreeRequest() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateNewTreeRequest", reflect.TypeOf((*MockSyncClient)(nil).CreateNewTreeRequest)) } -// PeerManager mocks base method. -func (m *MockSyncClient) PeerManager() peermanager.PeerManager { +// MessagePool mocks base method. +func (m *MockSyncClient) MessagePool() objectsync.MessagePool { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "PeerManager") - ret0, _ := ret[0].(peermanager.PeerManager) + ret := m.ctrl.Call(m, "MessagePool") + ret0, _ := ret[0].(objectsync.MessagePool) return ret0 } -// PeerManager indicates an expected call of PeerManager. -func (mr *MockSyncClientMockRecorder) PeerManager() *gomock.Call { +// MessagePool indicates an expected call of MessagePool. +func (mr *MockSyncClientMockRecorder) MessagePool() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PeerManager", reflect.TypeOf((*MockSyncClient)(nil).PeerManager)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MessagePool", reflect.TypeOf((*MockSyncClient)(nil).MessagePool)) +} + +// SendSync mocks base method. +func (m *MockSyncClient) SendSync(arg0 context.Context, arg1, arg2 string, arg3 *treechangeproto.TreeSyncMessage) (*spacesyncproto.ObjectSyncMessage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendSync", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(*spacesyncproto.ObjectSyncMessage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SendSync indicates an expected call of SendSync. +func (mr *MockSyncClientMockRecorder) SendSync(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendSync", reflect.TypeOf((*MockSyncClient)(nil).SendSync), arg0, arg1, arg2, arg3) } // SendWithReply mocks base method. diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index 8bb9b5f2..7a7a0d69 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -24,7 +24,7 @@ var log = logger.NewNamed("common.commonspace.objectsync") type ObjectSync interface { LastUsage synchandler.SyncHandler - MessagePool() MessagePool + SyncClient() SyncClient Close() (err error) } @@ -59,9 +59,9 @@ func NewObjectSync( cancelSync: cancel, spaceIsDeleted: spaceIsDeleted, configuration: configuration, - syncClient: NewSyncClient(spaceId, peerManager, GetRequestFactory()), } os.messagePool = newMessagePool(peerManager, os.handleMessage) + os.syncClient = NewSyncClient(spaceId, os.messagePool, GetRequestFactory()) return os } @@ -79,7 +79,10 @@ func (s *objectSync) HandleMessage(ctx context.Context, senderId string, message } func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { - log := log.With(zap.String("objectId", msg.ObjectId), zap.String("replyId", msg.ReplyId)) + log := log.With( + zap.String("objectId", msg.ObjectId), + zap.String("requestId", msg.RequestId), + zap.String("replyId", msg.ReplyId)) if s.spaceIsDeleted.Load() { log = log.With(zap.Bool("isDeleted", true)) // preventing sync with other clients if they are not just syncing the settings tree @@ -97,11 +100,11 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp treeMsg := &treechangeproto.TreeSyncMessage{} err = proto.Unmarshal(msg.Payload, treeMsg) if err != nil { - return s.sendError(ctx, nil, spacesyncproto.ErrUnexpected, senderId, msg.ObjectId, msg.ReplyId) + return s.sendError(ctx, nil, spacesyncproto.ErrUnexpected, senderId, msg.ObjectId, msg.RequestId) } // this means that we don't have the tree locally and therefore can't return it if s.isEmptyFullSyncRequest(treeMsg) { - return s.sendError(ctx, nil, treechangeproto.ErrGetTree, senderId, msg.ObjectId, msg.ReplyId) + return s.sendError(ctx, nil, treechangeproto.ErrGetTree, senderId, msg.ObjectId, msg.RequestId) } } obj, err := s.objectGetter.GetObject(ctx, msg.ObjectId) @@ -112,8 +115,8 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp return obj.HandleMessage(ctx, senderId, msg) } -func (s *objectSync) MessagePool() MessagePool { - return s.messagePool +func (s *objectSync) SyncClient() SyncClient { + return s.syncClient } func (s *objectSync) unmarshallSendError(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, respErr error, senderId, objectId string) (err error) { @@ -122,7 +125,7 @@ func (s *objectSync) unmarshallSendError(ctx context.Context, msg *spacesyncprot if err != nil { return } - return s.sendError(ctx, unmarshalled.RootChange, respErr, senderId, objectId, msg.ReplyId) + return s.sendError(ctx, unmarshalled.RootChange, respErr, senderId, objectId, msg.RequestId) } func (s *objectSync) sendError(ctx context.Context, root *treechangeproto.RawTreeChangeWithId, respErr error, senderId, objectId, replyId string) (err error) { diff --git a/commonspace/objectsync/syncclient.go b/commonspace/objectsync/syncclient.go index 4a95cae1..e6fd515e 100644 --- a/commonspace/objectsync/syncclient.go +++ b/commonspace/objectsync/syncclient.go @@ -3,7 +3,6 @@ package objectsync import ( "context" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" ) @@ -11,21 +10,22 @@ type SyncClient interface { RequestFactory Broadcast(ctx context.Context, msg *treechangeproto.TreeSyncMessage) (err error) SendWithReply(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) - PeerManager() peermanager.PeerManager + SendSync(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage) (reply *spacesyncproto.ObjectSyncMessage, err error) + MessagePool() MessagePool } type syncClient struct { RequestFactory spaceId string - peerManager peermanager.PeerManager + messagePool MessagePool } func NewSyncClient( spaceId string, - peerManager peermanager.PeerManager, + messagePool MessagePool, factory RequestFactory) SyncClient { return &syncClient{ - peerManager: peerManager, + messagePool: messagePool, RequestFactory: factory, spaceId: spaceId, } @@ -36,7 +36,15 @@ func (s *syncClient) Broadcast(ctx context.Context, msg *treechangeproto.TreeSyn if err != nil { return } - return s.peerManager.Broadcast(ctx, objMsg) + return s.messagePool.Broadcast(ctx, objMsg) +} + +func (s *syncClient) SendSync(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage) (reply *spacesyncproto.ObjectSyncMessage, err error) { + objMsg, err := MarshallTreeMessage(msg, s.spaceId, objectId, "") + if err != nil { + return + } + return s.messagePool.SendSync(ctx, peerId, objMsg) } func (s *syncClient) SendWithReply(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) { @@ -44,11 +52,11 @@ func (s *syncClient) SendWithReply(ctx context.Context, peerId, objectId string, if err != nil { return } - return s.peerManager.SendPeer(ctx, peerId, objMsg) + return s.messagePool.SendPeer(ctx, peerId, objMsg) } -func (s *syncClient) PeerManager() peermanager.PeerManager { - return s.peerManager +func (s *syncClient) MessagePool() MessagePool { + return s.messagePool } func MarshallTreeMessage(message *treechangeproto.TreeSyncMessage, spaceId, objectId, replyId string) (objMsg *spacesyncproto.ObjectSyncMessage, err error) { diff --git a/commonspace/space.go b/commonspace/space.go index 88fb5692..10090b95 100644 --- a/commonspace/space.go +++ b/commonspace/space.go @@ -179,7 +179,7 @@ func (s *space) Init(ctx context.Context) (err error) { if err != nil { return } - s.aclList = syncacl.NewSyncAcl(aclList, s.objectSync.MessagePool()) + s.aclList = syncacl.NewSyncAcl(aclList, s.objectSync.SyncClient().MessagePool()) s.treeManager.AddObject(s.aclList) deletionState := settingsstate.NewObjectDeletionState(s.storage) @@ -284,7 +284,7 @@ func (s *space) PutTree(ctx context.Context, payload treestorage.TreeStorageCrea } deps := synctree.BuildDeps{ SpaceId: s.id, - ObjectSync: s.objectSync, + SyncClient: s.objectSync.SyncClient(), Configuration: s.configuration, HeadNotifiable: s.headSync, Listener: listener, @@ -322,7 +322,7 @@ func (s *space) BuildTree(ctx context.Context, id string, opts BuildTreeOpts) (t deps := synctree.BuildDeps{ SpaceId: s.id, - ObjectSync: s.objectSync, + SyncClient: s.objectSync.SyncClient(), Configuration: s.configuration, HeadNotifiable: s.headSync, Listener: opts.Listener, From 4c71a9a1c4b3ed266750bbf130c37833b6f7180f Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 19 Apr 2023 22:52:06 +0200 Subject: [PATCH 20/27] Simplify request factory --- commonspace/object/tree/synctree/treeremotegetter.go | 3 +-- commonspace/object/tree/synctree/utils_test.go | 8 ++++---- commonspace/objectsync/objectsync.go | 2 +- commonspace/objectsync/requestfactory.go | 6 ++---- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/commonspace/object/tree/synctree/treeremotegetter.go b/commonspace/object/tree/synctree/treeremotegetter.go index bcf547b0..3156373f 100644 --- a/commonspace/object/tree/synctree/treeremotegetter.go +++ b/commonspace/object/tree/synctree/treeremotegetter.go @@ -6,7 +6,6 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" - "github.com/anytypeio/any-sync/commonspace/objectsync" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/net/peer" "github.com/anytypeio/any-sync/net/rpc/rpcerr" @@ -47,7 +46,7 @@ func (t treeRemoteGetter) getPeers(ctx context.Context) (peerIds []string, err e } func (t treeRemoteGetter) treeRequest(ctx context.Context, peerId string) (msg *treechangeproto.TreeSyncMessage, err error) { - newTreeRequest := objectsync.GetRequestFactory().CreateNewTreeRequest() + newTreeRequest := t.deps.SyncClient.CreateNewTreeRequest() resp, err := t.deps.SyncClient.SendSync(ctx, peerId, t.treeId, newTreeRequest) if err != nil { return diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index 451f7536..1d844e90 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -94,7 +94,7 @@ type testSyncHandler struct { // createSyncHandler creates a sync handler when a tree is already created func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *testSyncHandler { - factory := objectsync.GetRequestFactory() + factory := objectsync.NewRequestFactory() syncClient := objectsync.NewSyncClient(spaceId, newTestMessagePool(peerId, log), factory) netTree := &broadcastTree{ ObjectTree: objTree, @@ -106,7 +106,7 @@ func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, lo // createEmptySyncHandler creates a sync handler when the tree will be provided later (this emulates the situation when we have no tree) func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *testSyncHandler { - factory := objectsync.GetRequestFactory() + factory := objectsync.NewRequestFactory() syncClient := objectsync.NewSyncClient(spaceId, newTestMessagePool(peerId, log), factory) batcher := mb.New[protocolMsg](0) @@ -138,7 +138,7 @@ func (h *testSyncHandler) HandleMessage(ctx context.Context, senderId string, re return } if unmarshalled.Content.GetFullSyncResponse() == nil { - newTreeRequest := objectsync.GetRequestFactory().CreateNewTreeRequest() + newTreeRequest := objectsync.NewRequestFactory().CreateNewTreeRequest() var objMsg *spacesyncproto.ObjectSyncMessage objMsg, err = objectsync.MarshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") if err != nil { @@ -165,7 +165,7 @@ func (h *testSyncHandler) HandleMessage(ctx context.Context, senderId string, re } h.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, h.syncClient, syncstatus.NewNoOpSyncStatus()) var objMsg *spacesyncproto.ObjectSyncMessage - newTreeRequest := objectsync.GetRequestFactory().CreateHeadUpdate(netTree, res.Added) + newTreeRequest := objectsync.NewRequestFactory().CreateHeadUpdate(netTree, res.Added) objMsg, err = objectsync.MarshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") if err != nil { return diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index 7a7a0d69..02e6019c 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -61,7 +61,7 @@ func NewObjectSync( configuration: configuration, } os.messagePool = newMessagePool(peerManager, os.handleMessage) - os.syncClient = NewSyncClient(spaceId, os.messagePool, GetRequestFactory()) + os.syncClient = NewSyncClient(spaceId, os.messagePool, NewRequestFactory()) return os } diff --git a/commonspace/objectsync/requestfactory.go b/commonspace/objectsync/requestfactory.go index 0608ad33..4d6a9124 100644 --- a/commonspace/objectsync/requestfactory.go +++ b/commonspace/objectsync/requestfactory.go @@ -14,10 +14,8 @@ type RequestFactory interface { CreateFullSyncResponse(t objecttree.ObjectTree, theirHeads, theirSnapshotPath []string) (*treechangeproto.TreeSyncMessage, error) } -var sharedFactory = &requestFactory{} - -func GetRequestFactory() RequestFactory { - return sharedFactory +func NewRequestFactory() RequestFactory { + return &requestFactory{} } type requestFactory struct{} From 25f90d5f5f767df3037e40f322c4d98fdbd071b9 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 20 Apr 2023 00:04:16 +0200 Subject: [PATCH 21/27] Fix proto --- .../treechangeproto/protos/treechange.proto | 3 +- .../tree/treechangeproto/treechange.pb.go | 146 ++++++++++++------ 2 files changed, 101 insertions(+), 48 deletions(-) diff --git a/commonspace/object/tree/treechangeproto/protos/treechange.proto b/commonspace/object/tree/treechangeproto/protos/treechange.proto index ae7cebd5..eb331567 100644 --- a/commonspace/object/tree/treechangeproto/protos/treechange.proto +++ b/commonspace/object/tree/treechangeproto/protos/treechange.proto @@ -101,7 +101,8 @@ message TreeFullSyncResponse { // TreeErrorResponse is an error sent as a response for a full sync request message TreeErrorResponse { - uint64 errCode = 1; + string error = 1; + uint64 errCode = 2; } // TreeChangeInfo is used internally in Tree implementation for ease of marshalling diff --git a/commonspace/object/tree/treechangeproto/treechange.pb.go b/commonspace/object/tree/treechangeproto/treechange.pb.go index f4098fc6..34d57487 100644 --- a/commonspace/object/tree/treechangeproto/treechange.pb.go +++ b/commonspace/object/tree/treechangeproto/treechange.pb.go @@ -722,7 +722,8 @@ func (m *TreeFullSyncResponse) GetSnapshotPath() []string { // TreeErrorResponse is an error sent as a response for a full sync request type TreeErrorResponse struct { - ErrCode uint64 `protobuf:"varint,1,opt,name=errCode,proto3" json:"errCode,omitempty"` + Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` + ErrCode uint64 `protobuf:"varint,2,opt,name=errCode,proto3" json:"errCode,omitempty"` } func (m *TreeErrorResponse) Reset() { *m = TreeErrorResponse{} } @@ -758,6 +759,13 @@ func (m *TreeErrorResponse) XXX_DiscardUnknown() { var xxx_messageInfo_TreeErrorResponse proto.InternalMessageInfo +func (m *TreeErrorResponse) GetError() string { + if m != nil { + return m.Error + } + return "" +} + func (m *TreeErrorResponse) GetErrCode() uint64 { if m != nil { return m.ErrCode @@ -838,53 +846,54 @@ func init() { } var fileDescriptor_5033f0301ef9b772 = []byte{ - // 732 bytes of a gzipped FileDescriptorProto + // 741 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0x4f, 0x4f, 0xfb, 0x46, 0x10, 0xf5, 0x3a, 0x81, 0x90, 0x49, 0x08, 0xe9, 0xc2, 0xc1, 0x42, 0xad, 0x6b, 0x59, 0x55, 0x1b, - 0x55, 0x2a, 0x48, 0xf4, 0xd4, 0xaa, 0x12, 0x2a, 0x29, 0x90, 0x08, 0xb5, 0x45, 0xcb, 0x9f, 0x4a, - 0xdc, 0x16, 0x7b, 0x42, 0x5c, 0x25, 0xb6, 0xeb, 0xdd, 0x94, 0xe6, 0x03, 0xf4, 0xd2, 0x4a, 0x88, - 0xaf, 0xd4, 0x5b, 0x8f, 0x1c, 0x39, 0xfe, 0x04, 0x5f, 0xe4, 0x27, 0xef, 0xc6, 0x89, 0xed, 0xe4, - 0xc0, 0x8d, 0x8b, 0xe3, 0x79, 0x9e, 0x79, 0xf3, 0xe6, 0xed, 0x9f, 0xc0, 0xa1, 0x17, 0x8d, 0xc7, - 0x51, 0x28, 0x62, 0xee, 0xe1, 0x7e, 0x74, 0xfb, 0x3b, 0x7a, 0x72, 0x5f, 0x26, 0x88, 0xea, 0xe1, - 0x0d, 0x79, 0x78, 0x87, 0x71, 0x12, 0xc9, 0x68, 0x5f, 0x3d, 0x45, 0x0e, 0xde, 0x53, 0x08, 0x85, - 0x05, 0xe2, 0x3e, 0x13, 0x00, 0x16, 0x45, 0xb2, 0xab, 0x42, 0xfa, 0x29, 0xd4, 0xb9, 0x37, 0xea, - 0x21, 0xf7, 0xfb, 0xbe, 0x45, 0x1c, 0xd2, 0xa9, 0xb3, 0x05, 0x40, 0x2d, 0xa8, 0xa9, 0xae, 0x7d, - 0xdf, 0x32, 0xd5, 0xb7, 0x2c, 0xa4, 0x36, 0x80, 0x26, 0xbc, 0x9c, 0xc6, 0x68, 0x55, 0xd4, 0xc7, - 0x1c, 0x92, 0xf2, 0xca, 0x60, 0x8c, 0x42, 0xf2, 0x71, 0x6c, 0x55, 0x1d, 0xd2, 0xa9, 0xb0, 0x05, - 0x40, 0x29, 0x54, 0x05, 0xa2, 0x6f, 0xad, 0x39, 0xa4, 0xd3, 0x64, 0xea, 0x9d, 0xee, 0xc2, 0x46, - 0xe0, 0x63, 0x28, 0x03, 0x39, 0xb5, 0xd6, 0x15, 0x3e, 0x8f, 0xe9, 0x17, 0xb0, 0xa9, 0xb9, 0xcf, - 0xf9, 0x74, 0x14, 0x71, 0xdf, 0xaa, 0xa9, 0x84, 0x22, 0xe8, 0x3e, 0x98, 0x00, 0x97, 0x09, 0xe2, - 0x6c, 0x34, 0x07, 0x1a, 0xe9, 0xdc, 0x7a, 0x14, 0x61, 0x11, 0xa7, 0xd2, 0xa9, 0xb3, 0x3c, 0x54, - 0x1c, 0xde, 0x2c, 0x0f, 0xff, 0x25, 0xb4, 0x44, 0xc8, 0x63, 0x31, 0x8c, 0xe4, 0x11, 0x17, 0xa9, - 0x07, 0x7a, 0xcc, 0x12, 0x9a, 0xf6, 0xd1, 0x3a, 0xc4, 0x4f, 0x5c, 0x72, 0x35, 0x6c, 0x93, 0xe5, - 0xa1, 0xb4, 0x4f, 0x82, 0xdc, 0x3f, 0xc3, 0x69, 0x5f, 0xcf, 0x5c, 0x67, 0x0b, 0xa0, 0x68, 0xd5, - 0x7a, 0xd9, 0xaa, 0xbc, 0x2d, 0xb5, 0x92, 0x2d, 0x36, 0x40, 0x20, 0x2e, 0x66, 0x6a, 0xac, 0x0d, - 0x87, 0x74, 0x36, 0x58, 0x0e, 0x71, 0x4f, 0x61, 0x93, 0xf1, 0xfb, 0x9c, 0x25, 0x16, 0xd4, 0xe2, - 0x99, 0x83, 0x44, 0x71, 0x65, 0x61, 0x2a, 0x42, 0x04, 0x77, 0x21, 0x97, 0x93, 0x04, 0x95, 0x15, - 0x4d, 0xb6, 0x00, 0xdc, 0x2e, 0x6c, 0x17, 0x88, 0x7e, 0x0b, 0xe4, 0x50, 0x2b, 0x4f, 0xf8, 0xbd, - 0x86, 0x66, 0x84, 0x0b, 0x80, 0xb6, 0xc0, 0x0c, 0x32, 0x5b, 0xcd, 0xc0, 0x77, 0x1f, 0x08, 0x6c, - 0xa5, 0x14, 0x17, 0xd3, 0xd0, 0xfb, 0x19, 0x85, 0xe0, 0x77, 0x48, 0xbf, 0x87, 0x9a, 0x17, 0x85, - 0x12, 0x43, 0xa9, 0xea, 0x1b, 0x07, 0xce, 0x5e, 0x6e, 0xf7, 0x66, 0xd9, 0x5d, 0x9d, 0x72, 0xcd, - 0x47, 0x13, 0x64, 0x59, 0x01, 0x3d, 0x04, 0x48, 0xe6, 0x1b, 0x59, 0xf5, 0x69, 0x1c, 0x7c, 0x9e, - 0x2f, 0x5f, 0x21, 0x99, 0xe5, 0x4a, 0xdc, 0xff, 0x4c, 0xd8, 0x59, 0xd5, 0x82, 0xfe, 0x00, 0x30, - 0x44, 0xee, 0x5f, 0xc5, 0x3e, 0x97, 0x38, 0x13, 0xb6, 0x5b, 0x16, 0xd6, 0x9b, 0x67, 0xf4, 0x0c, - 0x96, 0xcb, 0xa7, 0x67, 0xb0, 0x35, 0x98, 0x8c, 0x46, 0x29, 0x2b, 0xc3, 0x3f, 0x26, 0x28, 0xe4, - 0x2a, 0x71, 0x29, 0xc5, 0x49, 0x31, 0xad, 0x67, 0xb0, 0x72, 0x25, 0xfd, 0x05, 0xda, 0x0b, 0x48, - 0xc4, 0x51, 0x28, 0xf4, 0x69, 0x5b, 0xe1, 0xd4, 0x49, 0x29, 0xaf, 0x67, 0xb0, 0xa5, 0x5a, 0x7a, - 0x0c, 0x9b, 0x98, 0x24, 0x51, 0x32, 0x27, 0xab, 0x2a, 0xb2, 0xcf, 0xca, 0x64, 0xc7, 0xf9, 0xa4, - 0x9e, 0xc1, 0x8a, 0x55, 0x47, 0x35, 0x58, 0xfb, 0x33, 0xb5, 0xca, 0xfd, 0x9b, 0x40, 0xab, 0xe8, - 0x06, 0xdd, 0x81, 0xb5, 0xd4, 0x8d, 0xec, 0xc4, 0xe9, 0x80, 0x7e, 0x07, 0xb5, 0xd9, 0x91, 0xb0, - 0x4c, 0xa7, 0xf2, 0x96, 0xa5, 0xca, 0xf2, 0xa9, 0x0b, 0xcd, 0xec, 0xc8, 0x9d, 0x73, 0x39, 0xb4, - 0x2a, 0x8a, 0xb7, 0x80, 0xb9, 0xff, 0x10, 0xd8, 0x5e, 0x61, 0xe9, 0xfb, 0x88, 0xf9, 0x97, 0xe8, - 0x8d, 0x55, 0x5e, 0x91, 0xf7, 0x51, 0xf3, 0x0d, 0x7c, 0xb2, 0xb4, 0xa2, 0xe9, 0x4d, 0x80, 0x49, - 0xd2, 0x8d, 0x7c, 0xbd, 0xbf, 0xab, 0x2c, 0x0b, 0xdd, 0x6b, 0xbd, 0xa0, 0xba, 0x5f, 0x3f, 0x1c, - 0x44, 0xa5, 0xbb, 0x9e, 0x2c, 0xdd, 0xf5, 0x4b, 0xb7, 0xb3, 0xb9, 0xe2, 0x76, 0xfe, 0xfa, 0x06, - 0x40, 0x49, 0x48, 0x9b, 0x08, 0xda, 0x02, 0xb8, 0x0a, 0xf1, 0xaf, 0x18, 0x3d, 0x89, 0x7e, 0xdb, - 0xa0, 0x6d, 0x68, 0x9e, 0xa2, 0x9c, 0xeb, 0x6c, 0x13, 0x6a, 0xc1, 0x4e, 0x69, 0x31, 0xf5, 0x17, - 0x93, 0xb6, 0xa1, 0xa1, 0x5e, 0x7f, 0x1d, 0x0c, 0x04, 0xca, 0xf6, 0x63, 0xe5, 0xe8, 0xc7, 0xff, - 0x5f, 0x6c, 0xf2, 0xf4, 0x62, 0x93, 0x0f, 0x2f, 0x36, 0x79, 0x7c, 0xb5, 0x8d, 0xa7, 0x57, 0xdb, - 0x78, 0x7e, 0xb5, 0x8d, 0x9b, 0xaf, 0xde, 0xf8, 0xdf, 0x79, 0xbb, 0xae, 0x7e, 0xbe, 0xfd, 0x18, - 0x00, 0x00, 0xff, 0xff, 0xb5, 0x68, 0xf7, 0x3f, 0x6d, 0x07, 0x00, 0x00, + 0xf5, 0x00, 0x12, 0x3d, 0xb5, 0xaa, 0x84, 0x4a, 0x0a, 0x24, 0x42, 0x6d, 0xd1, 0xf2, 0xa7, 0x12, + 0xb7, 0xc5, 0x9e, 0x10, 0x57, 0x89, 0xed, 0x7a, 0x37, 0xa5, 0xf9, 0x00, 0xbd, 0xb4, 0x12, 0xe2, + 0x2b, 0xf5, 0xf6, 0x3b, 0x72, 0xe4, 0xf8, 0x13, 0x7c, 0x91, 0x9f, 0xbc, 0x1b, 0x27, 0xb6, 0x93, + 0x03, 0x37, 0x2e, 0x89, 0xe7, 0x79, 0xe6, 0xed, 0x9b, 0x37, 0xde, 0x5d, 0x38, 0xf4, 0xa2, 0xf1, + 0x38, 0x0a, 0x45, 0xcc, 0x3d, 0xdc, 0x8f, 0x6e, 0xff, 0x40, 0x4f, 0xee, 0xcb, 0x04, 0x51, 0xfd, + 0x78, 0x43, 0x1e, 0xde, 0x61, 0x9c, 0x44, 0x32, 0xda, 0x57, 0xbf, 0x22, 0x07, 0xef, 0x29, 0x84, + 0xc2, 0x02, 0x71, 0x9f, 0x09, 0x00, 0x8b, 0x22, 0xd9, 0x55, 0x21, 0xfd, 0x1c, 0xea, 0xdc, 0x1b, + 0xf5, 0x90, 0xfb, 0x7d, 0xdf, 0x22, 0x0e, 0xe9, 0xd4, 0xd9, 0x02, 0xa0, 0x16, 0xd4, 0xd4, 0xaa, + 0x7d, 0xdf, 0x32, 0xd5, 0xbb, 0x2c, 0xa4, 0x36, 0x80, 0x26, 0xbc, 0x9c, 0xc6, 0x68, 0x55, 0xd4, + 0xcb, 0x1c, 0x92, 0xf2, 0xca, 0x60, 0x8c, 0x42, 0xf2, 0x71, 0x6c, 0x55, 0x1d, 0xd2, 0xa9, 0xb0, + 0x05, 0x40, 0x29, 0x54, 0x05, 0xa2, 0x6f, 0xad, 0x39, 0xa4, 0xd3, 0x64, 0xea, 0x99, 0xee, 0xc2, + 0x46, 0xe0, 0x63, 0x28, 0x03, 0x39, 0xb5, 0xd6, 0x15, 0x3e, 0x8f, 0xe9, 0x57, 0xb0, 0xa9, 0xb9, + 0xcf, 0xf9, 0x74, 0x14, 0x71, 0xdf, 0xaa, 0xa9, 0x84, 0x22, 0xe8, 0x3e, 0x98, 0x00, 0x97, 0x09, + 0xe2, 0xac, 0x35, 0x07, 0x1a, 0x69, 0xdf, 0xba, 0x15, 0x61, 0x11, 0xa7, 0xd2, 0xa9, 0xb3, 0x3c, + 0x54, 0x6c, 0xde, 0x2c, 0x37, 0xff, 0x35, 0xb4, 0x44, 0xc8, 0x63, 0x31, 0x8c, 0xe4, 0x11, 0x17, + 0xa9, 0x07, 0xba, 0xcd, 0x12, 0x9a, 0xae, 0xa3, 0x75, 0x88, 0x9f, 0xb9, 0xe4, 0xaa, 0xd9, 0x26, + 0xcb, 0x43, 0xe9, 0x3a, 0x09, 0x72, 0xff, 0x0c, 0xa7, 0x7d, 0xdd, 0x73, 0x9d, 0x2d, 0x80, 0xa2, + 0x55, 0xeb, 0x65, 0xab, 0xf2, 0xb6, 0xd4, 0x4a, 0xb6, 0xd8, 0x00, 0x81, 0xb8, 0x98, 0xa9, 0xb1, + 0x36, 0x1c, 0xd2, 0xd9, 0x60, 0x39, 0xc4, 0x3d, 0x85, 0x4d, 0xc6, 0xef, 0x73, 0x96, 0x58, 0x50, + 0x8b, 0x67, 0x0e, 0x12, 0xc5, 0x95, 0x85, 0xa9, 0x08, 0x11, 0xdc, 0x85, 0x5c, 0x4e, 0x12, 0x54, + 0x56, 0x34, 0xd9, 0x02, 0x70, 0xbb, 0xb0, 0x5d, 0x20, 0xfa, 0x3d, 0x90, 0x43, 0xad, 0x3c, 0xe1, + 0xf7, 0x1a, 0x9a, 0x11, 0x2e, 0x00, 0xda, 0x02, 0x33, 0xc8, 0x6c, 0x35, 0x03, 0xdf, 0x7d, 0x20, + 0xb0, 0x95, 0x52, 0x5c, 0x4c, 0x43, 0xef, 0x17, 0x14, 0x82, 0xdf, 0x21, 0xfd, 0x01, 0x6a, 0x5e, + 0x14, 0x4a, 0x0c, 0xa5, 0xaa, 0x6f, 0x1c, 0x38, 0x7b, 0xb9, 0xaf, 0x37, 0xcb, 0xee, 0xea, 0x94, + 0x6b, 0x3e, 0x9a, 0x20, 0xcb, 0x0a, 0xe8, 0x21, 0x40, 0x32, 0xff, 0x90, 0xd5, 0x3a, 0x8d, 0x83, + 0x2f, 0xf3, 0xe5, 0x2b, 0x24, 0xb3, 0x5c, 0x89, 0xfb, 0xbf, 0x09, 0x3b, 0xab, 0x96, 0xa0, 0x3f, + 0x02, 0x0c, 0x91, 0xfb, 0x57, 0xb1, 0xcf, 0x25, 0xce, 0x84, 0xed, 0x96, 0x85, 0xf5, 0xe6, 0x19, + 0x3d, 0x83, 0xe5, 0xf2, 0xe9, 0x19, 0x6c, 0x0d, 0x26, 0xa3, 0x51, 0xca, 0xca, 0xf0, 0xcf, 0x09, + 0x0a, 0xb9, 0x4a, 0x5c, 0x4a, 0x71, 0x52, 0x4c, 0xeb, 0x19, 0xac, 0x5c, 0x49, 0x7f, 0x85, 0xf6, + 0x02, 0x12, 0x71, 0x14, 0x0a, 0xbd, 0xdb, 0x56, 0x38, 0x75, 0x52, 0xca, 0xeb, 0x19, 0x6c, 0xa9, + 0x96, 0x1e, 0xc3, 0x26, 0x26, 0x49, 0x94, 0xcc, 0xc9, 0xaa, 0x8a, 0xec, 0x8b, 0x32, 0xd9, 0x71, + 0x3e, 0xa9, 0x67, 0xb0, 0x62, 0xd5, 0x51, 0x0d, 0xd6, 0xfe, 0x4a, 0xad, 0x72, 0xff, 0x21, 0xd0, + 0x2a, 0xba, 0x41, 0x77, 0x60, 0x2d, 0x75, 0x23, 0xdb, 0x71, 0x3a, 0xa0, 0xdf, 0x43, 0x6d, 0xb6, + 0x25, 0x2c, 0xd3, 0xa9, 0xbc, 0x65, 0x54, 0x59, 0x3e, 0x75, 0xa1, 0x99, 0x6d, 0xb9, 0x73, 0x2e, + 0x87, 0x56, 0x45, 0xf1, 0x16, 0x30, 0xf7, 0x5f, 0x02, 0xdb, 0x2b, 0x2c, 0x7d, 0x1f, 0x31, 0xff, + 0x11, 0xfd, 0x61, 0x95, 0x27, 0xf2, 0x3e, 0x6a, 0xba, 0xf0, 0xd9, 0xd2, 0x44, 0x53, 0x25, 0x6a, + 0xa2, 0xb3, 0x33, 0x5f, 0x07, 0xe9, 0xf9, 0x80, 0x49, 0xd2, 0x8d, 0x7c, 0xbd, 0x9f, 0xaa, 0x2c, + 0x0b, 0xdd, 0x6b, 0x3d, 0x66, 0xad, 0xa2, 0x1f, 0x0e, 0xa2, 0xd2, 0x0d, 0x40, 0x96, 0x6e, 0x80, + 0xa5, 0x33, 0xdb, 0x5c, 0x71, 0x66, 0x7f, 0x7b, 0x03, 0xa0, 0x84, 0xa5, 0x8b, 0x08, 0xda, 0x02, + 0xb8, 0x0a, 0xf1, 0xef, 0x18, 0x3d, 0x89, 0x7e, 0xdb, 0xa0, 0x6d, 0x68, 0x9e, 0xa2, 0x9c, 0xab, + 0x6f, 0x13, 0x6a, 0xc1, 0x4e, 0x69, 0xc4, 0xfa, 0x8d, 0x49, 0xdb, 0xd0, 0x50, 0x8f, 0xbf, 0x0d, + 0x06, 0x02, 0x65, 0xfb, 0xb1, 0x72, 0xf4, 0xd3, 0x87, 0x17, 0x9b, 0x3c, 0xbd, 0xd8, 0xe4, 0xe3, + 0x8b, 0x4d, 0x1e, 0x5f, 0x6d, 0xe3, 0xe9, 0xd5, 0x36, 0x9e, 0x5f, 0x6d, 0xe3, 0xe6, 0x9b, 0x37, + 0xde, 0xa8, 0xb7, 0xeb, 0xea, 0xef, 0xbb, 0x4f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x1b, 0x35, 0x02, + 0x10, 0x83, 0x07, 0x00, 0x00, } func (m *RootChange) Marshal() (dAtA []byte, err error) { @@ -1464,7 +1473,14 @@ func (m *TreeErrorResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { if m.ErrCode != 0 { i = encodeVarintTreechange(dAtA, i, uint64(m.ErrCode)) i-- - dAtA[i] = 0x8 + dAtA[i] = 0x10 + } + if len(m.Error) > 0 { + i -= len(m.Error) + copy(dAtA[i:], m.Error) + i = encodeVarintTreechange(dAtA, i, uint64(len(m.Error))) + i-- + dAtA[i] = 0xa } return len(dAtA) - i, nil } @@ -1792,6 +1808,10 @@ func (m *TreeErrorResponse) Size() (n int) { } var l int _ = l + l = len(m.Error) + if l > 0 { + n += 1 + l + sovTreechange(uint64(l)) + } if m.ErrCode != 0 { n += 1 + sovTreechange(uint64(m.ErrCode)) } @@ -3393,6 +3413,38 @@ func (m *TreeErrorResponse) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTreechange + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTreechange + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthTreechange + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Error = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ErrCode", wireType) } From 6f42dc2f7a318f936857e711104dbdc6244e56ab Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 20 Apr 2023 11:26:53 +0200 Subject: [PATCH 22/27] Refactor genchanges --- .../object/tree/synctree/utils_test.go | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index 1d844e90..d0b207b4 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -382,7 +382,7 @@ type genResult struct { snapshotId string } -// genChanges generates several levels of tree changes where each level is connected with only previous one +// genChanges generates several levels of tree changes where each level is connected only with previous one func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res genResult) { src := rand.NewSource(time.Now().Unix()) rnd := rand.New(src) @@ -393,10 +393,18 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge prevHeads = append(prevHeads, params.prevHeads...) for i := 0; i < params.levels; i++ { - if params.isSnapshot() { - newId := fmt.Sprintf("%s.%d.%d", params.prefix, params.startIdx+i, 0) - newCh := creator.CreateRaw(newId, params.aclId, snapshotId, true, prevHeads...) + var ( + newHeads []string + usedIds = map[string]struct{}{} + ) + newChange := func(isSnapshot bool, idx int, prevIds []string) string { + newId := fmt.Sprintf("%s.%d.%d", params.prefix, params.startIdx+i, idx) + newCh := creator.CreateRaw(newId, params.aclId, snapshotId, isSnapshot, prevIds...) res.changes = append(res.changes, newCh) + return newId + } + if params.isSnapshot() { + newId := newChange(true, 0, prevHeads) prevHeads = []string{newId} snapshotId = newId continue @@ -405,10 +413,6 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge if perLevel == 0 { perLevel = 1 } - var ( - newHeads []string - usedIds = map[string]struct{}{} - ) for j := 0; j < perLevel; j++ { prevConns := rnd.Intn(len(prevHeads)) if prevConns == 0 { @@ -428,14 +432,12 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge prevHeads = unusedIds prevConns = len(prevHeads) } - var prevChId []string + var prevIds []string for k := 0; k < prevConns; k++ { - prevChId = append(prevChId, prevHeads[k]) + prevIds = append(prevIds, prevHeads[k]) usedIds[prevHeads[k]] = struct{}{} } - newId := fmt.Sprintf("%s.%d.%d", params.prefix, params.startIdx+i, j) - newCh := creator.CreateRaw(newId, params.aclId, snapshotId, false, prevChId...) - res.changes = append(res.changes, newCh) + newId := newChange(false, j, prevIds) newHeads = append(newHeads, newId) } prevHeads = newHeads From 2894b8ef665ec2d86aac722b92028681f3948c7b Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 20 Apr 2023 11:46:19 +0200 Subject: [PATCH 23/27] Fix full sync request condition --- commonspace/objectsync/objectsync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index 02e6019c..e22db4bd 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -134,5 +134,5 @@ func (s *objectSync) sendError(ctx context.Context, root *treechangeproto.RawTre } func (s *objectSync) isEmptyFullSyncRequest(msg *treechangeproto.TreeSyncMessage) bool { - return len(msg.GetContent().GetFullSyncRequest().GetHeads()) == 0 + return msg.GetContent().GetFullSyncRequest() != nil && len(msg.GetContent().GetFullSyncRequest().GetHeads()) == 0 } From 7979c48ca3e3242184f1a479725da1022628ceb6 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 20 Apr 2023 12:46:36 +0200 Subject: [PATCH 24/27] Change error to typed in treeremotegetter --- commonspace/object/tree/synctree/treeremotegetter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commonspace/object/tree/synctree/treeremotegetter.go b/commonspace/object/tree/synctree/treeremotegetter.go index 3156373f..7c401a5e 100644 --- a/commonspace/object/tree/synctree/treeremotegetter.go +++ b/commonspace/object/tree/synctree/treeremotegetter.go @@ -119,7 +119,7 @@ func (t treeRemoteGetter) getTree(ctx context.Context) (treeStorage treestorage. err = rpcerr.Err(errResp.ErrCode) return case resp.GetContent().GetFullSyncResponse() == nil: - err = fmt.Errorf("expected to get full sync response, but got something else") + err = treechangeproto.ErrUnexpected return default: break From e896a88d29384a1c942c0ae6ae9c1dba3f70a9fd Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 20 Apr 2023 12:48:25 +0200 Subject: [PATCH 25/27] Change test name for fuzzy merge --- commonspace/object/tree/synctree/syncprotocol_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commonspace/object/tree/synctree/syncprotocol_test.go b/commonspace/object/tree/synctree/syncprotocol_test.go index cc7ebec4..d7c7a935 100644 --- a/commonspace/object/tree/synctree/syncprotocol_test.go +++ b/commonspace/object/tree/synctree/syncprotocol_test.go @@ -59,7 +59,7 @@ func TestEmptyClientGetsFullHistory(t *testing.T) { require.Len(t, fullResponseMsg.changes, 2) } -func TestRandomMerge(t *testing.T) { +func TestTreeFuzzyMerge(t *testing.T) { var ( rnd = rand.New(rand.NewSource(time.Now().Unix())) levels = 20 From 5d13fb6a6ebee4ed634afd91fb326030bcf0a18a Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 20 Apr 2023 12:52:25 +0200 Subject: [PATCH 26/27] Change config parameter name --- commonspace/config.go | 6 +++--- commonspace/spaceservice.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/commonspace/config.go b/commonspace/config.go index f7444fdd..f23f8e33 100644 --- a/commonspace/config.go +++ b/commonspace/config.go @@ -5,7 +5,7 @@ type ConfigGetter interface { } type Config struct { - GCTTL int `yaml:"gcTTL"` - SyncPeriod int `yaml:"syncPeriod"` - TreeInMemoryData bool `yaml:"treeInMemoryData"` + GCTTL int `yaml:"gcTTL"` + SyncPeriod int `yaml:"syncPeriod"` + TreeNoInMemoryData bool `yaml:"treeNoInMemoryData"` } diff --git a/commonspace/spaceservice.go b/commonspace/spaceservice.go index ef59e336..d59ae751 100644 --- a/commonspace/spaceservice.go +++ b/commonspace/spaceservice.go @@ -154,7 +154,7 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) { syncStatus = syncstatus.NewSyncStatusProvider(st.Id(), syncstatus.DefaultDeps(lastConfiguration, st)) } var builder objecttree.BuildObjectTreeFunc - if s.config.TreeInMemoryData { + if s.config.TreeNoInMemoryData { builder = objecttree.BuildEmptyDataObjectTree } else { builder = objecttree.BuildObjectTree From 5c954d1bdd6ec733c651f3a5765a7760f9903b98 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 20 Apr 2023 17:12:39 +0200 Subject: [PATCH 27/27] Change inmemory parameter naming --- commonspace/config.go | 6 +++--- commonspace/spaceservice.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/commonspace/config.go b/commonspace/config.go index f23f8e33..e5485068 100644 --- a/commonspace/config.go +++ b/commonspace/config.go @@ -5,7 +5,7 @@ type ConfigGetter interface { } type Config struct { - GCTTL int `yaml:"gcTTL"` - SyncPeriod int `yaml:"syncPeriod"` - TreeNoInMemoryData bool `yaml:"treeNoInMemoryData"` + GCTTL int `yaml:"gcTTL"` + SyncPeriod int `yaml:"syncPeriod"` + KeepTreeDataInMemory bool `yaml:"keepTreeDataInMemory"` } diff --git a/commonspace/spaceservice.go b/commonspace/spaceservice.go index d59ae751..addf1f7f 100644 --- a/commonspace/spaceservice.go +++ b/commonspace/spaceservice.go @@ -154,10 +154,10 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) { syncStatus = syncstatus.NewSyncStatusProvider(st.Id(), syncstatus.DefaultDeps(lastConfiguration, st)) } var builder objecttree.BuildObjectTreeFunc - if s.config.TreeNoInMemoryData { - builder = objecttree.BuildEmptyDataObjectTree - } else { + if s.config.KeepTreeDataInMemory { builder = objecttree.BuildObjectTree + } else { + builder = objecttree.BuildEmptyDataObjectTree } peerManager, err := s.peermanagerProvider.NewPeerManager(ctx, id)