From accf68fc6000d470747d8b15fcb0469c479e2806 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 13 Mar 2023 21:47:07 +0100 Subject: [PATCH 01/45] Start sync tests --- .../object/acl/list/aclrecordbuilder_test.go | 8 +- .../object/tree/objecttree/objecttree_test.go | 142 ++++++------------ .../object/tree/objecttree/testutils.go | 76 ++++++++++ .../object/tree/synctree/syncclient.go | 20 +-- .../object/tree/synctree/syncclient_test.go | 54 +++++++ commonspace/object/tree/synctree/synctree.go | 3 +- 6 files changed, 190 insertions(+), 113 deletions(-) create mode 100644 commonspace/object/tree/objecttree/testutils.go create mode 100644 commonspace/object/tree/synctree/syncclient_test.go diff --git a/commonspace/object/acl/list/aclrecordbuilder_test.go b/commonspace/object/acl/list/aclrecordbuilder_test.go index 5afdb2a8..0a5d3a77 100644 --- a/commonspace/object/acl/list/aclrecordbuilder_test.go +++ b/commonspace/object/acl/list/aclrecordbuilder_test.go @@ -3,7 +3,7 @@ package list import ( "github.com/anytypeio/any-sync/commonspace/object/accountdata" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" - acllistbuilder2 "github.com/anytypeio/any-sync/commonspace/object/acl/testutils/acllistbuilder" + acllistbuilder "github.com/anytypeio/any-sync/commonspace/object/acl/testutils/acllistbuilder" "github.com/anytypeio/any-sync/commonspace/object/keychain" "github.com/anytypeio/any-sync/util/cidutil" "github.com/anytypeio/any-sync/util/keys/asymmetric/signingkey" @@ -12,10 +12,10 @@ import ( ) func TestAclRecordBuilder_BuildUserJoin(t *testing.T) { - st, err := acllistbuilder2.NewListStorageWithTestName("userjoinexample.yml") + st, err := acllistbuilder.NewListStorageWithTestName("userjoinexample.yml") require.NoError(t, err, "building storage should not result in error") - testKeychain := st.(*acllistbuilder2.AclListStorageBuilder).GetKeychain() + testKeychain := st.(*acllistbuilder.AclListStorageBuilder).GetKeychain() identity := testKeychain.GeneratedIdentities["D"] signPrivKey := testKeychain.SigningKeysByYAMLName["D"] encPrivKey := testKeychain.EncryptionKeysByYAMLName["D"] @@ -28,7 +28,7 @@ func TestAclRecordBuilder_BuildUserJoin(t *testing.T) { aclList, err := BuildAclListWithIdentity(acc, st) require.NoError(t, err, "building acl list should be without error") recordBuilder := newAclRecordBuilder(aclList.Id(), keychain.NewKeychain()) - rk, err := testKeychain.GetKey("key.Read.EncKey").(*acllistbuilder2.SymKey).Key.Raw() + rk, err := testKeychain.GetKey("key.Read.EncKey").(*acllistbuilder.SymKey).Key.Raw() require.NoError(t, err) privKey, err := testKeychain.GetKey("key.Sign.Onetime1").(signingkey.PrivKey).Raw() require.NoError(t, err) diff --git a/commonspace/object/tree/objecttree/objecttree_test.go b/commonspace/object/tree/objecttree/objecttree_test.go index ee8989e9..26a8995b 100644 --- a/commonspace/object/tree/objecttree/objecttree_test.go +++ b/commonspace/object/tree/objecttree/objecttree_test.go @@ -11,59 +11,11 @@ import ( "testing" ) -type mockChangeCreator struct{} - -func (c *mockChangeCreator) createRoot(id, aclId string) *treechangeproto.RawTreeChangeWithId { - aclChange := &treechangeproto.RootChange{ - AclHeadId: aclId, - } - res, _ := aclChange.Marshal() - - raw := &treechangeproto.RawTreeChange{ - Payload: res, - Signature: nil, - } - rawMarshalled, _ := raw.Marshal() - - return &treechangeproto.RawTreeChangeWithId{ - RawChange: rawMarshalled, - Id: id, - } -} - -func (c *mockChangeCreator) createRaw(id, aclId, snapshotId string, isSnapshot bool, prevIds ...string) *treechangeproto.RawTreeChangeWithId { - aclChange := &treechangeproto.TreeChange{ - TreeHeadIds: prevIds, - AclHeadId: aclId, - SnapshotBaseId: snapshotId, - ChangesData: nil, - IsSnapshot: isSnapshot, - } - res, _ := aclChange.Marshal() - - raw := &treechangeproto.RawTreeChange{ - Payload: res, - Signature: nil, - } - rawMarshalled, _ := raw.Marshal() - - return &treechangeproto.RawTreeChangeWithId{ - RawChange: rawMarshalled, - Id: id, - } -} - -func (c *mockChangeCreator) createNewTreeStorage(treeId, aclHeadId string) treestorage.TreeStorage { - root := c.createRoot(treeId, aclHeadId) - treeStorage, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root}) - return treeStorage -} - type testTreeContext struct { aclList list.AclList treeStorage treestorage.TreeStorage changeBuilder ChangeBuilder - changeCreator *mockChangeCreator + changeCreator *MockChangeCreator objTree ObjectTree } @@ -77,9 +29,9 @@ func prepareAclList(t *testing.T) list.AclList { return aclList } -func prepareTreeDeps(aclList list.AclList) (*mockChangeCreator, objectTreeDeps) { - changeCreator := &mockChangeCreator{} - treeStorage := changeCreator.createNewTreeStorage("0", aclList.Head().Id) +func prepareTreeDeps(aclList list.AclList) (*MockChangeCreator, objectTreeDeps) { + changeCreator := &MockChangeCreator{} + treeStorage := changeCreator.CreateNewTreeStorage("0", aclList.Head().Id) root, _ := treeStorage.Root() changeBuilder := &nonVerifiableChangeBuilder{ ChangeBuilder: NewChangeBuilder(nil, root), @@ -96,8 +48,8 @@ func prepareTreeDeps(aclList list.AclList) (*mockChangeCreator, objectTreeDeps) } func prepareTreeContext(t *testing.T, aclList list.AclList) testTreeContext { - changeCreator := &mockChangeCreator{} - treeStorage := changeCreator.createNewTreeStorage("0", aclList.Head().Id) + changeCreator := &MockChangeCreator{} + treeStorage := changeCreator.CreateNewTreeStorage("0", aclList.Head().Id) root, _ := treeStorage.Root() changeBuilder := &nonVerifiableChangeBuilder{ ChangeBuilder: NewChangeBuilder(nil, root), @@ -142,8 +94,8 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -187,7 +139,7 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("0", aclList.Head().Id, "", true, ""), + changeCreator.CreateRaw("0", aclList.Head().Id, "", true, ""), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -211,7 +163,7 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -237,10 +189,10 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "3", false, "3"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "3", false, "3"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -287,9 +239,9 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -311,12 +263,12 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } payload := RawChangesPayload{ @@ -390,13 +342,13 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), // main difference from tree example - changeCreator.createRaw("6", aclList.Head().Id, "0", true, "3", "4", "5"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", true, "3", "4", "5"), } payload := RawChangesPayload{ @@ -471,9 +423,9 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -485,9 +437,9 @@ func TestObjectTree(t *testing.T) { require.Equal(t, "3", objTree.Root().Id) rawChanges = []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } payload = RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -531,12 +483,12 @@ func TestObjectTree(t *testing.T) { changeCreator, deps := prepareTreeDeps(aclList) rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } deps.treeStorage.TransactionAdd(rawChanges, []string{"6"}) hTree, err := buildHistoryTree(deps, HistoryTreeParams{ @@ -562,12 +514,12 @@ func TestObjectTree(t *testing.T) { changeCreator, deps := prepareTreeDeps(aclList) rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } deps.treeStorage.TransactionAdd(rawChanges, []string{"6"}) hTree, err := buildHistoryTree(deps, HistoryTreeParams{ diff --git a/commonspace/object/tree/objecttree/testutils.go b/commonspace/object/tree/objecttree/testutils.go new file mode 100644 index 00000000..61cd0fe7 --- /dev/null +++ b/commonspace/object/tree/objecttree/testutils.go @@ -0,0 +1,76 @@ +package objecttree + +import ( + "github.com/anytypeio/any-sync/commonspace/object/acl/list" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" +) + +type MockChangeCreator struct{} + +func NewMockChangeCreator() *MockChangeCreator { + return &MockChangeCreator{} +} + +func (c *MockChangeCreator) CreateRoot(id, aclId string) *treechangeproto.RawTreeChangeWithId { + aclChange := &treechangeproto.RootChange{ + AclHeadId: aclId, + } + res, _ := aclChange.Marshal() + + raw := &treechangeproto.RawTreeChange{ + Payload: res, + Signature: nil, + } + rawMarshalled, _ := raw.Marshal() + + return &treechangeproto.RawTreeChangeWithId{ + RawChange: rawMarshalled, + Id: id, + } +} + +func (c *MockChangeCreator) CreateRaw(id, aclId, snapshotId string, isSnapshot bool, prevIds ...string) *treechangeproto.RawTreeChangeWithId { + aclChange := &treechangeproto.TreeChange{ + TreeHeadIds: prevIds, + AclHeadId: aclId, + SnapshotBaseId: snapshotId, + ChangesData: nil, + IsSnapshot: isSnapshot, + } + res, _ := aclChange.Marshal() + + raw := &treechangeproto.RawTreeChange{ + Payload: res, + Signature: nil, + } + rawMarshalled, _ := raw.Marshal() + + return &treechangeproto.RawTreeChangeWithId{ + RawChange: rawMarshalled, + Id: id, + } +} + +func (c *MockChangeCreator) CreateNewTreeStorage(treeId, aclHeadId string) treestorage.TreeStorage { + root := c.CreateRoot(treeId, aclHeadId) + treeStorage, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root}) + return treeStorage +} + +func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { + root, _ := treeStorage.Root() + changeBuilder := &nonVerifiableChangeBuilder{ + ChangeBuilder: NewChangeBuilder(nil, root), + } + deps := objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: newTreeBuilder(treeStorage, changeBuilder), + treeStorage: treeStorage, + rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), + validator: &noOpTreeValidator{}, + aclList: aclList, + } + + return buildObjectTree(deps) +} diff --git a/commonspace/object/tree/synctree/syncclient.go b/commonspace/object/tree/synctree/syncclient.go index 6ad98c7c..f6e6b990 100644 --- a/commonspace/object/tree/synctree/syncclient.go +++ b/commonspace/object/tree/synctree/syncclient.go @@ -4,9 +4,8 @@ package synctree import ( "context" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/objectsync" + "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" - "github.com/anytypeio/any-sync/nodeconf" ) type SyncClient interface { @@ -16,21 +15,18 @@ type SyncClient interface { } type syncClient struct { - objectsync.MessagePool + peermanager.PeerManager RequestFactory - spaceId string - configuration nodeconf.Configuration + spaceId string } func newSyncClient( spaceId string, - pool objectsync.MessagePool, - factory RequestFactory, - configuration nodeconf.Configuration) SyncClient { + peerManager peermanager.PeerManager, + factory RequestFactory) SyncClient { return &syncClient{ - MessagePool: pool, + PeerManager: peerManager, RequestFactory: factory, - configuration: configuration, spaceId: spaceId, } } @@ -40,7 +36,7 @@ func (s *syncClient) Broadcast(ctx context.Context, msg *treechangeproto.TreeSyn if err != nil { return } - return s.MessagePool.Broadcast(ctx, objMsg) + return s.PeerManager.Broadcast(ctx, objMsg) } func (s *syncClient) SendWithReply(ctx context.Context, peerId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) { @@ -48,7 +44,7 @@ func (s *syncClient) SendWithReply(ctx context.Context, peerId string, msg *tree if err != nil { return } - return s.MessagePool.SendPeer(ctx, peerId, objMsg) + return s.PeerManager.SendPeer(ctx, peerId, objMsg) } func marshallTreeMessage(message *treechangeproto.TreeSyncMessage, spaceId, objectId, replyId string) (objMsg *spacesyncproto.ObjectSyncMessage, err error) { diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go new file mode 100644 index 00000000..cc7f7e18 --- /dev/null +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -0,0 +1,54 @@ +package synctree + +import ( + "context" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" + "github.com/anytypeio/any-sync/commonspace/peermanager" + "github.com/anytypeio/any-sync/commonspace/spacesyncproto" + "github.com/anytypeio/any-sync/commonspace/syncstatus" + "github.com/anytypeio/any-sync/net/peer" +) + +type mockPeerManager struct { +} + +func (m *mockPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { + //TODO implement me + panic("implement me") +} + +func (m *mockPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { + //TODO implement me + panic("implement me") +} + +func (m *mockPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { + //TODO implement me + panic("implement me") +} + +type broadcastTree struct { + objecttree.ObjectTree + SyncClient +} + +func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) (objecttree.AddResult, error) { + res, err := b.ObjectTree.AddRawChanges(ctx, changes) + if err != nil { + return objecttree.AddResult{}, err + } + upd := b.SyncClient.CreateHeadUpdate(b.ObjectTree, res.Added) + b.SyncClient.Broadcast(ctx, upd) + return res, nil +} + +func build(spaceId string, objTree objecttree.ObjectTree, peerManager peermanager.PeerManager) synchandler.SyncHandler { + factory := GetRequestFactory() + syncClient := newSyncClient(spaceId, peerManager, factory) + netTree := &broadcastTree{ + ObjectTree: objTree, + SyncClient: syncClient, + } + return newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) +} diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index 56d07c1c..a1374b48 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -101,8 +101,7 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t Sy syncClient := createSyncClient( deps.SpaceId, deps.ObjectSync.MessagePool(), - sharedFactory, - deps.Configuration) + sharedFactory) syncTree := &syncTree{ ObjectTree: objTree, syncClient: syncClient, From 5f5aefef53627b1db810efa6eb41dac40d14ec04 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 13 Mar 2023 22:03:38 +0100 Subject: [PATCH 02/45] Add test component structure --- .../object/tree/synctree/syncclient_test.go | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index cc7f7e18..14e8cacc 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -2,29 +2,31 @@ package synctree import ( "context" + "github.com/anytypeio/any-sync/commonspace/object/acl/list" + "github.com/anytypeio/any-sync/commonspace/object/acl/testutils/acllistbuilder" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/anytypeio/any-sync/net/peer" + "github.com/stretchr/testify/require" + "testing" ) type mockPeerManager struct { } func (m *mockPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { - //TODO implement me panic("implement me") } func (m *mockPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { - //TODO implement me panic("implement me") } func (m *mockPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { - //TODO implement me panic("implement me") } @@ -43,7 +45,7 @@ func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.Ra return res, nil } -func build(spaceId string, objTree objecttree.ObjectTree, peerManager peermanager.PeerManager) synchandler.SyncHandler { +func createSyncHandler(spaceId string, objTree objecttree.ObjectTree, peerManager peermanager.PeerManager) synchandler.SyncHandler { factory := GetRequestFactory() syncClient := newSyncClient(spaceId, peerManager, factory) netTree := &broadcastTree{ @@ -52,3 +54,33 @@ func build(spaceId string, objTree objecttree.ObjectTree, peerManager peermanage } return newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) } + +func createAclList() (list.AclList, error) { + st, err := acllistbuilder.NewListStorageWithTestName("userjoinexample.yml") + if err != nil { + return nil, err + } + return list.BuildAclList(st) +} + +func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage { + changeCreator := objecttree.NewMockChangeCreator() + st := changeCreator.CreateNewTreeStorage(treeId, aclList.Head().Id) + return st +} + +func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (objecttree.ObjectTree, error) { + return objecttree.BuildTestableTree(aclList, storage) +} + +func TestSyncProtocol(t *testing.T) { + aclList, err := createAclList() + require.NoError(t, err) + treeId := "treeId" + spaceId := "spaceId" + storage := createStorage(treeId, aclList) + testTree, err := createTestTree(aclList, storage) + require.NoError(t, err) + peerManager := &mockPeerManager{} + _ = createSyncHandler(spaceId, testTree, peerManager) +} From 829159690fc4c91c30c5c405afdf66f4fa1c45c2 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 14 Mar 2023 10:33:59 +0100 Subject: [PATCH 03/45] Add more components for async communication --- .../object/tree/synctree/syncclient_test.go | 136 +++++++++++++++--- 1 file changed, 113 insertions(+), 23 deletions(-) diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index 14e8cacc..d7617987 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -2,32 +2,113 @@ package synctree import ( "context" + "fmt" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/acl/testutils/acllistbuilder" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" - "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/anytypeio/any-sync/net/peer" - "github.com/stretchr/testify/require" - "testing" + "github.com/cheggaaa/mb/v3" + "sync" ) -type mockPeerManager struct { +type processMsg struct { + msg *spacesyncproto.ObjectSyncMessage + senderId string + receiverId string } -func (m *mockPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { - panic("implement me") +type messageLog struct { + batcher *mb.MB[processMsg] } -func (m *mockPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { - panic("implement me") +func newMessageLog() *messageLog { + return &messageLog{batcher: mb.New[processMsg](0)} } -func (m *mockPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { - panic("implement me") +func (m *messageLog) addMessage(msg processMsg) { + m.batcher.Add(context.Background(), msg) +} + +type processSyncHandler struct { + synchandler.SyncHandler + batcher *mb.MB[processMsg] + peerId string +} + +func newProcessSyncHandler(peerId string, syncHandler synchandler.SyncHandler) *processSyncHandler { + batcher := mb.New[processMsg](0) + return &processSyncHandler{syncHandler, batcher, peerId} +} + +func (p *processSyncHandler) manager() *processPeerManager { + return p.SyncHandler.(*syncTreeHandler).syncClient.(*syncClient).PeerManager.(*processPeerManager) +} + +func (p *processSyncHandler) send(ctx context.Context, msg processMsg) (err error) { + return p.batcher.Add(ctx, msg) +} + +func (p *processSyncHandler) run(ctx context.Context, wg *sync.WaitGroup) { + wg.Add(1) + go func() { + defer wg.Done() + for { + res, err := p.batcher.WaitOne(ctx) + if err != nil { + return + } + err = p.SyncHandler.HandleMessage(ctx, res.senderId, res.msg) + if err != nil { + fmt.Println("error handling message", err.Error()) + continue + } + } + }() +} + +type processPeerManager struct { + peerId string + handlers map[string]*processSyncHandler + log *messageLog +} + +func newProcessPeerManager(peerId string, log *messageLog) *processPeerManager { + return &processPeerManager{handlers: map[string]*processSyncHandler{}, peerId: peerId, log: log} +} + +func (m *processPeerManager) addHandler(peerId string, handler *processSyncHandler) { + m.handlers[peerId] = handler +} + +func (m *processPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { + pMsg := processMsg{ + msg: msg, + senderId: m.peerId, + receiverId: peerId, + } + m.log.addMessage(pMsg) + return m.handlers[peerId].send(context.Background(), pMsg) +} + +func (m *processPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { + for _, handler := range m.handlers { + pMsg := processMsg{ + msg: msg, + senderId: m.peerId, + receiverId: handler.peerId, + } + m.log.addMessage(pMsg) + handler.send(context.Background(), pMsg) + } + return +} + +func (m *processPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { + panic("should not be called") } type broadcastTree struct { @@ -45,14 +126,15 @@ func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.Ra return res, nil } -func createSyncHandler(spaceId string, objTree objecttree.ObjectTree, peerManager peermanager.PeerManager) synchandler.SyncHandler { +func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *processSyncHandler { factory := GetRequestFactory() - syncClient := newSyncClient(spaceId, peerManager, factory) + syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) netTree := &broadcastTree{ ObjectTree: objTree, SyncClient: syncClient, } - return newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) + handler := newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) + return newProcessSyncHandler(peerId, handler) } func createAclList() (list.AclList, error) { @@ -73,14 +155,22 @@ func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (obje return objecttree.BuildTestableTree(aclList, storage) } -func TestSyncProtocol(t *testing.T) { - aclList, err := createAclList() - require.NoError(t, err) - treeId := "treeId" - spaceId := "spaceId" - storage := createStorage(treeId, aclList) - testTree, err := createTestTree(aclList, storage) - require.NoError(t, err) - peerManager := &mockPeerManager{} - _ = createSyncHandler(spaceId, testTree, peerManager) +type processFixture struct { + handlers []*processSyncHandler + log *messageLog + wg *sync.WaitGroup + ctx context.Context + cancel context.CancelFunc } + +//func TestSyncProtocol(t *testing.T) { +// aclList, err := createAclList() +// require.NoError(t, err) +// treeId := "treeId" +// spaceId := "spaceId" +// storage := createStorage(treeId, aclList) +// testTree, err := createTestTree(aclList, storage) +// require.NoError(t, err) +// peerManager := &processPeerManager{} +// _ = createSyncHandler(spaceId, testTree, peerManager) +//} From 0746526170d491ac8c05963fea05e69c26dc49e5 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 14 Mar 2023 11:41:29 +0100 Subject: [PATCH 04/45] Create fixture draft --- .../object/tree/synctree/syncclient_test.go | 71 ++++++++++++++++++- .../object/tree/treestorage/inmemory.go | 33 +++++---- 2 files changed, 90 insertions(+), 14 deletions(-) diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index d7617987..7a76b71f 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -12,13 +12,16 @@ import ( "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/anytypeio/any-sync/net/peer" "github.com/cheggaaa/mb/v3" + "github.com/stretchr/testify/require" "sync" + "testing" ) type processMsg struct { msg *spacesyncproto.ObjectSyncMessage senderId string receiverId string + userMsg *objecttree.RawChangesPayload } type messageLog struct { @@ -48,11 +51,19 @@ func (p *processSyncHandler) manager() *processPeerManager { return p.SyncHandler.(*syncTreeHandler).syncClient.(*syncClient).PeerManager.(*processPeerManager) } +func (p *processSyncHandler) tree() *broadcastTree { + return p.SyncHandler.(*syncTreeHandler).objTree.(*broadcastTree) +} + func (p *processSyncHandler) send(ctx context.Context, msg processMsg) (err error) { return p.batcher.Add(ctx, msg) } -func (p *processSyncHandler) run(ctx context.Context, wg *sync.WaitGroup) { +func (p *processSyncHandler) sendRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) { + p.batcher.Add(ctx, processMsg{userMsg: &changes}) +} + +func (p *processSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.WaitGroup) { wg.Add(1) go func() { defer wg.Done() @@ -61,6 +72,14 @@ func (p *processSyncHandler) run(ctx context.Context, wg *sync.WaitGroup) { if err != nil { return } + if res.userMsg != nil { + p.tree().Lock() + userRes, err := p.tree().AddRawChanges(ctx, *res.userMsg) + require.NoError(t, err) + fmt.Println("user add result", userRes.Heads) + p.tree().Unlock() + continue + } err = p.SyncHandler.HandleMessage(ctx, res.senderId, res.msg) if err != nil { fmt.Println("error handling message", err.Error()) @@ -155,14 +174,62 @@ func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (obje return objecttree.BuildTestableTree(aclList, storage) } +type fixtureDeps struct { + aclList list.AclList + initStorage *treestorage.InMemoryTreeStorage + connectionMap map[string][]string +} + type processFixture struct { - handlers []*processSyncHandler + handlers map[string]*processSyncHandler log *messageLog wg *sync.WaitGroup ctx context.Context cancel context.CancelFunc } +func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processFixture { + var ( + handlers = map[string]*processSyncHandler{} + log = newMessageLog() + wg = sync.WaitGroup{} + ctx, cancel = context.WithCancel(context.Background()) + ) + + for peerId := range deps.connectionMap { + stCopy := deps.initStorage.Copy() + testTree, err := createTestTree(deps.aclList, stCopy) + require.NoError(t, err) + handler := createSyncHandler(peerId, spaceId, testTree, log) + handlers[peerId] = handler + } + for peerId, connectionMap := range deps.connectionMap { + handler := handlers[peerId] + manager := handler.manager() + for _, connectionId := range connectionMap { + manager.addHandler(connectionId, handlers[connectionId]) + } + } + return &processFixture{ + handlers: handlers, + log: log, + wg: &wg, + ctx: ctx, + cancel: cancel, + } +} + +func (p *processFixture) runProcessFixture(t *testing.T) { + for _, handler := range p.handlers { + handler.run(p.ctx, t, p.wg) + } +} + +func (p *processFixture) stop() { + p.cancel() + p.wg.Wait() +} + //func TestSyncProtocol(t *testing.T) { // aclList, err := createAclList() // require.NoError(t, err) diff --git a/commonspace/object/tree/treestorage/inmemory.go b/commonspace/object/tree/treestorage/inmemory.go index c5c5a5cc..0f22037e 100644 --- a/commonspace/object/tree/treestorage/inmemory.go +++ b/commonspace/object/tree/treestorage/inmemory.go @@ -7,7 +7,7 @@ import ( "sync" ) -type inMemoryTreeStorage struct { +type InMemoryTreeStorage struct { id string root *treechangeproto.RawTreeChangeWithId heads []string @@ -16,7 +16,7 @@ type inMemoryTreeStorage struct { sync.RWMutex } -func (t *inMemoryTreeStorage) TransactionAdd(changes []*treechangeproto.RawTreeChangeWithId, heads []string) error { +func (t *InMemoryTreeStorage) TransactionAdd(changes []*treechangeproto.RawTreeChangeWithId, heads []string) error { t.RLock() defer t.RUnlock() @@ -37,46 +37,46 @@ func NewInMemoryTreeStorage( } allChanges[root.Id] = root - return &inMemoryTreeStorage{ + return &InMemoryTreeStorage{ id: root.Id, root: root, - heads: heads, + heads: append([]string(nil), heads...), changes: allChanges, RWMutex: sync.RWMutex{}, }, nil } -func (t *inMemoryTreeStorage) HasChange(ctx context.Context, id string) (bool, error) { +func (t *InMemoryTreeStorage) HasChange(ctx context.Context, id string) (bool, error) { _, exists := t.changes[id] return exists, nil } -func (t *inMemoryTreeStorage) Id() string { +func (t *InMemoryTreeStorage) Id() string { t.RLock() defer t.RUnlock() return t.id } -func (t *inMemoryTreeStorage) Root() (*treechangeproto.RawTreeChangeWithId, error) { +func (t *InMemoryTreeStorage) Root() (*treechangeproto.RawTreeChangeWithId, error) { t.RLock() defer t.RUnlock() return t.root, nil } -func (t *inMemoryTreeStorage) Heads() ([]string, error) { +func (t *InMemoryTreeStorage) Heads() ([]string, error) { t.RLock() defer t.RUnlock() return t.heads, nil } -func (t *inMemoryTreeStorage) SetHeads(heads []string) error { +func (t *InMemoryTreeStorage) SetHeads(heads []string) error { t.Lock() defer t.Unlock() t.heads = append(t.heads[:0], heads...) return nil } -func (t *inMemoryTreeStorage) AddRawChange(change *treechangeproto.RawTreeChangeWithId) error { +func (t *InMemoryTreeStorage) AddRawChange(change *treechangeproto.RawTreeChangeWithId) error { t.Lock() defer t.Unlock() // TODO: better to do deep copy @@ -84,7 +84,7 @@ func (t *inMemoryTreeStorage) AddRawChange(change *treechangeproto.RawTreeChange return nil } -func (t *inMemoryTreeStorage) GetRawChange(ctx context.Context, changeId string) (*treechangeproto.RawTreeChangeWithId, error) { +func (t *InMemoryTreeStorage) GetRawChange(ctx context.Context, changeId string) (*treechangeproto.RawTreeChangeWithId, error) { t.RLock() defer t.RUnlock() if res, exists := t.changes[changeId]; exists { @@ -93,6 +93,15 @@ func (t *inMemoryTreeStorage) GetRawChange(ctx context.Context, changeId string) return nil, fmt.Errorf("could not get change with id: %s", changeId) } -func (t *inMemoryTreeStorage) Delete() error { +func (t *InMemoryTreeStorage) Delete() error { return nil } + +func (t *InMemoryTreeStorage) Copy() *InMemoryTreeStorage { + var changes []*treechangeproto.RawTreeChangeWithId + for _, ch := range t.changes { + changes = append(changes, ch) + } + other, _ := NewInMemoryTreeStorage(t.root, t.heads, changes) + return other.(*InMemoryTreeStorage) +} From afad73fe8bb08e5d8b1cc15d17d49dce88b88c9c Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 20 Mar 2023 16:00:44 +0100 Subject: [PATCH 05/45] Add simple test --- .../object/tree/synctree/syncclient_test.go | 54 ++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index 7a76b71f..c1f4fccb 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -6,6 +6,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/acl/testutils/acllistbuilder" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" @@ -13,8 +14,10 @@ import ( "github.com/anytypeio/any-sync/net/peer" "github.com/cheggaaa/mb/v3" "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" "sync" "testing" + "time" ) type processMsg struct { @@ -219,7 +222,7 @@ func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processF } } -func (p *processFixture) runProcessFixture(t *testing.T) { +func (p *processFixture) run(t *testing.T) { for _, handler := range p.handlers { handler.run(p.ctx, t, p.wg) } @@ -230,14 +233,41 @@ func (p *processFixture) stop() { p.wg.Wait() } -//func TestSyncProtocol(t *testing.T) { -// aclList, err := createAclList() -// require.NoError(t, err) -// treeId := "treeId" -// spaceId := "spaceId" -// storage := createStorage(treeId, aclList) -// testTree, err := createTestTree(aclList, storage) -// require.NoError(t, err) -// peerManager := &processPeerManager{} -// _ = createSyncHandler(spaceId, testTree, peerManager) -//} +func TestSimple(t *testing.T) { + aclList, err := createAclList() + require.NoError(t, err) + treeId := "treeId" + spaceId := "spaceId" + storage := createStorage(treeId, aclList) + changeCreator := objecttree.NewMockChangeCreator() + deps := fixtureDeps{ + aclList: aclList, + initStorage: storage.(*treestorage.InMemoryTreeStorage), + connectionMap: map[string][]string{ + "peer1": []string{"peer2"}, + "peer2": []string{"peer1"}, + }, + } + fx := newProcessFixture(t, spaceId, deps) + fx.run(t) + fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: nil, + RawChanges: []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("1", aclList.Id(), treeId, false, treeId), + }, + }) + fx.handlers["peer2"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: nil, + RawChanges: []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("2", aclList.Id(), treeId, false, treeId), + }, + }) + time.Sleep(100 * time.Millisecond) + fx.stop() + firstHeads := fx.handlers["peer1"].tree().Heads() + secondHeads := fx.handlers["peer2"].tree().Heads() + slices.Sort(firstHeads) + slices.Sort(secondHeads) + require.Equal(t, firstHeads, secondHeads) + require.Equal(t, []string{"1", "2"}, firstHeads) +} From 0a1dc26a9506f2d8ea1218615b2bc920fea62f34 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 20 Mar 2023 16:24:07 +0100 Subject: [PATCH 06/45] Add second test and description --- .../object/tree/synctree/syncclient_test.go | 85 ++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index c1f4fccb..06c96c25 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -13,6 +13,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/anytypeio/any-sync/net/peer" "github.com/cheggaaa/mb/v3" + "github.com/gogo/protobuf/proto" "github.com/stretchr/testify/require" "golang.org/x/exp/slices" "sync" @@ -27,6 +28,40 @@ type processMsg struct { userMsg *objecttree.RawChangesPayload } +type msgDescription struct { + name string + from string + to string + heads []string +} + +func (p *processMsg) description() (descr msgDescription) { + unmarshalled := &treechangeproto.TreeSyncMessage{} + err := proto.Unmarshal(p.msg.Payload, unmarshalled) + if err != nil { + panic(err) + } + descr = msgDescription{ + from: p.senderId, + to: p.receiverId, + } + switch { + case unmarshalled.GetContent().GetHeadUpdate() != nil: + cnt := unmarshalled.GetContent().GetHeadUpdate() + descr.name = "HeadUpdate" + descr.heads = cnt.Heads + case unmarshalled.GetContent().GetFullSyncRequest() != nil: + cnt := unmarshalled.GetContent().GetFullSyncRequest() + descr.name = "FullSyncRequest" + descr.heads = cnt.Heads + case unmarshalled.GetContent().GetFullSyncResponse() != nil: + cnt := unmarshalled.GetContent().GetFullSyncResponse() + descr.name = "FullSyncResponse" + descr.heads = cnt.Heads + } + return +} + type messageLog struct { batcher *mb.MB[processMsg] } @@ -233,7 +268,7 @@ func (p *processFixture) stop() { p.wg.Wait() } -func TestSimple(t *testing.T) { +func TestSimple_TwoPeers(t *testing.T) { aclList, err := createAclList() require.NoError(t, err) treeId := "treeId" @@ -270,4 +305,52 @@ func TestSimple(t *testing.T) { slices.Sort(secondHeads) require.Equal(t, firstHeads, secondHeads) require.Equal(t, []string{"1", "2"}, firstHeads) + logMsgs := fx.log.batcher.GetAll() + for _, msg := range logMsgs { + fmt.Println(msg.description()) + } +} + +func TestSimple_ThreePeers(t *testing.T) { + aclList, err := createAclList() + require.NoError(t, err) + treeId := "treeId" + spaceId := "spaceId" + storage := createStorage(treeId, aclList) + changeCreator := objecttree.NewMockChangeCreator() + deps := fixtureDeps{ + aclList: aclList, + initStorage: storage.(*treestorage.InMemoryTreeStorage), + connectionMap: map[string][]string{ + "peer1": []string{"node1"}, + "peer2": []string{"node1"}, + "node1": []string{"peer1", "peer2"}, + }, + } + fx := newProcessFixture(t, spaceId, deps) + fx.run(t) + fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: nil, + RawChanges: []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("1", aclList.Id(), treeId, false, treeId), + }, + }) + fx.handlers["peer2"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: nil, + RawChanges: []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("2", aclList.Id(), treeId, false, treeId), + }, + }) + time.Sleep(100 * time.Millisecond) + fx.stop() + firstHeads := fx.handlers["peer1"].tree().Heads() + secondHeads := fx.handlers["peer2"].tree().Heads() + slices.Sort(firstHeads) + slices.Sort(secondHeads) + require.Equal(t, firstHeads, secondHeads) + require.Equal(t, []string{"1", "2"}, firstHeads) + logMsgs := fx.log.batcher.GetAll() + for _, msg := range logMsgs { + fmt.Println(msg.description()) + } } From e101389af072038743936e96afebd35eec4f1ba1 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 13 Apr 2023 21:32:39 +0200 Subject: [PATCH 07/45] Add empty client logic --- .../object/tree/synctree/syncclient_test.go | 120 ++++++++++++++++-- 1 file changed, 111 insertions(+), 9 deletions(-) diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index 7feb15d5..086cb883 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -76,17 +76,59 @@ func (m *messageLog) addMessage(msg processMsg) { type processSyncHandler struct { synchandler.SyncHandler - batcher *mb.MB[processMsg] - peerId string + batcher *mb.MB[processMsg] + peerId string + aclList list.AclList + log *messageLog + syncClient SyncClient +} + +func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { + if p.SyncHandler != nil { + return p.SyncHandler.HandleMessage(ctx, senderId, request) + } + unmarshalled := &treechangeproto.TreeSyncMessage{} + err = proto.Unmarshal(request.Payload, unmarshalled) + if err != nil { + return + } + if unmarshalled.Content.GetFullSyncResponse() == nil { + newTreeRequest := GetRequestFactory().CreateNewTreeRequest() + var objMsg *spacesyncproto.ObjectSyncMessage + objMsg, err = marshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") + if err != nil { + return + } + return p.manager().SendPeer(context.Background(), senderId, objMsg) + } + fullSyncResponse := unmarshalled.Content.GetFullSyncResponse() + treeStorage, _ := treestorage.NewInMemoryTreeStorage(unmarshalled.RootChange, fullSyncResponse.Heads, fullSyncResponse.Changes) + tree, err := createTestTree(p.aclList, treeStorage) + if err != nil { + return + } + netTree := &broadcastTree{ + ObjectTree: tree, + SyncClient: p.syncClient, + } + p.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, p.syncClient, syncstatus.NewNoOpSyncStatus()) + return } func newProcessSyncHandler(peerId string, syncHandler synchandler.SyncHandler) *processSyncHandler { batcher := mb.New[processMsg](0) - return &processSyncHandler{syncHandler, batcher, peerId} + return &processSyncHandler{ + SyncHandler: syncHandler, + batcher: batcher, + peerId: peerId, + } } func (p *processSyncHandler) manager() *processPeerManager { - return p.SyncHandler.(*syncTreeHandler).syncClient.(*syncClient).PeerManager.(*processPeerManager) + if p.SyncHandler != nil { + return p.SyncHandler.(*syncTreeHandler).syncClient.(*syncClient).PeerManager.(*processPeerManager) + } + return p.syncClient.(*syncClient).PeerManager.(*processPeerManager) } func (p *processSyncHandler) tree() *broadcastTree { @@ -118,7 +160,7 @@ func (p *processSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.Wai p.tree().Unlock() continue } - err = p.SyncHandler.HandleMessage(ctx, res.senderId, res.msg) + err = p.HandleMessage(ctx, res.senderId, res.msg) if err != nil { fmt.Println("error handling message", err.Error()) continue @@ -194,6 +236,20 @@ func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, lo return newProcessSyncHandler(peerId, handler) } +func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *processSyncHandler { + factory := GetRequestFactory() + syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) + + batcher := mb.New[processMsg](0) + return &processSyncHandler{ + batcher: batcher, + peerId: peerId, + aclList: aclList, + log: log, + syncClient: syncClient, + } +} + func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage { changeCreator := objecttree.NewMockChangeCreator() st := changeCreator.CreateNewTreeStorage(treeId, aclList.Head().Id) @@ -208,6 +264,7 @@ type fixtureDeps struct { aclList list.AclList initStorage *treestorage.InMemoryTreeStorage connectionMap map[string][]string + emptyTrees []string } type processFixture struct { @@ -227,10 +284,15 @@ func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processF ) for peerId := range deps.connectionMap { - stCopy := deps.initStorage.Copy() - testTree, err := createTestTree(deps.aclList, stCopy) - require.NoError(t, err) - handler := createSyncHandler(peerId, spaceId, testTree, log) + var handler *processSyncHandler + if slices.Contains(deps.emptyTrees, peerId) { + handler = createEmptySyncHandler(peerId, spaceId, deps.aclList, log) + } else { + stCopy := deps.initStorage.Copy() + testTree, err := createTestTree(deps.aclList, stCopy) + require.NoError(t, err) + handler = createSyncHandler(peerId, spaceId, testTree, log) + } handlers[peerId] = handler } for peerId, connectionMap := range deps.connectionMap { @@ -260,6 +322,46 @@ func (p *processFixture) stop() { p.wg.Wait() } +func TestSend_EmptyClient(t *testing.T) { + treeId := "treeId" + spaceId := "spaceId" + keys, err := accountdata.NewRandom() + require.NoError(t, err) + aclList, err := list.NewTestDerivedAcl(spaceId, keys) + require.NoError(t, err) + storage := createStorage(treeId, aclList) + changeCreator := objecttree.NewMockChangeCreator() + deps := fixtureDeps{ + aclList: aclList, + initStorage: storage.(*treestorage.InMemoryTreeStorage), + connectionMap: map[string][]string{ + "peer1": []string{"peer2"}, + "peer2": []string{"peer1"}, + }, + emptyTrees: []string{"peer2"}, + } + fx := newProcessFixture(t, spaceId, deps) + fx.run(t) + fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: nil, + RawChanges: []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("1", aclList.Id(), treeId, false, treeId), + }, + }) + time.Sleep(100 * time.Millisecond) + fx.stop() + firstHeads := fx.handlers["peer1"].tree().Heads() + secondHeads := fx.handlers["peer2"].tree().Heads() + slices.Sort(firstHeads) + slices.Sort(secondHeads) + require.Equal(t, firstHeads, secondHeads) + require.Equal(t, []string{"1"}, firstHeads) + logMsgs := fx.log.batcher.GetAll() + for _, msg := range logMsgs { + fmt.Println(msg.description()) + } +} + func TestSimple_TwoPeers(t *testing.T) { treeId := "treeId" spaceId := "spaceId" From 673925df84b70c722b8f11dbaf8237c7230b19e5 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 13 Apr 2023 23:55:28 +0200 Subject: [PATCH 08/45] Improve validator logic and add tests --- .../object/tree/objecttree/objecttree_test.go | 224 +++++++++--------- .../tree/objecttree/objecttreefactory.go | 6 +- .../tree/objecttree/objecttreevalidator.go | 20 +- .../object/tree/objecttree/testutils.go | 56 ++++- .../object/tree/synctree/syncclient_test.go | 26 +- 5 files changed, 199 insertions(+), 133 deletions(-) diff --git a/commonspace/object/tree/objecttree/objecttree_test.go b/commonspace/object/tree/objecttree/objecttree_test.go index 3ecd33ff..7ec9fac9 100644 --- a/commonspace/object/tree/objecttree/objecttree_test.go +++ b/commonspace/object/tree/objecttree/objecttree_test.go @@ -11,59 +11,10 @@ import ( "testing" ) -type mockChangeCreator struct{} - -func (c *mockChangeCreator) createRoot(id, aclId string) *treechangeproto.RawTreeChangeWithId { - aclChange := &treechangeproto.RootChange{ - AclHeadId: aclId, - } - res, _ := aclChange.Marshal() - - raw := &treechangeproto.RawTreeChange{ - Payload: res, - Signature: nil, - } - rawMarshalled, _ := raw.Marshal() - - return &treechangeproto.RawTreeChangeWithId{ - RawChange: rawMarshalled, - Id: id, - } -} - -func (c *mockChangeCreator) createRaw(id, aclId, snapshotId string, isSnapshot bool, prevIds ...string) *treechangeproto.RawTreeChangeWithId { - aclChange := &treechangeproto.TreeChange{ - TreeHeadIds: prevIds, - AclHeadId: aclId, - SnapshotBaseId: snapshotId, - ChangesData: nil, - IsSnapshot: isSnapshot, - } - res, _ := aclChange.Marshal() - - raw := &treechangeproto.RawTreeChange{ - Payload: res, - Signature: nil, - } - rawMarshalled, _ := raw.Marshal() - - return &treechangeproto.RawTreeChangeWithId{ - RawChange: rawMarshalled, - Id: id, - } -} - -func (c *mockChangeCreator) createNewTreeStorage(treeId, aclHeadId string) treestorage.TreeStorage { - root := c.createRoot(treeId, aclHeadId) - treeStorage, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root}) - return treeStorage -} - type testTreeContext struct { aclList list.AclList treeStorage treestorage.TreeStorage - changeBuilder ChangeBuilder - changeCreator *mockChangeCreator + changeCreator *MockChangeCreator objTree ObjectTree } @@ -76,12 +27,12 @@ func prepareAclList(t *testing.T) list.AclList { return aclList } -func prepareTreeDeps(aclList list.AclList) (*mockChangeCreator, objectTreeDeps) { - changeCreator := &mockChangeCreator{} - treeStorage := changeCreator.createNewTreeStorage("0", aclList.Head().Id) +func prepareTreeDeps(aclList list.AclList) (*MockChangeCreator, objectTreeDeps) { + changeCreator := NewMockChangeCreator() + treeStorage := changeCreator.CreateNewTreeStorage("0", aclList.Head().Id) root, _ := treeStorage.Root() changeBuilder := &nonVerifiableChangeBuilder{ - ChangeBuilder: NewChangeBuilder(newKeyStorage(), root), + ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root), } deps := objectTreeDeps{ changeBuilder: changeBuilder, @@ -95,23 +46,9 @@ func prepareTreeDeps(aclList list.AclList) (*mockChangeCreator, objectTreeDeps) } func prepareTreeContext(t *testing.T, aclList list.AclList) testTreeContext { - changeCreator := &mockChangeCreator{} - treeStorage := changeCreator.createNewTreeStorage("0", aclList.Head().Id) - root, _ := treeStorage.Root() - changeBuilder := &nonVerifiableChangeBuilder{ - ChangeBuilder: NewChangeBuilder(newKeyStorage(), root), - } - deps := objectTreeDeps{ - changeBuilder: changeBuilder, - treeBuilder: newTreeBuilder(treeStorage, changeBuilder), - treeStorage: treeStorage, - rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), - validator: &noOpTreeValidator{}, - aclList: aclList, - } - - // check build - objTree, err := buildObjectTree(deps) + changeCreator := NewMockChangeCreator() + treeStorage := changeCreator.CreateNewTreeStorage("0", aclList.Head().Id) + objTree, err := BuildTestableTree(aclList, treeStorage) require.NoError(t, err, "building tree should be without error") // check tree iterate @@ -125,7 +62,6 @@ func prepareTreeContext(t *testing.T, aclList list.AclList) testTreeContext { return testTreeContext{ aclList: aclList, treeStorage: treeStorage, - changeBuilder: changeBuilder, changeCreator: changeCreator, objTree: objTree, } @@ -141,8 +77,8 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -186,7 +122,7 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("0", aclList.Head().Id, "", true, ""), + changeCreator.CreateRaw("0", aclList.Head().Id, "", true, ""), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -210,7 +146,33 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + } + payload := RawChangesPayload{ + NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, + RawChanges: rawChanges, + } + res, err := objTree.AddRawChanges(context.Background(), payload) + require.NoError(t, err, "adding changes should be without error") + + // check result + assert.Equal(t, []string{"0"}, res.OldHeads) + assert.Equal(t, []string{"0"}, res.Heads) + assert.Equal(t, 0, len(res.Added)) + assert.Equal(t, Nothing, res.Mode) + + // check tree heads + assert.Equal(t, []string{"0"}, objTree.Heads()) + }) + + t.Run("add not connected changes", func(t *testing.T) { + ctx := prepareTreeContext(t, aclList) + changeCreator := ctx.changeCreator + objTree := ctx.objTree + + // this change could in theory replace current snapshot, we should prevent that + rawChanges := []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("2", aclList.Head().Id, "0", true, "1"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -236,10 +198,10 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "3", false, "3"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "3", false, "3"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -286,9 +248,9 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -310,12 +272,12 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } payload := RawChangesPayload{ @@ -389,13 +351,13 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), // main difference from tree example - changeCreator.createRaw("6", aclList.Head().Id, "0", true, "3", "4", "5"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", true, "3", "4", "5"), } payload := RawChangesPayload{ @@ -470,9 +432,9 @@ func TestObjectTree(t *testing.T) { objTree := ctx.objTree rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), } payload := RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -484,9 +446,9 @@ func TestObjectTree(t *testing.T) { require.Equal(t, "3", objTree.Root().Id) rawChanges = []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } payload = RawChangesPayload{ NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, @@ -530,12 +492,12 @@ func TestObjectTree(t *testing.T) { changeCreator, deps := prepareTreeDeps(aclList) rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } deps.treeStorage.TransactionAdd(rawChanges, []string{"6"}) hTree, err := buildHistoryTree(deps, HistoryTreeParams{ @@ -561,12 +523,12 @@ func TestObjectTree(t *testing.T) { changeCreator, deps := prepareTreeDeps(aclList) rawChanges := []*treechangeproto.RawTreeChangeWithId{ - changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"), - changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"), - changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"), - changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"), - changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "2"), + changeCreator.CreateRaw("5", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"), } deps.treeStorage.TransactionAdd(rawChanges, []string{"6"}) hTree, err := buildHistoryTree(deps, HistoryTreeParams{ @@ -608,4 +570,40 @@ func TestObjectTree(t *testing.T) { assert.Equal(t, []string{"0"}, iterChangesId) assert.Equal(t, "0", hTree.Root().Id) }) + + t.Run("validate correct tree", func(t *testing.T) { + ctx := prepareTreeContext(t, aclList) + changeCreator := ctx.changeCreator + + rawChanges := []*treechangeproto.RawTreeChangeWithId{ + ctx.objTree.Header(), + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + } + defaultObjectTreeDeps = nonVerifiableTreeDeps + err := ValidateRawTree(treestorage.TreeStorageCreatePayload{ + RootRawChange: ctx.objTree.Header(), + Heads: []string{"3"}, + Changes: rawChanges, + }, ctx.aclList) + require.NoError(t, err) + }) + + t.Run("fail to validate not connected tree", func(t *testing.T) { + ctx := prepareTreeContext(t, aclList) + changeCreator := ctx.changeCreator + + rawChanges := []*treechangeproto.RawTreeChangeWithId{ + ctx.objTree.Header(), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + } + defaultObjectTreeDeps = nonVerifiableTreeDeps + err := ValidateRawTree(treestorage.TreeStorageCreatePayload{ + RootRawChange: ctx.objTree.Header(), + Heads: []string{"3"}, + Changes: rawChanges, + }, ctx.aclList) + require.Equal(t, ErrHasInvalidChanges, err) + }) } diff --git a/commonspace/object/tree/objecttree/objecttreefactory.go b/commonspace/object/tree/objecttree/objecttreefactory.go index b3b757fb..0ab51ec1 100644 --- a/commonspace/object/tree/objecttree/objecttreefactory.go +++ b/commonspace/object/tree/objecttree/objecttreefactory.go @@ -33,7 +33,9 @@ type objectTreeDeps struct { aclList list.AclList } -func defaultObjectTreeDeps( +var defaultObjectTreeDeps = verifiableTreeDeps + +func verifiableTreeDeps( rootChange *treechangeproto.RawTreeChangeWithId, treeStorage treestorage.TreeStorage, aclList list.AclList) objectTreeDeps { @@ -53,7 +55,7 @@ func nonVerifiableTreeDeps( rootChange *treechangeproto.RawTreeChangeWithId, treeStorage treestorage.TreeStorage, aclList list.AclList) objectTreeDeps { - changeBuilder := &nonVerifiableChangeBuilder{NewChangeBuilder(nil, rootChange)} + changeBuilder := &nonVerifiableChangeBuilder{NewChangeBuilder(newMockKeyStorage(), rootChange)} treeBuilder := newTreeBuilder(treeStorage, changeBuilder) return objectTreeDeps{ changeBuilder: changeBuilder, diff --git a/commonspace/object/tree/objecttree/objecttreevalidator.go b/commonspace/object/tree/objecttree/objecttreevalidator.go index 937dcc96..e2816246 100644 --- a/commonspace/object/tree/objecttree/objecttreevalidator.go +++ b/commonspace/object/tree/objecttree/objecttreevalidator.go @@ -1,10 +1,12 @@ package objecttree import ( + "context" "fmt" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/util/slice" ) type ObjectTreeValidator interface { @@ -88,11 +90,23 @@ func (v *objectTreeValidator) validateChange(tree *Tree, aclList list.AclList, c } func ValidateRawTree(payload treestorage.TreeStorageCreatePayload, aclList list.AclList) (err error) { - treeStorage, err := treestorage.NewInMemoryTreeStorage(payload.RootRawChange, payload.Heads, payload.Changes) + treeStorage, err := treestorage.NewInMemoryTreeStorage(payload.RootRawChange, []string{payload.RootRawChange.Id}, nil) if err != nil { return } - - _, err = BuildObjectTree(treeStorage, aclList) + tree, err := BuildObjectTree(treeStorage, aclList) + if err != nil { + return + } + res, err := tree.AddRawChanges(context.Background(), RawChangesPayload{ + NewHeads: payload.Heads, + RawChanges: payload.Changes, + }) + if err != nil { + return + } + if !slice.UnsortedEquals(res.Heads, payload.Heads) { + return ErrHasInvalidChanges + } return } diff --git a/commonspace/object/tree/objecttree/testutils.go b/commonspace/object/tree/objecttree/testutils.go index 9fbeed5f..6385b9f7 100644 --- a/commonspace/object/tree/objecttree/testutils.go +++ b/commonspace/object/tree/objecttree/testutils.go @@ -1,24 +1,64 @@ package objecttree import ( - "crypto/rand" + "fmt" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/util/crypto" + libcrypto "github.com/libp2p/go-libp2p/core/crypto" ) -type mockKeyStorage struct { - key crypto.PubKey +type mockPubKey struct { } -func newKeyStorage() mockKeyStorage { - _, pk, _ := crypto.GenerateEd25519Key(rand.Reader) - return mockKeyStorage{pk} +const mockKeyValue = "mockKey" + +func (m mockPubKey) Equals(key crypto.Key) bool { + return true +} + +func (m mockPubKey) Raw() ([]byte, error) { + return []byte(mockKeyValue), nil +} + +func (m mockPubKey) Encrypt(message []byte) ([]byte, error) { + return message, nil +} + +func (m mockPubKey) Verify(data []byte, sig []byte) (bool, error) { + return true, nil +} + +func (m mockPubKey) Marshall() ([]byte, error) { + return []byte(mockKeyValue), nil +} + +func (m mockPubKey) Storage() []byte { + return []byte(mockKeyValue) +} + +func (m mockPubKey) Account() string { + return mockKeyValue +} + +func (m mockPubKey) PeerId() string { + return mockKeyValue +} + +func (m mockPubKey) LibP2P() (libcrypto.PubKey, error) { + return nil, fmt.Errorf("can't be converted in libp2p") +} + +type mockKeyStorage struct { +} + +func newMockKeyStorage() mockKeyStorage { + return mockKeyStorage{} } func (m mockKeyStorage) PubKeyFromProto(protoBytes []byte) (crypto.PubKey, error) { - return m.key, nil + return mockPubKey{}, nil } type MockChangeCreator struct{} @@ -76,7 +116,7 @@ func (c *MockChangeCreator) CreateNewTreeStorage(treeId, aclHeadId string) trees func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { root, _ := treeStorage.Root() changeBuilder := &nonVerifiableChangeBuilder{ - ChangeBuilder: NewChangeBuilder(newKeyStorage(), root), + ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root), } deps := objectTreeDeps{ changeBuilder: changeBuilder, diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index 086cb883..73db3f42 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -29,10 +29,11 @@ type processMsg struct { } type msgDescription struct { - name string - from string - to string - heads []string + name string + from string + to string + heads []string + changes []*treechangeproto.RawTreeChangeWithId } func (p *processMsg) description() (descr msgDescription) { @@ -50,14 +51,17 @@ func (p *processMsg) description() (descr msgDescription) { cnt := unmarshalled.GetContent().GetHeadUpdate() descr.name = "HeadUpdate" descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetHeadUpdate().Changes case unmarshalled.GetContent().GetFullSyncRequest() != nil: cnt := unmarshalled.GetContent().GetFullSyncRequest() descr.name = "FullSyncRequest" descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetFullSyncRequest().Changes case unmarshalled.GetContent().GetFullSyncResponse() != nil: cnt := unmarshalled.GetContent().GetFullSyncResponse() descr.name = "FullSyncResponse" descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetFullSyncResponse().Changes } return } @@ -322,7 +326,7 @@ func (p *processFixture) stop() { p.wg.Wait() } -func TestSend_EmptyClient(t *testing.T) { +func TestSend_EmptyClientGetsFullHistory(t *testing.T) { treeId := "treeId" spaceId := "spaceId" keys, err := accountdata.NewRandom() @@ -345,7 +349,7 @@ func TestSend_EmptyClient(t *testing.T) { fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ NewHeads: nil, RawChanges: []*treechangeproto.RawTreeChangeWithId{ - changeCreator.CreateRaw("1", aclList.Id(), treeId, false, treeId), + changeCreator.CreateRaw("1", aclList.Id(), treeId, true, treeId), }, }) time.Sleep(100 * time.Millisecond) @@ -357,9 +361,17 @@ func TestSend_EmptyClient(t *testing.T) { require.Equal(t, firstHeads, secondHeads) require.Equal(t, []string{"1"}, firstHeads) logMsgs := fx.log.batcher.GetAll() + + var fullResponseMsg *processMsg for _, msg := range logMsgs { - fmt.Println(msg.description()) + descr := msg.description() + if descr.name == "FullSyncResponse" { + fullResponseMsg = &msg + } } + require.NotNil(t, fullResponseMsg) + // that means that we got not only the last snapshot, but also the first one + require.Len(t, fullResponseMsg.description().changes, 2) } func TestSimple_TwoPeers(t *testing.T) { From 125d8b46269fd16dae53dac63d62bbef604026a1 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 17 Apr 2023 21:49:51 +0200 Subject: [PATCH 09/45] Add test with random trees --- .../object/tree/synctree/syncclient_test.go | 441 +++-------------- .../object/tree/synctree/synctreehandler.go | 22 +- .../object/tree/synctree/utils_test.go | 446 ++++++++++++++++++ 3 files changed, 518 insertions(+), 391 deletions(-) create mode 100644 commonspace/object/tree/synctree/utils_test.go diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncclient_test.go index 73db3f42..ed6dd710 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncclient_test.go @@ -8,325 +8,14 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" - "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" - "github.com/anytypeio/any-sync/commonspace/spacesyncproto" - "github.com/anytypeio/any-sync/commonspace/syncstatus" - "github.com/anytypeio/any-sync/net/peer" - "github.com/cheggaaa/mb/v3" - "github.com/gogo/protobuf/proto" + "github.com/anytypeio/any-sync/util/slice" "github.com/stretchr/testify/require" - "golang.org/x/exp/slices" - "sync" + "math/rand" "testing" "time" ) -type processMsg struct { - msg *spacesyncproto.ObjectSyncMessage - senderId string - receiverId string - userMsg *objecttree.RawChangesPayload -} - -type msgDescription struct { - name string - from string - to string - heads []string - changes []*treechangeproto.RawTreeChangeWithId -} - -func (p *processMsg) description() (descr msgDescription) { - unmarshalled := &treechangeproto.TreeSyncMessage{} - err := proto.Unmarshal(p.msg.Payload, unmarshalled) - if err != nil { - panic(err) - } - descr = msgDescription{ - from: p.senderId, - to: p.receiverId, - } - switch { - case unmarshalled.GetContent().GetHeadUpdate() != nil: - cnt := unmarshalled.GetContent().GetHeadUpdate() - descr.name = "HeadUpdate" - descr.heads = cnt.Heads - descr.changes = unmarshalled.GetContent().GetHeadUpdate().Changes - case unmarshalled.GetContent().GetFullSyncRequest() != nil: - cnt := unmarshalled.GetContent().GetFullSyncRequest() - descr.name = "FullSyncRequest" - descr.heads = cnt.Heads - descr.changes = unmarshalled.GetContent().GetFullSyncRequest().Changes - case unmarshalled.GetContent().GetFullSyncResponse() != nil: - cnt := unmarshalled.GetContent().GetFullSyncResponse() - descr.name = "FullSyncResponse" - descr.heads = cnt.Heads - descr.changes = unmarshalled.GetContent().GetFullSyncResponse().Changes - } - return -} - -type messageLog struct { - batcher *mb.MB[processMsg] -} - -func newMessageLog() *messageLog { - return &messageLog{batcher: mb.New[processMsg](0)} -} - -func (m *messageLog) addMessage(msg processMsg) { - m.batcher.Add(context.Background(), msg) -} - -type processSyncHandler struct { - synchandler.SyncHandler - batcher *mb.MB[processMsg] - peerId string - aclList list.AclList - log *messageLog - syncClient SyncClient -} - -func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { - if p.SyncHandler != nil { - return p.SyncHandler.HandleMessage(ctx, senderId, request) - } - unmarshalled := &treechangeproto.TreeSyncMessage{} - err = proto.Unmarshal(request.Payload, unmarshalled) - if err != nil { - return - } - if unmarshalled.Content.GetFullSyncResponse() == nil { - newTreeRequest := GetRequestFactory().CreateNewTreeRequest() - var objMsg *spacesyncproto.ObjectSyncMessage - objMsg, err = marshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") - if err != nil { - return - } - return p.manager().SendPeer(context.Background(), senderId, objMsg) - } - fullSyncResponse := unmarshalled.Content.GetFullSyncResponse() - treeStorage, _ := treestorage.NewInMemoryTreeStorage(unmarshalled.RootChange, fullSyncResponse.Heads, fullSyncResponse.Changes) - tree, err := createTestTree(p.aclList, treeStorage) - if err != nil { - return - } - netTree := &broadcastTree{ - ObjectTree: tree, - SyncClient: p.syncClient, - } - p.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, p.syncClient, syncstatus.NewNoOpSyncStatus()) - return -} - -func newProcessSyncHandler(peerId string, syncHandler synchandler.SyncHandler) *processSyncHandler { - batcher := mb.New[processMsg](0) - return &processSyncHandler{ - SyncHandler: syncHandler, - batcher: batcher, - peerId: peerId, - } -} - -func (p *processSyncHandler) manager() *processPeerManager { - if p.SyncHandler != nil { - return p.SyncHandler.(*syncTreeHandler).syncClient.(*syncClient).PeerManager.(*processPeerManager) - } - return p.syncClient.(*syncClient).PeerManager.(*processPeerManager) -} - -func (p *processSyncHandler) tree() *broadcastTree { - return p.SyncHandler.(*syncTreeHandler).objTree.(*broadcastTree) -} - -func (p *processSyncHandler) send(ctx context.Context, msg processMsg) (err error) { - return p.batcher.Add(ctx, msg) -} - -func (p *processSyncHandler) sendRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) { - p.batcher.Add(ctx, processMsg{userMsg: &changes}) -} - -func (p *processSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.WaitGroup) { - wg.Add(1) - go func() { - defer wg.Done() - for { - res, err := p.batcher.WaitOne(ctx) - if err != nil { - return - } - if res.userMsg != nil { - p.tree().Lock() - userRes, err := p.tree().AddRawChanges(ctx, *res.userMsg) - require.NoError(t, err) - fmt.Println("user add result", userRes.Heads) - p.tree().Unlock() - continue - } - err = p.HandleMessage(ctx, res.senderId, res.msg) - if err != nil { - fmt.Println("error handling message", err.Error()) - continue - } - } - }() -} - -type processPeerManager struct { - peerId string - handlers map[string]*processSyncHandler - log *messageLog -} - -func newProcessPeerManager(peerId string, log *messageLog) *processPeerManager { - return &processPeerManager{handlers: map[string]*processSyncHandler{}, peerId: peerId, log: log} -} - -func (m *processPeerManager) addHandler(peerId string, handler *processSyncHandler) { - m.handlers[peerId] = handler -} - -func (m *processPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { - pMsg := processMsg{ - msg: msg, - senderId: m.peerId, - receiverId: peerId, - } - m.log.addMessage(pMsg) - return m.handlers[peerId].send(context.Background(), pMsg) -} - -func (m *processPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { - for _, handler := range m.handlers { - pMsg := processMsg{ - msg: msg, - senderId: m.peerId, - receiverId: handler.peerId, - } - m.log.addMessage(pMsg) - handler.send(context.Background(), pMsg) - } - return -} - -func (m *processPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { - panic("should not be called") -} - -type broadcastTree struct { - objecttree.ObjectTree - SyncClient -} - -func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) (objecttree.AddResult, error) { - res, err := b.ObjectTree.AddRawChanges(ctx, changes) - if err != nil { - return objecttree.AddResult{}, err - } - upd := b.SyncClient.CreateHeadUpdate(b.ObjectTree, res.Added) - b.SyncClient.Broadcast(ctx, upd) - return res, nil -} - -func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *processSyncHandler { - factory := GetRequestFactory() - syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) - netTree := &broadcastTree{ - ObjectTree: objTree, - SyncClient: syncClient, - } - handler := newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) - return newProcessSyncHandler(peerId, handler) -} - -func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *processSyncHandler { - factory := GetRequestFactory() - syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) - - batcher := mb.New[processMsg](0) - return &processSyncHandler{ - batcher: batcher, - peerId: peerId, - aclList: aclList, - log: log, - syncClient: syncClient, - } -} - -func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage { - changeCreator := objecttree.NewMockChangeCreator() - st := changeCreator.CreateNewTreeStorage(treeId, aclList.Head().Id) - return st -} - -func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (objecttree.ObjectTree, error) { - return objecttree.BuildTestableTree(aclList, storage) -} - -type fixtureDeps struct { - aclList list.AclList - initStorage *treestorage.InMemoryTreeStorage - connectionMap map[string][]string - emptyTrees []string -} - -type processFixture struct { - handlers map[string]*processSyncHandler - log *messageLog - wg *sync.WaitGroup - ctx context.Context - cancel context.CancelFunc -} - -func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processFixture { - var ( - handlers = map[string]*processSyncHandler{} - log = newMessageLog() - wg = sync.WaitGroup{} - ctx, cancel = context.WithCancel(context.Background()) - ) - - for peerId := range deps.connectionMap { - var handler *processSyncHandler - if slices.Contains(deps.emptyTrees, peerId) { - handler = createEmptySyncHandler(peerId, spaceId, deps.aclList, log) - } else { - stCopy := deps.initStorage.Copy() - testTree, err := createTestTree(deps.aclList, stCopy) - require.NoError(t, err) - handler = createSyncHandler(peerId, spaceId, testTree, log) - } - handlers[peerId] = handler - } - for peerId, connectionMap := range deps.connectionMap { - handler := handlers[peerId] - manager := handler.manager() - for _, connectionId := range connectionMap { - manager.addHandler(connectionId, handlers[connectionId]) - } - } - return &processFixture{ - handlers: handlers, - log: log, - wg: &wg, - ctx: ctx, - cancel: cancel, - } -} - -func (p *processFixture) run(t *testing.T) { - for _, handler := range p.handlers { - handler.run(p.ctx, t, p.wg) - } -} - -func (p *processFixture) stop() { - p.cancel() - p.wg.Wait() -} - -func TestSend_EmptyClientGetsFullHistory(t *testing.T) { +func TestEmptyClientGetsFullHistory(t *testing.T) { treeId := "treeId" spaceId := "spaceId" keys, err := accountdata.NewRandom() @@ -356,70 +45,22 @@ func TestSend_EmptyClientGetsFullHistory(t *testing.T) { fx.stop() firstHeads := fx.handlers["peer1"].tree().Heads() secondHeads := fx.handlers["peer2"].tree().Heads() - slices.Sort(firstHeads) - slices.Sort(secondHeads) - require.Equal(t, firstHeads, secondHeads) + require.True(t, slice.SortedEquals(firstHeads, secondHeads)) require.Equal(t, []string{"1"}, firstHeads) logMsgs := fx.log.batcher.GetAll() - var fullResponseMsg *processMsg + var fullResponseMsg msgDescription for _, msg := range logMsgs { descr := msg.description() if descr.name == "FullSyncResponse" { - fullResponseMsg = &msg + fullResponseMsg = descr } } - require.NotNil(t, fullResponseMsg) // that means that we got not only the last snapshot, but also the first one - require.Len(t, fullResponseMsg.description().changes, 2) + require.Len(t, fullResponseMsg.changes, 2) } -func TestSimple_TwoPeers(t *testing.T) { - treeId := "treeId" - spaceId := "spaceId" - keys, err := accountdata.NewRandom() - require.NoError(t, err) - aclList, err := list.NewTestDerivedAcl(spaceId, keys) - require.NoError(t, err) - storage := createStorage(treeId, aclList) - changeCreator := objecttree.NewMockChangeCreator() - deps := fixtureDeps{ - aclList: aclList, - initStorage: storage.(*treestorage.InMemoryTreeStorage), - connectionMap: map[string][]string{ - "peer1": []string{"peer2"}, - "peer2": []string{"peer1"}, - }, - } - fx := newProcessFixture(t, spaceId, deps) - fx.run(t) - fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ - NewHeads: nil, - RawChanges: []*treechangeproto.RawTreeChangeWithId{ - changeCreator.CreateRaw("1", aclList.Id(), treeId, false, treeId), - }, - }) - fx.handlers["peer2"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ - NewHeads: nil, - RawChanges: []*treechangeproto.RawTreeChangeWithId{ - changeCreator.CreateRaw("2", aclList.Id(), treeId, false, treeId), - }, - }) - time.Sleep(100 * time.Millisecond) - fx.stop() - firstHeads := fx.handlers["peer1"].tree().Heads() - secondHeads := fx.handlers["peer2"].tree().Heads() - slices.Sort(firstHeads) - slices.Sort(secondHeads) - require.Equal(t, firstHeads, secondHeads) - require.Equal(t, []string{"1", "2"}, firstHeads) - logMsgs := fx.log.batcher.GetAll() - for _, msg := range logMsgs { - fmt.Println(msg.description()) - } -} - -func TestSimple_ThreePeers(t *testing.T) { +func TestRandomTreeMerge(t *testing.T) { treeId := "treeId" spaceId := "spaceId" keys, err := accountdata.NewRandom() @@ -427,6 +68,21 @@ func TestSimple_ThreePeers(t *testing.T) { aclList, err := list.NewTestDerivedAcl(spaceId, keys) storage := createStorage(treeId, aclList) changeCreator := objecttree.NewMockChangeCreator() + rnd := rand.New(rand.NewSource(time.Now().Unix())) + params := genParams{ + prefix: "peer1", + aclId: aclList.Id(), + startIdx: 0, + levels: 10, + snapshotId: treeId, + prevHeads: []string{treeId}, + isSnapshot: func() bool { + return rnd.Intn(100) > 80 + }, + } + initialRes := genChanges(changeCreator, params) + err = storage.TransactionAdd(initialRes.changes, initialRes.heads) + require.NoError(t, err) deps := fixtureDeps{ aclList: aclList, initStorage: storage.(*treestorage.InMemoryTreeStorage), @@ -435,31 +91,44 @@ func TestSimple_ThreePeers(t *testing.T) { "peer2": []string{"node1"}, "node1": []string{"peer1", "peer2"}, }, + emptyTrees: []string{"peer2", "node1"}, } fx := newProcessFixture(t, spaceId, deps) fx.run(t) fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ - NewHeads: nil, - RawChanges: []*treechangeproto.RawTreeChangeWithId{ - changeCreator.CreateRaw("1", aclList.Id(), treeId, false, treeId), - }, + NewHeads: initialRes.heads, + RawChanges: initialRes.changes, }) - fx.handlers["peer2"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ - NewHeads: nil, - RawChanges: []*treechangeproto.RawTreeChangeWithId{ - changeCreator.CreateRaw("2", aclList.Id(), treeId, false, treeId), - }, - }) - time.Sleep(100 * time.Millisecond) - fx.stop() + time.Sleep(1000 * time.Millisecond) firstHeads := fx.handlers["peer1"].tree().Heads() secondHeads := fx.handlers["peer2"].tree().Heads() - slices.Sort(firstHeads) - slices.Sort(secondHeads) - require.Equal(t, firstHeads, secondHeads) - require.Equal(t, []string{"1", "2"}, firstHeads) - logMsgs := fx.log.batcher.GetAll() - for _, msg := range logMsgs { - fmt.Println(msg.description()) + require.True(t, slice.UnsortedEquals(firstHeads, secondHeads)) + params = genParams{ + prefix: "peer1", + aclId: aclList.Id(), + startIdx: 11, + levels: 10, + snapshotId: initialRes.snapshotId, + prevHeads: initialRes.heads, + isSnapshot: func() bool { + return rnd.Intn(100) > 80 + }, } + peer1Res := genChanges(changeCreator, params) + params.prefix = "peer2" + peer2Res := genChanges(changeCreator, params) + fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: peer1Res.heads, + RawChanges: peer1Res.changes, + }) + fx.handlers["peer2"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: peer2Res.heads, + RawChanges: peer2Res.changes, + }) + time.Sleep(1000 * time.Millisecond) + fx.stop() + firstHeads = fx.handlers["peer1"].tree().Heads() + secondHeads = fx.handlers["peer2"].tree().Heads() + fmt.Println(firstHeads) + fmt.Println(secondHeads) } diff --git a/commonspace/object/tree/synctree/synctreehandler.go b/commonspace/object/tree/synctree/synctreehandler.go index de79d0de..a41a2bad 100644 --- a/commonspace/object/tree/synctree/synctreehandler.go +++ b/commonspace/object/tree/synctree/synctreehandler.go @@ -83,13 +83,19 @@ func (s *syncTreeHandler) handleHeadUpdate( objTree = s.objTree ) - log := log.With(zap.Strings("heads", objTree.Heads()), zap.String("treeId", objTree.Id()), zap.String("spaceId", s.spaceId)) + log := log.With( + zap.Strings("update heads", update.Heads), + zap.String("treeId", objTree.Id()), + zap.String("spaceId", s.spaceId), + zap.Int("len(update changes)", len(update.Changes))) log.DebugCtx(ctx, "received head update message") defer func() { if err != nil { log.With(zap.Error(err)).Debug("head update finished with error") } else if fullRequest != nil { + cnt := fullRequest.Content.GetFullSyncRequest() + log = log.With(zap.Strings("request heads", cnt.Heads), zap.Int("len(request changes)", len(cnt.Changes))) log.DebugCtx(ctx, "sending full sync request") } else { if !isEmptyUpdate { @@ -151,19 +157,21 @@ func (s *syncTreeHandler) handleFullSyncRequest( ) log := log.With(zap.String("senderId", senderId), - zap.Strings("heads", request.Heads), + zap.Strings("request heads", request.Heads), zap.String("treeId", s.objTree.Id()), zap.String("replyId", replyId), - zap.String("spaceId", s.spaceId)) + zap.String("spaceId", s.spaceId), + zap.Int("len(request changes)", len(request.Changes))) log.DebugCtx(ctx, "received full sync request message") defer func() { if err != nil { log.With(zap.Error(err)).DebugCtx(ctx, "full sync request finished with error") - s.syncClient.SendWithReply(ctx, senderId, treechangeproto.WrapError(err, header), replyId) return } else if fullResponse != nil { + cnt := fullResponse.Content.GetFullSyncResponse() + log = log.With(zap.Strings("response heads", cnt.Heads), zap.Int("len(response changes)", len(cnt.Changes))) log.DebugCtx(ctx, "full sync response sent") } }() @@ -192,7 +200,11 @@ func (s *syncTreeHandler) handleFullSyncResponse( var ( objTree = s.objTree ) - log := log.With(zap.Strings("heads", response.Heads), zap.String("treeId", s.objTree.Id()), zap.String("spaceId", s.spaceId)) + log := log.With( + zap.Strings("heads", response.Heads), + zap.String("treeId", s.objTree.Id()), + zap.String("spaceId", s.spaceId), + zap.Int("len(changes)", len(response.Changes))) log.DebugCtx(ctx, "received full sync response message") defer func() { diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go new file mode 100644 index 00000000..0399d239 --- /dev/null +++ b/commonspace/object/tree/synctree/utils_test.go @@ -0,0 +1,446 @@ +package synctree + +import ( + "context" + "fmt" + "github.com/anytypeio/any-sync/commonspace/object/accountdata" + "github.com/anytypeio/any-sync/commonspace/object/acl/list" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" + "github.com/anytypeio/any-sync/commonspace/spacesyncproto" + "github.com/anytypeio/any-sync/commonspace/syncstatus" + "github.com/anytypeio/any-sync/net/peer" + "github.com/cheggaaa/mb/v3" + "github.com/gogo/protobuf/proto" + "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" + "math/rand" + "sync" + "testing" + "time" +) + +type processMsg struct { + msg *spacesyncproto.ObjectSyncMessage + senderId string + receiverId string + userMsg *objecttree.RawChangesPayload +} + +type msgDescription struct { + name string + from string + to string + heads []string + changes []*treechangeproto.RawTreeChangeWithId +} + +func (p *processMsg) description() (descr msgDescription) { + unmarshalled := &treechangeproto.TreeSyncMessage{} + err := proto.Unmarshal(p.msg.Payload, unmarshalled) + if err != nil { + panic(err) + } + descr = msgDescription{ + from: p.senderId, + to: p.receiverId, + } + switch { + case unmarshalled.GetContent().GetHeadUpdate() != nil: + cnt := unmarshalled.GetContent().GetHeadUpdate() + descr.name = "HeadUpdate" + descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetHeadUpdate().Changes + case unmarshalled.GetContent().GetFullSyncRequest() != nil: + cnt := unmarshalled.GetContent().GetFullSyncRequest() + descr.name = "FullSyncRequest" + descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetFullSyncRequest().Changes + case unmarshalled.GetContent().GetFullSyncResponse() != nil: + cnt := unmarshalled.GetContent().GetFullSyncResponse() + descr.name = "FullSyncResponse" + descr.heads = cnt.Heads + descr.changes = unmarshalled.GetContent().GetFullSyncResponse().Changes + } + return +} + +type messageLog struct { + batcher *mb.MB[processMsg] +} + +func newMessageLog() *messageLog { + return &messageLog{batcher: mb.New[processMsg](0)} +} + +func (m *messageLog) addMessage(msg processMsg) { + m.batcher.Add(context.Background(), msg) +} + +type processSyncHandler struct { + synchandler.SyncHandler + batcher *mb.MB[processMsg] + peerId string + aclList list.AclList + log *messageLog + syncClient SyncClient +} + +func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { + if p.SyncHandler != nil { + return p.SyncHandler.HandleMessage(ctx, senderId, request) + } + unmarshalled := &treechangeproto.TreeSyncMessage{} + err = proto.Unmarshal(request.Payload, unmarshalled) + if err != nil { + return + } + if unmarshalled.Content.GetFullSyncResponse() == nil { + newTreeRequest := GetRequestFactory().CreateNewTreeRequest() + var objMsg *spacesyncproto.ObjectSyncMessage + objMsg, err = marshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") + if err != nil { + return + } + return p.manager().SendPeer(context.Background(), senderId, objMsg) + } + fullSyncResponse := unmarshalled.Content.GetFullSyncResponse() + treeStorage, _ := treestorage.NewInMemoryTreeStorage(unmarshalled.RootChange, []string{unmarshalled.RootChange.Id}, nil) + tree, err := createTestTree(p.aclList, treeStorage) + if err != nil { + return + } + netTree := &broadcastTree{ + ObjectTree: tree, + SyncClient: p.syncClient, + } + res, err := netTree.AddRawChanges(context.Background(), objecttree.RawChangesPayload{ + NewHeads: fullSyncResponse.Heads, + RawChanges: fullSyncResponse.Changes, + }) + if err != nil { + return + } + p.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, p.syncClient, syncstatus.NewNoOpSyncStatus()) + var objMsg *spacesyncproto.ObjectSyncMessage + newTreeRequest := GetRequestFactory().CreateHeadUpdate(netTree, res.Added) + objMsg, err = marshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") + if err != nil { + return + } + return p.manager().Broadcast(context.Background(), objMsg) +} + +func newProcessSyncHandler(peerId string, syncHandler synchandler.SyncHandler) *processSyncHandler { + batcher := mb.New[processMsg](0) + return &processSyncHandler{ + SyncHandler: syncHandler, + batcher: batcher, + peerId: peerId, + } +} + +func (p *processSyncHandler) manager() *processPeerManager { + if p.SyncHandler != nil { + return p.SyncHandler.(*syncTreeHandler).syncClient.(*syncClient).PeerManager.(*processPeerManager) + } + return p.syncClient.(*syncClient).PeerManager.(*processPeerManager) +} + +func (p *processSyncHandler) tree() *broadcastTree { + return p.SyncHandler.(*syncTreeHandler).objTree.(*broadcastTree) +} + +func (p *processSyncHandler) send(ctx context.Context, msg processMsg) (err error) { + return p.batcher.Add(ctx, msg) +} + +func (p *processSyncHandler) sendRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) { + p.batcher.Add(ctx, processMsg{userMsg: &changes}) +} + +func (p *processSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.WaitGroup) { + wg.Add(1) + go func() { + defer wg.Done() + for { + res, err := p.batcher.WaitOne(ctx) + if err != nil { + return + } + if res.userMsg != nil { + p.tree().Lock() + userRes, err := p.tree().AddRawChanges(ctx, *res.userMsg) + require.NoError(t, err) + fmt.Println("user add result", userRes.Heads) + p.tree().Unlock() + continue + } + err = p.HandleMessage(ctx, res.senderId, res.msg) + if err != nil { + fmt.Println("error handling message", err.Error()) + continue + } + } + }() +} + +type processPeerManager struct { + peerId string + handlers map[string]*processSyncHandler + log *messageLog +} + +func newProcessPeerManager(peerId string, log *messageLog) *processPeerManager { + return &processPeerManager{handlers: map[string]*processSyncHandler{}, peerId: peerId, log: log} +} + +func (m *processPeerManager) addHandler(peerId string, handler *processSyncHandler) { + m.handlers[peerId] = handler +} + +func (m *processPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { + pMsg := processMsg{ + msg: msg, + senderId: m.peerId, + receiverId: peerId, + } + m.log.addMessage(pMsg) + return m.handlers[peerId].send(context.Background(), pMsg) +} + +func (m *processPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { + for _, handler := range m.handlers { + pMsg := processMsg{ + msg: msg, + senderId: m.peerId, + receiverId: handler.peerId, + } + m.log.addMessage(pMsg) + handler.send(context.Background(), pMsg) + } + return +} + +func (m *processPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { + panic("should not be called") +} + +type broadcastTree struct { + objecttree.ObjectTree + SyncClient +} + +func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) (objecttree.AddResult, error) { + res, err := b.ObjectTree.AddRawChanges(ctx, changes) + if err != nil { + return objecttree.AddResult{}, err + } + upd := b.SyncClient.CreateHeadUpdate(b.ObjectTree, res.Added) + b.SyncClient.Broadcast(ctx, upd) + return res, nil +} + +func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *processSyncHandler { + factory := GetRequestFactory() + syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) + netTree := &broadcastTree{ + ObjectTree: objTree, + SyncClient: syncClient, + } + handler := newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) + return newProcessSyncHandler(peerId, handler) +} + +func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *processSyncHandler { + factory := GetRequestFactory() + syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) + + batcher := mb.New[processMsg](0) + return &processSyncHandler{ + batcher: batcher, + peerId: peerId, + aclList: aclList, + log: log, + syncClient: syncClient, + } +} + +func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage { + changeCreator := objecttree.NewMockChangeCreator() + st := changeCreator.CreateNewTreeStorage(treeId, aclList.Head().Id) + return st +} + +func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (objecttree.ObjectTree, error) { + return objecttree.BuildTestableTree(aclList, storage) +} + +type fixtureDeps struct { + aclList list.AclList + initStorage *treestorage.InMemoryTreeStorage + connectionMap map[string][]string + emptyTrees []string +} + +type processFixture struct { + handlers map[string]*processSyncHandler + log *messageLog + wg *sync.WaitGroup + ctx context.Context + cancel context.CancelFunc +} + +func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processFixture { + var ( + handlers = map[string]*processSyncHandler{} + log = newMessageLog() + wg = sync.WaitGroup{} + ctx, cancel = context.WithCancel(context.Background()) + ) + + for peerId := range deps.connectionMap { + var handler *processSyncHandler + if slices.Contains(deps.emptyTrees, peerId) { + handler = createEmptySyncHandler(peerId, spaceId, deps.aclList, log) + } else { + stCopy := deps.initStorage.Copy() + testTree, err := createTestTree(deps.aclList, stCopy) + require.NoError(t, err) + handler = createSyncHandler(peerId, spaceId, testTree, log) + } + handlers[peerId] = handler + } + for peerId, connectionMap := range deps.connectionMap { + handler := handlers[peerId] + manager := handler.manager() + for _, connectionId := range connectionMap { + manager.addHandler(connectionId, handlers[connectionId]) + } + } + return &processFixture{ + handlers: handlers, + log: log, + wg: &wg, + ctx: ctx, + cancel: cancel, + } +} + +func (p *processFixture) run(t *testing.T) { + for _, handler := range p.handlers { + handler.run(p.ctx, t, p.wg) + } +} + +func (p *processFixture) stop() { + p.cancel() + p.wg.Wait() +} + +type genParams struct { + prefix string + aclId string + startIdx int + levels int + snapshotId string + prevHeads []string + isSnapshot func() bool +} + +type genResult struct { + changes []*treechangeproto.RawTreeChangeWithId + heads []string + snapshotId string +} + +func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res genResult) { + src := rand.NewSource(time.Now().Unix()) + rnd := rand.New(src) + var ( + prevHeads []string + snapshotId = params.snapshotId + ) + prevHeads = append(prevHeads, params.prevHeads...) + + for i := 0; i < params.levels; i++ { + if params.isSnapshot() { + newId := fmt.Sprintf("%s.%d.%d", params.prefix, params.startIdx+i, 0) + newCh := creator.CreateRaw(newId, params.aclId, snapshotId, true, prevHeads...) + res.changes = append(res.changes, newCh) + prevHeads = []string{newId} + snapshotId = newId + continue + } + perLevel := rnd.Intn(10) + if perLevel == 0 { + perLevel = 1 + } + var ( + newHeads []string + usedIds = map[string]struct{}{} + ) + for j := 0; j < perLevel; j++ { + // if we didn't connect with all prev ones + prevConns := rnd.Intn(len(prevHeads)) + if prevConns == 0 { + prevConns = 1 + } + rnd.Shuffle(len(prevHeads), func(i, j int) { + prevHeads[i], prevHeads[j] = prevHeads[j], prevHeads[i] + }) + if j == perLevel-1 && len(usedIds) != len(prevHeads) { + var unusedIds []string + for _, id := range prevHeads { + if _, exists := usedIds[id]; !exists { + unusedIds = append(unusedIds, id) + } + } + prevHeads = unusedIds + prevConns = len(prevHeads) + } + var prevChId []string + for k := 0; k < prevConns; k++ { + prevChId = append(prevChId, prevHeads[k]) + usedIds[prevHeads[k]] = struct{}{} + } + newId := fmt.Sprintf("%s.%d.%d", params.prefix, params.startIdx+i, j) + newCh := creator.CreateRaw(newId, params.aclId, snapshotId, false, prevChId...) + res.changes = append(res.changes, newCh) + newHeads = append(newHeads, newId) + } + prevHeads = newHeads + } + res.heads = prevHeads + res.snapshotId = snapshotId + return +} + +func TestGenChanges(t *testing.T) { + treeId := "treeId" + spaceId := "spaceId" + keys, err := accountdata.NewRandom() + require.NoError(t, err) + aclList, err := list.NewTestDerivedAcl(spaceId, keys) + storage := createStorage(treeId, aclList) + creator := objecttree.NewMockChangeCreator() + rnd := rand.New(rand.NewSource(time.Now().Unix())) + params := genParams{ + prefix: "peerId", + aclId: aclList.Id(), + startIdx: 0, + levels: 10, + snapshotId: treeId, + prevHeads: []string{treeId}, + isSnapshot: func() bool { + return rnd.Intn(100) > 80 + }, + } + res := genChanges(creator, params) + storage.TransactionAdd(res.changes, res.heads) + tr, err := createTestTree(aclList, storage) + require.NoError(t, err) + fmt.Println(tr.Debug(objecttree.NoOpDescriptionParser)) +} From 678ec256bf8347e45651b90a2adea113fe42b6d0 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 17 Apr 2023 23:51:17 +0200 Subject: [PATCH 10/45] Add tests and don't load data into changes --- .../object/tree/objecttree/changebuilder.go | 33 ++++++++++- .../object/tree/objecttree/objecttree.go | 14 +---- .../tree/objecttree/objecttreefactory.go | 59 +++++++++++++++++++ .../object/tree/objecttree/rawloader.go | 23 ++++++-- .../object/tree/objecttree/testutils.go | 18 ------ ...yncclient_test.go => syncprotocol_test.go} | 59 ++++++++++++------- .../object/tree/synctree/utils_test.go | 35 ++--------- .../object/tree/treestorage/inmemory.go | 19 ++++++ 8 files changed, 171 insertions(+), 89 deletions(-) rename commonspace/object/tree/synctree/{syncclient_test.go => syncprotocol_test.go} (75%) diff --git a/commonspace/object/tree/objecttree/changebuilder.go b/commonspace/object/tree/objecttree/changebuilder.go index cb22d249..2f10fca9 100644 --- a/commonspace/object/tree/objecttree/changebuilder.go +++ b/commonspace/object/tree/objecttree/changebuilder.go @@ -52,6 +52,18 @@ func (c *nonVerifiableChangeBuilder) Marshall(ch *Change) (raw *treechangeproto. return c.ChangeBuilder.Marshall(ch) } +type emptyDataChangeBuilder struct { + ChangeBuilder +} + +func (c *emptyDataChangeBuilder) Build(payload BuilderContent) (ch *Change, raw *treechangeproto.RawTreeChangeWithId, err error) { + panic("should not be called") +} + +func (c *emptyDataChangeBuilder) Marshall(ch *Change) (raw *treechangeproto.RawTreeChangeWithId, err error) { + panic("should not be called") +} + type ChangeBuilder interface { Unmarshall(rawIdChange *treechangeproto.RawTreeChangeWithId, verify bool) (ch *Change, err error) Build(payload BuilderContent) (ch *Change, raw *treechangeproto.RawTreeChangeWithId, err error) @@ -59,13 +71,28 @@ type ChangeBuilder interface { Marshall(ch *Change) (*treechangeproto.RawTreeChangeWithId, error) } +type newChangeFunc = func(id string, identity crypto.PubKey, ch *treechangeproto.TreeChange, signature []byte) *Change + type changeBuilder struct { rootChange *treechangeproto.RawTreeChangeWithId keys crypto.KeyStorage + newChange newChangeFunc +} + +func NewEmptyDataBuilder(keys crypto.KeyStorage, rootChange *treechangeproto.RawTreeChangeWithId) ChangeBuilder { + return &emptyDataChangeBuilder{&changeBuilder{ + rootChange: rootChange, + keys: keys, + newChange: func(id string, identity crypto.PubKey, ch *treechangeproto.TreeChange, signature []byte) *Change { + c := NewChange(id, identity, ch, signature) + c.Data = nil + return c + }, + }} } func NewChangeBuilder(keys crypto.KeyStorage, rootChange *treechangeproto.RawTreeChangeWithId) ChangeBuilder { - return &changeBuilder{keys: keys, rootChange: rootChange} + return &changeBuilder{keys: keys, rootChange: rootChange, newChange: NewChange} } func (c *changeBuilder) Unmarshall(rawIdChange *treechangeproto.RawTreeChangeWithId, verify bool) (ch *Change, err error) { @@ -197,7 +224,7 @@ func (c *changeBuilder) Build(payload BuilderContent) (ch *Change, rawIdChange * if err != nil { return } - ch = NewChange(id, payload.PrivKey.GetPublic(), change, signature) + ch = c.newChange(id, payload.PrivKey.GetPublic(), change, signature) rawIdChange = &treechangeproto.RawTreeChangeWithId{ RawChange: marshalledRawChange, Id: id, @@ -268,7 +295,7 @@ func (c *changeBuilder) unmarshallRawChange(raw *treechangeproto.RawTreeChange, if err != nil { return } - ch = NewChange(id, key, unmarshalled, raw.Signature) + ch = c.newChange(id, key, unmarshalled, raw.Signature) return } diff --git a/commonspace/object/tree/objecttree/objecttree.go b/commonspace/object/tree/objecttree/objecttree.go index 430329c2..5f2d98e4 100644 --- a/commonspace/object/tree/objecttree/objecttree.go +++ b/commonspace/object/tree/objecttree/objecttree.go @@ -621,19 +621,7 @@ func (ot *objectTree) ChangesAfterCommonSnapshot(theirPath, theirHeads []string) } } - if commonSnapshot == ot.tree.RootId() { - return ot.getChangesFromTree(theirHeads) - } else { - return ot.getChangesFromDB(commonSnapshot, theirHeads) - } -} - -func (ot *objectTree) getChangesFromTree(theirHeads []string) (rawChanges []*treechangeproto.RawTreeChangeWithId, err error) { - return ot.rawChangeLoader.LoadFromTree(ot.tree, theirHeads) -} - -func (ot *objectTree) getChangesFromDB(commonSnapshot string, theirHeads []string) (rawChanges []*treechangeproto.RawTreeChangeWithId, err error) { - return ot.rawChangeLoader.LoadFromStorage(commonSnapshot, ot.tree.headIds, theirHeads) + return ot.rawChangeLoader.Load(commonSnapshot, ot.tree, theirHeads) } func (ot *objectTree) snapshotPathIsActual() bool { diff --git a/commonspace/object/tree/objecttree/objecttreefactory.go b/commonspace/object/tree/objecttree/objecttreefactory.go index 0ab51ec1..5b9196e4 100644 --- a/commonspace/object/tree/objecttree/objecttreefactory.go +++ b/commonspace/object/tree/objecttree/objecttreefactory.go @@ -51,6 +51,22 @@ func verifiableTreeDeps( } } +func emptyDataTreeDeps( + rootChange *treechangeproto.RawTreeChangeWithId, + treeStorage treestorage.TreeStorage, + aclList list.AclList) objectTreeDeps { + changeBuilder := NewEmptyDataBuilder(crypto.NewKeyStorage(), rootChange) + treeBuilder := newTreeBuilder(treeStorage, changeBuilder) + return objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: treeBuilder, + treeStorage: treeStorage, + validator: newTreeValidator(), + rawChangeLoader: newStorageLoader(treeStorage, changeBuilder), + aclList: aclList, + } +} + func nonVerifiableTreeDeps( rootChange *treechangeproto.RawTreeChangeWithId, treeStorage treestorage.TreeStorage, @@ -80,6 +96,49 @@ func DeriveObjectTreeRoot(payload ObjectTreeCreatePayload, aclList list.AclList) return createObjectTreeRoot(payload, 0, nil, aclList) } +func BuildEmptyDataObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) { + rootChange, err := treeStorage.Root() + if err != nil { + return nil, err + } + deps := emptyDataTreeDeps(rootChange, treeStorage, aclList) + return buildObjectTree(deps) +} + +func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { + root, _ := treeStorage.Root() + changeBuilder := &nonVerifiableChangeBuilder{ + ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root), + } + deps := objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: newTreeBuilder(treeStorage, changeBuilder), + treeStorage: treeStorage, + rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), + validator: &noOpTreeValidator{}, + aclList: aclList, + } + + return buildObjectTree(deps) +} + +func BuildEmptyDataTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { + root, _ := treeStorage.Root() + changeBuilder := &nonVerifiableChangeBuilder{ + ChangeBuilder: NewEmptyDataBuilder(newMockKeyStorage(), root), + } + deps := objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: newTreeBuilder(treeStorage, changeBuilder), + treeStorage: treeStorage, + rawChangeLoader: newStorageLoader(treeStorage, changeBuilder), + validator: &noOpTreeValidator{}, + aclList: aclList, + } + + return buildObjectTree(deps) +} + func BuildObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) { rootChange, err := treeStorage.Root() if err != nil { diff --git a/commonspace/object/tree/objecttree/rawloader.go b/commonspace/object/tree/objecttree/rawloader.go index cae32cb3..c9234b5b 100644 --- a/commonspace/object/tree/objecttree/rawloader.go +++ b/commonspace/object/tree/objecttree/rawloader.go @@ -9,8 +9,9 @@ import ( ) type rawChangeLoader struct { - treeStorage treestorage.TreeStorage - changeBuilder ChangeBuilder + treeStorage treestorage.TreeStorage + changeBuilder ChangeBuilder + alwaysFromStorage bool // buffers idStack []string @@ -23,6 +24,12 @@ type rawCacheEntry struct { position int } +func newStorageLoader(treeStorage treestorage.TreeStorage, changeBuilder ChangeBuilder) *rawChangeLoader { + loader := newRawChangeLoader(treeStorage, changeBuilder) + loader.alwaysFromStorage = true + return loader +} + func newRawChangeLoader(treeStorage treestorage.TreeStorage, changeBuilder ChangeBuilder) *rawChangeLoader { return &rawChangeLoader{ treeStorage: treeStorage, @@ -30,7 +37,15 @@ func newRawChangeLoader(treeStorage treestorage.TreeStorage, changeBuilder Chang } } -func (r *rawChangeLoader) LoadFromTree(t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { +func (r *rawChangeLoader) Load(commonSnapshot string, t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { + if commonSnapshot == t.root.Id && !r.alwaysFromStorage { + return r.loadFromTree(t, breakpoints) + } else { + return r.loadFromStorage(commonSnapshot, t.Heads(), breakpoints) + } +} + +func (r *rawChangeLoader) loadFromTree(t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { var stack []*Change for _, h := range t.headIds { stack = append(stack, t.attached[h]) @@ -98,7 +113,7 @@ func (r *rawChangeLoader) LoadFromTree(t *Tree, breakpoints []string) ([]*treech return convert(results) } -func (r *rawChangeLoader) LoadFromStorage(commonSnapshot string, heads, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { +func (r *rawChangeLoader) loadFromStorage(commonSnapshot string, heads, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { // resetting cache r.cache = make(map[string]rawCacheEntry) defer func() { diff --git a/commonspace/object/tree/objecttree/testutils.go b/commonspace/object/tree/objecttree/testutils.go index 6385b9f7..fdcd4a9d 100644 --- a/commonspace/object/tree/objecttree/testutils.go +++ b/commonspace/object/tree/objecttree/testutils.go @@ -2,7 +2,6 @@ package objecttree import ( "fmt" - "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/util/crypto" @@ -112,20 +111,3 @@ func (c *MockChangeCreator) CreateNewTreeStorage(treeId, aclHeadId string) trees treeStorage, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root}) return treeStorage } - -func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { - root, _ := treeStorage.Root() - changeBuilder := &nonVerifiableChangeBuilder{ - ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root), - } - deps := objectTreeDeps{ - changeBuilder: changeBuilder, - treeBuilder: newTreeBuilder(treeStorage, changeBuilder), - treeStorage: treeStorage, - rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), - validator: &noOpTreeValidator{}, - aclList: aclList, - } - - return buildObjectTree(deps) -} diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncprotocol_test.go similarity index 75% rename from commonspace/object/tree/synctree/syncclient_test.go rename to commonspace/object/tree/synctree/syncprotocol_test.go index ed6dd710..4b81b166 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncprotocol_test.go @@ -2,7 +2,6 @@ package synctree import ( "context" - "fmt" "github.com/anytypeio/any-sync/commonspace/object/accountdata" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" @@ -60,7 +59,24 @@ func TestEmptyClientGetsFullHistory(t *testing.T) { require.Len(t, fullResponseMsg.changes, 2) } -func TestRandomTreeMerge(t *testing.T) { +func TestRandomMerge(t *testing.T) { + var ( + rnd = rand.New(rand.NewSource(time.Now().Unix())) + levels = 20 + perLevel = 20 + ) + testTreeMerge(t, levels, perLevel, func() bool { + return true + }) + testTreeMerge(t, levels, perLevel, func() bool { + return false + }) + testTreeMerge(t, levels, perLevel, func() bool { + return rnd.Intn(10) > 8 + }) +} + +func testTreeMerge(t *testing.T, levels, perlevel int, isSnapshot func() bool) { treeId := "treeId" spaceId := "spaceId" keys, err := accountdata.NewRandom() @@ -68,17 +84,15 @@ func TestRandomTreeMerge(t *testing.T) { aclList, err := list.NewTestDerivedAcl(spaceId, keys) storage := createStorage(treeId, aclList) changeCreator := objecttree.NewMockChangeCreator() - rnd := rand.New(rand.NewSource(time.Now().Unix())) params := genParams{ prefix: "peer1", aclId: aclList.Id(), startIdx: 0, - levels: 10, + levels: levels, + perLevel: perlevel, snapshotId: treeId, prevHeads: []string{treeId}, - isSnapshot: func() bool { - return rnd.Intn(100) > 80 - }, + isSnapshot: isSnapshot, } initialRes := genChanges(changeCreator, params) err = storage.TransactionAdd(initialRes.changes, initialRes.heads) @@ -99,20 +113,20 @@ func TestRandomTreeMerge(t *testing.T) { NewHeads: initialRes.heads, RawChanges: initialRes.changes, }) - time.Sleep(1000 * time.Millisecond) + time.Sleep(100 * time.Millisecond) firstHeads := fx.handlers["peer1"].tree().Heads() secondHeads := fx.handlers["peer2"].tree().Heads() require.True(t, slice.UnsortedEquals(firstHeads, secondHeads)) params = genParams{ - prefix: "peer1", - aclId: aclList.Id(), - startIdx: 11, - levels: 10, + prefix: "peer1", + aclId: aclList.Id(), + startIdx: levels, + levels: levels, + perLevel: perlevel, + snapshotId: initialRes.snapshotId, prevHeads: initialRes.heads, - isSnapshot: func() bool { - return rnd.Intn(100) > 80 - }, + isSnapshot: isSnapshot, } peer1Res := genChanges(changeCreator, params) params.prefix = "peer2" @@ -125,10 +139,15 @@ func TestRandomTreeMerge(t *testing.T) { NewHeads: peer2Res.heads, RawChanges: peer2Res.changes, }) - time.Sleep(1000 * time.Millisecond) + time.Sleep(100 * time.Millisecond) fx.stop() - firstHeads = fx.handlers["peer1"].tree().Heads() - secondHeads = fx.handlers["peer2"].tree().Heads() - fmt.Println(firstHeads) - fmt.Println(secondHeads) + firstTree := fx.handlers["peer1"].tree() + secondTree := fx.handlers["peer2"].tree() + firstHeads = firstTree.Heads() + secondHeads = secondTree.Heads() + require.True(t, slice.UnsortedEquals(firstHeads, secondHeads)) + require.True(t, slice.UnsortedEquals(firstHeads, append(peer1Res.heads, peer2Res.heads...))) + firstStorage := firstTree.Storage().(*treestorage.InMemoryTreeStorage) + secondStorage := secondTree.Storage().(*treestorage.InMemoryTreeStorage) + require.True(t, firstStorage.Equal(secondStorage)) } diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index 0399d239..ff2f5c06 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -3,7 +3,6 @@ package synctree import ( "context" "fmt" - "github.com/anytypeio/any-sync/commonspace/object/accountdata" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" @@ -275,7 +274,7 @@ func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage } func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (objecttree.ObjectTree, error) { - return objecttree.BuildTestableTree(aclList, storage) + return objecttree.BuildEmptyDataTestableTree(aclList, storage) } type fixtureDeps struct { @@ -345,6 +344,7 @@ type genParams struct { aclId string startIdx int levels int + perLevel int snapshotId string prevHeads []string isSnapshot func() bool @@ -374,7 +374,7 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge snapshotId = newId continue } - perLevel := rnd.Intn(10) + perLevel := rnd.Intn(params.perLevel) if perLevel == 0 { perLevel = 1 } @@ -383,7 +383,6 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge usedIds = map[string]struct{}{} ) for j := 0; j < perLevel; j++ { - // if we didn't connect with all prev ones prevConns := rnd.Intn(len(prevHeads)) if prevConns == 0 { prevConns = 1 @@ -391,6 +390,7 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge rnd.Shuffle(len(prevHeads), func(i, j int) { prevHeads[i], prevHeads[j] = prevHeads[j], prevHeads[i] }) + // if we didn't connect with all prev ones if j == perLevel-1 && len(usedIds) != len(prevHeads) { var unusedIds []string for _, id := range prevHeads { @@ -417,30 +417,3 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge res.snapshotId = snapshotId return } - -func TestGenChanges(t *testing.T) { - treeId := "treeId" - spaceId := "spaceId" - keys, err := accountdata.NewRandom() - require.NoError(t, err) - aclList, err := list.NewTestDerivedAcl(spaceId, keys) - storage := createStorage(treeId, aclList) - creator := objecttree.NewMockChangeCreator() - rnd := rand.New(rand.NewSource(time.Now().Unix())) - params := genParams{ - prefix: "peerId", - aclId: aclList.Id(), - startIdx: 0, - levels: 10, - snapshotId: treeId, - prevHeads: []string{treeId}, - isSnapshot: func() bool { - return rnd.Intn(100) > 80 - }, - } - res := genChanges(creator, params) - storage.TransactionAdd(res.changes, res.heads) - tr, err := createTestTree(aclList, storage) - require.NoError(t, err) - fmt.Println(tr.Debug(objecttree.NoOpDescriptionParser)) -} diff --git a/commonspace/object/tree/treestorage/inmemory.go b/commonspace/object/tree/treestorage/inmemory.go index 0f22037e..f5b192ba 100644 --- a/commonspace/object/tree/treestorage/inmemory.go +++ b/commonspace/object/tree/treestorage/inmemory.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/util/slice" "sync" ) @@ -105,3 +106,21 @@ func (t *InMemoryTreeStorage) Copy() *InMemoryTreeStorage { other, _ := NewInMemoryTreeStorage(t.root, t.heads, changes) return other.(*InMemoryTreeStorage) } + +func (t *InMemoryTreeStorage) Equal(other *InMemoryTreeStorage) bool { + if !slice.UnsortedEquals(t.heads, other.heads) { + return false + } + if len(t.changes) != len(other.changes) { + return false + } + for k, v := range t.changes { + if otherV, exists := other.changes[k]; exists { + if otherV.Id == v.Id { + continue + } + } + return false + } + return true +} From 8fc1f03f54b5f891c5f67ec318f566beb8c12dcd Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 18 Apr 2023 00:34:14 +0200 Subject: [PATCH 11/45] Change treegetter to treemanager and return BuildObjectTreeFunc --- commonspace/commongetter.go | 12 +- commonspace/headsync/diffsyncer.go | 6 +- commonspace/headsync/diffsyncer_test.go | 4 +- commonspace/headsync/headsync.go | 4 +- .../object/tree/objecttree/changebuilder.go | 2 +- .../tree/objecttree/objecttreefactory.go | 2 + commonspace/object/tree/synctree/synctree.go | 4 +- .../object/tree/synctree/synctreehandler.go | 1 - .../tree/synctree/synctreehandler_test.go | 20 ++- .../mock_treegetter/mock_treegetter.go | 136 ---------------- .../mock_treemanager/mock_treemanager.go | 152 ++++++++++++++++++ .../treemanager.go} | 9 +- commonspace/settings/deleter.go | 6 +- commonspace/settings/deleter_test.go | 12 +- commonspace/settings/deletionmanager.go | 10 +- commonspace/settings/deletionmanager_test.go | 14 +- commonspace/settings/settings.go | 24 +-- commonspace/settings/settings_test.go | 10 +- commonspace/space.go | 30 ++-- commonspace/spaceservice.go | 10 +- 20 files changed, 248 insertions(+), 220 deletions(-) delete mode 100644 commonspace/object/treegetter/mock_treegetter/mock_treegetter.go create mode 100644 commonspace/object/treemanager/mock_treemanager/mock_treemanager.go rename commonspace/object/{treegetter/treegetter.go => treemanager/treemanager.go} (55%) diff --git a/commonspace/commongetter.go b/commonspace/commongetter.go index 6807265b..74356281 100644 --- a/commonspace/commongetter.go +++ b/commonspace/commongetter.go @@ -4,20 +4,20 @@ import ( "context" "github.com/anytypeio/any-sync/commonspace/object/syncobjectgetter" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "sync/atomic" ) type commonGetter struct { - treegetter.TreeGetter + treemanager.TreeManager spaceId string reservedObjects []syncobjectgetter.SyncObject spaceIsClosed *atomic.Bool } -func newCommonGetter(spaceId string, getter treegetter.TreeGetter, spaceIsClosed *atomic.Bool) *commonGetter { +func newCommonGetter(spaceId string, getter treemanager.TreeManager, spaceIsClosed *atomic.Bool) *commonGetter { return &commonGetter{ - TreeGetter: getter, + TreeManager: getter, spaceId: spaceId, spaceIsClosed: spaceIsClosed, } @@ -34,7 +34,7 @@ func (c *commonGetter) GetTree(ctx context.Context, spaceId, treeId string) (obj if obj := c.getReservedObject(treeId); obj != nil { return obj.(objecttree.ObjectTree), nil } - return c.TreeGetter.GetTree(ctx, spaceId, treeId) + return c.TreeManager.GetTree(ctx, spaceId, treeId) } func (c *commonGetter) getReservedObject(id string) syncobjectgetter.SyncObject { @@ -53,7 +53,7 @@ func (c *commonGetter) GetObject(ctx context.Context, objectId string) (obj sync if obj := c.getReservedObject(objectId); obj != nil { return obj, nil } - t, err := c.TreeGetter.GetTree(ctx, c.spaceId, objectId) + t, err := c.TreeManager.GetTree(ctx, c.spaceId, objectId) if err != nil { return } diff --git a/commonspace/headsync/diffsyncer.go b/commonspace/headsync/diffsyncer.go index 9d3434c1..1b8a15dc 100644 --- a/commonspace/headsync/diffsyncer.go +++ b/commonspace/headsync/diffsyncer.go @@ -7,7 +7,7 @@ import ( "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/commonspace/credentialprovider" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" @@ -30,7 +30,7 @@ func newDiffSyncer( spaceId string, diff ldiff.Diff, peerManager peermanager.PeerManager, - cache treegetter.TreeGetter, + cache treemanager.TreeManager, storage spacestorage.SpaceStorage, clientFactory spacesyncproto.ClientFactory, syncStatus syncstatus.StatusUpdater, @@ -53,7 +53,7 @@ type diffSyncer struct { spaceId string diff ldiff.Diff peerManager peermanager.PeerManager - cache treegetter.TreeGetter + cache treemanager.TreeManager storage spacestorage.SpaceStorage clientFactory spacesyncproto.ClientFactory log logger.CtxLogger diff --git a/commonspace/headsync/diffsyncer_test.go b/commonspace/headsync/diffsyncer_test.go index ae40e5a7..e6558b61 100644 --- a/commonspace/headsync/diffsyncer_test.go +++ b/commonspace/headsync/diffsyncer_test.go @@ -12,7 +12,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/acl/liststorage/mock_liststorage" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" mock_treestorage "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage/mock_treestorage" - "github.com/anytypeio/any-sync/commonspace/object/treegetter/mock_treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager/mock_treemanager" "github.com/anytypeio/any-sync/commonspace/peermanager/mock_peermanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate/mock_settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage/mock_spacestorage" @@ -109,7 +109,7 @@ func TestDiffSyncer_Sync(t *testing.T) { diffMock := mock_ldiff.NewMockDiff(ctrl) peerManagerMock := mock_peermanager.NewMockPeerManager(ctrl) - cacheMock := mock_treegetter.NewMockTreeGetter(ctrl) + cacheMock := mock_treemanager.NewMockTreeManager(ctrl) stMock := mock_spacestorage.NewMockSpaceStorage(ctrl) clientMock := mock_spacesyncproto.NewMockDRPCSpaceSyncClient(ctrl) factory := spacesyncproto.ClientFactoryFunc(func(cc drpc.Conn) spacesyncproto.DRPCSpaceSyncClient { diff --git a/commonspace/headsync/headsync.go b/commonspace/headsync/headsync.go index 08d6d9e3..80fe77b8 100644 --- a/commonspace/headsync/headsync.go +++ b/commonspace/headsync/headsync.go @@ -6,7 +6,7 @@ import ( "github.com/anytypeio/any-sync/app/ldiff" "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/commonspace/credentialprovider" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" @@ -59,7 +59,7 @@ func NewHeadSync( configuration nodeconf.Configuration, storage spacestorage.SpaceStorage, peerManager peermanager.PeerManager, - cache treegetter.TreeGetter, + cache treemanager.TreeManager, syncStatus syncstatus.StatusUpdater, credentialProvider credentialprovider.CredentialProvider, log logger.CtxLogger) HeadSync { diff --git a/commonspace/object/tree/objecttree/changebuilder.go b/commonspace/object/tree/objecttree/changebuilder.go index 2f10fca9..b3dc65a1 100644 --- a/commonspace/object/tree/objecttree/changebuilder.go +++ b/commonspace/object/tree/objecttree/changebuilder.go @@ -84,7 +84,7 @@ func NewEmptyDataBuilder(keys crypto.KeyStorage, rootChange *treechangeproto.Raw rootChange: rootChange, keys: keys, newChange: func(id string, identity crypto.PubKey, ch *treechangeproto.TreeChange, signature []byte) *Change { - c := NewChange(id, identity, ch, signature) + c := NewChange(id, identity, ch, nil) c.Data = nil return c }, diff --git a/commonspace/object/tree/objecttree/objecttreefactory.go b/commonspace/object/tree/objecttree/objecttreefactory.go index 5b9196e4..a913d183 100644 --- a/commonspace/object/tree/objecttree/objecttreefactory.go +++ b/commonspace/object/tree/objecttree/objecttreefactory.go @@ -33,6 +33,8 @@ type objectTreeDeps struct { aclList list.AclList } +type BuildObjectTreeFunc = func(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) + var defaultObjectTreeDeps = verifiableTreeDeps func verifiableTreeDeps( diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index a1374b48..00b91846 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -54,7 +54,6 @@ type syncTree struct { var log = logger.NewNamed("common.commonspace.synctree") -var buildObjectTree = objecttree.BuildObjectTree var createSyncClient = newSyncClient type ResponsiblePeersGetter interface { @@ -73,6 +72,7 @@ type BuildDeps struct { OnClose func(id string) SyncStatus syncstatus.StatusUpdater PeerGetter ResponsiblePeersGetter + BuildObjectTree objecttree.BuildObjectTreeFunc WaitTreeRemoteSync bool } @@ -94,7 +94,7 @@ func PutSyncTree(ctx context.Context, payload treestorage.TreeStorageCreatePaylo } func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t SyncTree, err error) { - objTree, err := buildObjectTree(deps.TreeStorage, deps.AclList) + objTree, err := deps.BuildObjectTree(deps.TreeStorage, deps.AclList) if err != nil { return } diff --git a/commonspace/object/tree/synctree/synctreehandler.go b/commonspace/object/tree/synctree/synctreehandler.go index a41a2bad..63b708d9 100644 --- a/commonspace/object/tree/synctree/synctreehandler.go +++ b/commonspace/object/tree/synctree/synctreehandler.go @@ -82,7 +82,6 @@ func (s *syncTreeHandler) handleHeadUpdate( isEmptyUpdate = len(update.Changes) == 0 objTree = s.objTree ) - log := log.With( zap.Strings("update heads", update.Heads), zap.String("treeId", objTree.Id()), diff --git a/commonspace/object/tree/synctree/synctreehandler_test.go b/commonspace/object/tree/synctree/synctreehandler_test.go index 052479d2..f655b34a 100644 --- a/commonspace/object/tree/synctree/synctreehandler_test.go +++ b/commonspace/object/tree/synctree/synctreehandler_test.go @@ -89,6 +89,13 @@ func (fx *syncHandlerFixture) stop() { func TestSyncHandler_HandleHeadUpdate(t *testing.T) { ctx := context.Background() log = logger.CtxLogger{Logger: zap.NewNop()} + fullRequest := &treechangeproto.TreeSyncMessage{ + Content: &treechangeproto.TreeSyncContentValue{ + Value: &treechangeproto.TreeSyncContentValue_FullSyncRequest{ + FullSyncRequest: &treechangeproto.TreeFullSyncRequest{}, + }, + }, + } t.Run("head update non empty all heads added", func(t *testing.T) { fx := newSyncHandlerFixture(t) @@ -113,7 +120,6 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { RawChanges: []*treechangeproto.RawTreeChangeWithId{chWithId}, })). Return(objecttree.AddResult{}, nil) - fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2", "h1"}) fx.objectTreeMock.EXPECT().HasChanges(gomock.Eq([]string{"h1"})).Return(true) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) @@ -133,7 +139,6 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") - fullRequest := &treechangeproto.TreeSyncMessage{} fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2"}).AnyTimes() @@ -188,7 +193,6 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") - fullRequest := &treechangeproto.TreeSyncMessage{} fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2"}).AnyTimes() @@ -226,6 +230,13 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { ctx := context.Background() log = logger.CtxLogger{Logger: zap.NewNop()} + fullResponse := &treechangeproto.TreeSyncMessage{ + Content: &treechangeproto.TreeSyncContentValue{ + Value: &treechangeproto.TreeSyncContentValue_FullSyncResponse{ + FullSyncResponse: &treechangeproto.TreeFullSyncResponse{}, + }, + }, + } t.Run("full sync request with change", func(t *testing.T) { fx := newSyncHandlerFixture(t) @@ -240,7 +251,6 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") - fullResponse := &treechangeproto.TreeSyncMessage{} fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Header().Return(nil) @@ -274,7 +284,6 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") - fullResponse := &treechangeproto.TreeSyncMessage{} fx.objectTreeMock.EXPECT(). Id().AnyTimes().Return(treeId) @@ -305,7 +314,6 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") objectMsg.RequestId = replyId - fullResponse := &treechangeproto.TreeSyncMessage{} fx.objectTreeMock.EXPECT(). Id().AnyTimes().Return(treeId) diff --git a/commonspace/object/treegetter/mock_treegetter/mock_treegetter.go b/commonspace/object/treegetter/mock_treegetter/mock_treegetter.go deleted file mode 100644 index 6aee2208..00000000 --- a/commonspace/object/treegetter/mock_treegetter/mock_treegetter.go +++ /dev/null @@ -1,136 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/anytypeio/any-sync/commonspace/object/treegetter (interfaces: TreeGetter) - -// Package mock_treegetter is a generated GoMock package. -package mock_treegetter - -import ( - context "context" - reflect "reflect" - - app "github.com/anytypeio/any-sync/app" - objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" - gomock "github.com/golang/mock/gomock" -) - -// MockTreeGetter is a mock of TreeGetter interface. -type MockTreeGetter struct { - ctrl *gomock.Controller - recorder *MockTreeGetterMockRecorder -} - -// MockTreeGetterMockRecorder is the mock recorder for MockTreeGetter. -type MockTreeGetterMockRecorder struct { - mock *MockTreeGetter -} - -// NewMockTreeGetter creates a new mock instance. -func NewMockTreeGetter(ctrl *gomock.Controller) *MockTreeGetter { - mock := &MockTreeGetter{ctrl: ctrl} - mock.recorder = &MockTreeGetterMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockTreeGetter) EXPECT() *MockTreeGetterMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MockTreeGetter) Close(arg0 context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MockTreeGetterMockRecorder) Close(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTreeGetter)(nil).Close), arg0) -} - -// DeleteSpace mocks base method. -func (m *MockTreeGetter) DeleteSpace(arg0 context.Context, arg1 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteSpace", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteSpace indicates an expected call of DeleteSpace. -func (mr *MockTreeGetterMockRecorder) DeleteSpace(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSpace", reflect.TypeOf((*MockTreeGetter)(nil).DeleteSpace), arg0, arg1) -} - -// DeleteTree mocks base method. -func (m *MockTreeGetter) DeleteTree(arg0 context.Context, arg1, arg2 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteTree", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteTree indicates an expected call of DeleteTree. -func (mr *MockTreeGetterMockRecorder) DeleteTree(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteTree", reflect.TypeOf((*MockTreeGetter)(nil).DeleteTree), arg0, arg1, arg2) -} - -// GetTree mocks base method. -func (m *MockTreeGetter) GetTree(arg0 context.Context, arg1, arg2 string) (objecttree.ObjectTree, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTree", arg0, arg1, arg2) - ret0, _ := ret[0].(objecttree.ObjectTree) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTree indicates an expected call of GetTree. -func (mr *MockTreeGetterMockRecorder) GetTree(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTree", reflect.TypeOf((*MockTreeGetter)(nil).GetTree), arg0, arg1, arg2) -} - -// Init mocks base method. -func (m *MockTreeGetter) Init(arg0 *app.App) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Init", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Init indicates an expected call of Init. -func (mr *MockTreeGetterMockRecorder) Init(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockTreeGetter)(nil).Init), arg0) -} - -// Name mocks base method. -func (m *MockTreeGetter) Name() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Name") - ret0, _ := ret[0].(string) - return ret0 -} - -// Name indicates an expected call of Name. -func (mr *MockTreeGetterMockRecorder) Name() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockTreeGetter)(nil).Name)) -} - -// Run mocks base method. -func (m *MockTreeGetter) Run(arg0 context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Run", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Run indicates an expected call of Run. -func (mr *MockTreeGetterMockRecorder) Run(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockTreeGetter)(nil).Run), arg0) -} diff --git a/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go b/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go new file mode 100644 index 00000000..f378bc43 --- /dev/null +++ b/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go @@ -0,0 +1,152 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/anytypeio/any-sync/commonspace/object/treemanager (interfaces: TreeManager) + +// Package mock_treemanager is a generated GoMock package. +package mock_treemanager + +import ( + context "context" + reflect "reflect" + + app "github.com/anytypeio/any-sync/app" + list "github.com/anytypeio/any-sync/commonspace/object/acl/list" + objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + treestorage "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + gomock "github.com/golang/mock/gomock" +) + +// MockTreeManager is a mock of TreeManager interface. +type MockTreeManager struct { + ctrl *gomock.Controller + recorder *MockTreeManagerMockRecorder +} + +// MockTreeManagerMockRecorder is the mock recorder for MockTreeManager. +type MockTreeManagerMockRecorder struct { + mock *MockTreeManager +} + +// NewMockTreeManager creates a new mock instance. +func NewMockTreeManager(ctrl *gomock.Controller) *MockTreeManager { + mock := &MockTreeManager{ctrl: ctrl} + mock.recorder = &MockTreeManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockTreeManager) EXPECT() *MockTreeManagerMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockTreeManager) Close(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockTreeManagerMockRecorder) Close(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTreeManager)(nil).Close), arg0) +} + +// DeleteSpace mocks base method. +func (m *MockTreeManager) DeleteSpace(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteSpace", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteSpace indicates an expected call of DeleteSpace. +func (mr *MockTreeManagerMockRecorder) DeleteSpace(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSpace", reflect.TypeOf((*MockTreeManager)(nil).DeleteSpace), arg0, arg1) +} + +// DeleteTree mocks base method. +func (m *MockTreeManager) DeleteTree(arg0 context.Context, arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteTree", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteTree indicates an expected call of DeleteTree. +func (mr *MockTreeManagerMockRecorder) DeleteTree(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteTree", reflect.TypeOf((*MockTreeManager)(nil).DeleteTree), arg0, arg1, arg2) +} + +// GetTree mocks base method. +func (m *MockTreeManager) GetTree(arg0 context.Context, arg1, arg2 string) (objecttree.ObjectTree, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTree", arg0, arg1, arg2) + ret0, _ := ret[0].(objecttree.ObjectTree) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTree indicates an expected call of GetTree. +func (mr *MockTreeManagerMockRecorder) GetTree(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTree", reflect.TypeOf((*MockTreeManager)(nil).GetTree), arg0, arg1, arg2) +} + +// Init mocks base method. +func (m *MockTreeManager) Init(arg0 *app.App) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Init", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Init indicates an expected call of Init. +func (mr *MockTreeManagerMockRecorder) Init(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockTreeManager)(nil).Init), arg0) +} + +// Name mocks base method. +func (m *MockTreeManager) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockTreeManagerMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockTreeManager)(nil).Name)) +} + +// ObjectTreeBuilder mocks base method. +func (m *MockTreeManager) ObjectTreeBuilder() func(treestorage.TreeStorage, list.AclList) (objecttree.ObjectTree, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ObjectTreeBuilder") + ret0, _ := ret[0].(func(treestorage.TreeStorage, list.AclList) (objecttree.ObjectTree, error)) + return ret0 +} + +// ObjectTreeBuilder indicates an expected call of ObjectTreeBuilder. +func (mr *MockTreeManagerMockRecorder) ObjectTreeBuilder() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ObjectTreeBuilder", reflect.TypeOf((*MockTreeManager)(nil).ObjectTreeBuilder)) +} + +// Run mocks base method. +func (m *MockTreeManager) Run(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Run", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Run indicates an expected call of Run. +func (mr *MockTreeManagerMockRecorder) Run(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockTreeManager)(nil).Run), arg0) +} diff --git a/commonspace/object/treegetter/treegetter.go b/commonspace/object/treemanager/treemanager.go similarity index 55% rename from commonspace/object/treegetter/treegetter.go rename to commonspace/object/treemanager/treemanager.go index 4e604d10..00685ff5 100644 --- a/commonspace/object/treegetter/treegetter.go +++ b/commonspace/object/treemanager/treemanager.go @@ -1,5 +1,5 @@ -//go:generate mockgen -destination mock_treegetter/mock_treegetter.go github.com/anytypeio/any-sync/commonspace/object/treegetter TreeGetter -package treegetter +//go:generate mockgen -destination mock_treemanager/mock_treemanager.go github.com/anytypeio/any-sync/commonspace/object/treemanager TreeManager +package treemanager import ( "context" @@ -7,11 +7,12 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" ) -const CName = "common.object.treegetter" +const CName = "common.object.treemanager" -type TreeGetter interface { +type TreeManager interface { app.ComponentRunnable GetTree(ctx context.Context, spaceId, treeId string) (objecttree.ObjectTree, error) DeleteTree(ctx context.Context, spaceId, treeId string) error DeleteSpace(ctx context.Context, spaceId string) error + ObjectTreeBuilder() objecttree.BuildObjectTreeFunc } diff --git a/commonspace/settings/deleter.go b/commonspace/settings/deleter.go index 55f026c1..c50da268 100644 --- a/commonspace/settings/deleter.go +++ b/commonspace/settings/deleter.go @@ -2,7 +2,7 @@ package settings import ( "context" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" "go.uber.org/zap" @@ -15,10 +15,10 @@ type Deleter interface { type deleter struct { st spacestorage.SpaceStorage state settingsstate.ObjectDeletionState - getter treegetter.TreeGetter + getter treemanager.TreeManager } -func newDeleter(st spacestorage.SpaceStorage, state settingsstate.ObjectDeletionState, getter treegetter.TreeGetter) Deleter { +func newDeleter(st spacestorage.SpaceStorage, state settingsstate.ObjectDeletionState, getter treemanager.TreeManager) Deleter { return &deleter{st, state, getter} } diff --git a/commonspace/settings/deleter_test.go b/commonspace/settings/deleter_test.go index 3606225b..fe6f7c51 100644 --- a/commonspace/settings/deleter_test.go +++ b/commonspace/settings/deleter_test.go @@ -2,7 +2,7 @@ package settings import ( "fmt" - "github.com/anytypeio/any-sync/commonspace/object/treegetter/mock_treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager/mock_treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate/mock_settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/commonspace/spacestorage/mock_spacestorage" @@ -12,18 +12,18 @@ import ( func TestDeleter_Delete(t *testing.T) { ctrl := gomock.NewController(t) - treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) + treeManager := mock_treemanager.NewMockTreeManager(ctrl) st := mock_spacestorage.NewMockSpaceStorage(ctrl) delState := mock_settingsstate.NewMockObjectDeletionState(ctrl) - deleter := newDeleter(st, delState, treeGetter) + deleter := newDeleter(st, delState, treeManager) t.Run("deleter delete queued", func(t *testing.T) { id := "id" spaceId := "spaceId" delState.EXPECT().GetQueued().Return([]string{id}) st.EXPECT().Id().Return(spaceId) - treeGetter.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(nil) + treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(nil) delState.EXPECT().Delete(id).Return(nil) deleter.Delete() @@ -34,7 +34,7 @@ func TestDeleter_Delete(t *testing.T) { spaceId := "spaceId" delState.EXPECT().GetQueued().Return([]string{id}) st.EXPECT().Id().Return(spaceId) - treeGetter.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(spacestorage.ErrTreeStorageAlreadyDeleted) + treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(spacestorage.ErrTreeStorageAlreadyDeleted) delState.EXPECT().Delete(id).Return(nil) deleter.Delete() @@ -45,7 +45,7 @@ func TestDeleter_Delete(t *testing.T) { spaceId := "spaceId" delState.EXPECT().GetQueued().Return([]string{id}) st.EXPECT().Id().Return(spaceId) - treeGetter.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(fmt.Errorf("some error")) + treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(fmt.Errorf("some error")) deleter.Delete() }) diff --git a/commonspace/settings/deletionmanager.go b/commonspace/settings/deletionmanager.go index 8bfc15e3..366c2686 100644 --- a/commonspace/settings/deletionmanager.go +++ b/commonspace/settings/deletionmanager.go @@ -2,7 +2,7 @@ package settings import ( "context" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/util/slice" "go.uber.org/zap" @@ -20,12 +20,12 @@ func newDeletionManager( spaceId string, settingsId string, isResponsible bool, - treeGetter treegetter.TreeGetter, + treeManager treemanager.TreeManager, deletionState settingsstate.ObjectDeletionState, provider SpaceIdsProvider, onSpaceDelete func()) DeletionManager { return &deletionManager{ - treeGetter: treeGetter, + treeManager: treeManager, isResponsible: isResponsible, spaceId: spaceId, settingsId: settingsId, @@ -38,7 +38,7 @@ func newDeletionManager( type deletionManager struct { deletionState settingsstate.ObjectDeletionState provider SpaceIdsProvider - treeGetter treegetter.TreeGetter + treeManager treemanager.TreeManager spaceId string settingsId string isResponsible bool @@ -55,7 +55,7 @@ func (d *deletionManager) UpdateState(ctx context.Context, state *settingsstate. return nil } log.Debug("deleting space") - err = d.treeGetter.DeleteSpace(ctx, d.spaceId) + err = d.treeManager.DeleteSpace(ctx, d.spaceId) if err != nil { log.Debug("failed to notify on space deletion", zap.Error(err)) } diff --git a/commonspace/settings/deletionmanager_test.go b/commonspace/settings/deletionmanager_test.go index a4992903..990efc07 100644 --- a/commonspace/settings/deletionmanager_test.go +++ b/commonspace/settings/deletionmanager_test.go @@ -2,7 +2,7 @@ package settings import ( "context" - "github.com/anytypeio/any-sync/commonspace/object/treegetter/mock_treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager/mock_treemanager" "github.com/anytypeio/any-sync/commonspace/settings/mock_settings" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate/mock_settingsstate" @@ -27,15 +27,15 @@ func TestDeletionManager_UpdateState_NotResponsible(t *testing.T) { deleted = true } delState := mock_settingsstate.NewMockObjectDeletionState(ctrl) - treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) + treeManager := mock_treemanager.NewMockTreeManager(ctrl) delState.EXPECT().Add(state.DeletedIds).Return(nil) - treeGetter.EXPECT().DeleteSpace(ctx, spaceId).Return(nil) + treeManager.EXPECT().DeleteSpace(ctx, spaceId).Return(nil) delManager := newDeletionManager(spaceId, settingsId, false, - treeGetter, + treeManager, delState, nil, onDeleted) @@ -60,17 +60,17 @@ func TestDeletionManager_UpdateState_Responsible(t *testing.T) { deleted = true } delState := mock_settingsstate.NewMockObjectDeletionState(ctrl) - treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) + treeManager := mock_treemanager.NewMockTreeManager(ctrl) provider := mock_settings.NewMockSpaceIdsProvider(ctrl) delState.EXPECT().Add(state.DeletedIds).Return(nil) - treeGetter.EXPECT().DeleteSpace(ctx, spaceId).Return(nil) + treeManager.EXPECT().DeleteSpace(ctx, spaceId).Return(nil) provider.EXPECT().AllIds().Return([]string{"id", "otherId", settingsId}) delState.EXPECT().Add([]string{"id", "otherId"}).Return(nil) delManager := newDeletionManager(spaceId, settingsId, true, - treeGetter, + treeManager, delState, provider, onDeleted) diff --git a/commonspace/settings/settings.go b/commonspace/settings/settings.go index 1455faff..14430e0f 100644 --- a/commonspace/settings/settings.go +++ b/commonspace/settings/settings.go @@ -13,7 +13,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" @@ -47,7 +47,7 @@ type BuildTreeFunc func(ctx context.Context, id string, listener updatelistener. type Deps struct { BuildFunc BuildTreeFunc Account accountservice.Service - TreeGetter treegetter.TreeGetter + TreeManager treemanager.TreeManager Store spacestorage.SpaceStorage Configuration nodeconf.Configuration DeletionState settingsstate.ObjectDeletionState @@ -62,13 +62,13 @@ type Deps struct { type settingsObject struct { synctree.SyncTree - account accountservice.Service - spaceId string - treeGetter treegetter.TreeGetter - store spacestorage.SpaceStorage - builder settingsstate.StateBuilder - buildFunc BuildTreeFunc - loop *deleteLoop + account accountservice.Service + spaceId string + treeManager treemanager.TreeManager + store spacestorage.SpaceStorage + builder settingsstate.StateBuilder + buildFunc BuildTreeFunc + loop *deleteLoop state *settingsstate.State deletionState settingsstate.ObjectDeletionState @@ -84,7 +84,7 @@ func NewSettingsObject(deps Deps, spaceId string) (obj SettingsObject) { changeFactory settingsstate.ChangeFactory ) if deps.del == nil { - deleter = newDeleter(deps.Store, deps.DeletionState, deps.TreeGetter) + deleter = newDeleter(deps.Store, deps.DeletionState, deps.TreeManager) } else { deleter = deps.del } @@ -93,7 +93,7 @@ func NewSettingsObject(deps Deps, spaceId string) (obj SettingsObject) { spaceId, deps.Store.SpaceSettingsId(), deps.Configuration.IsResponsible(spaceId), - deps.TreeGetter, + deps.TreeManager, deps.DeletionState, deps.Provider, deps.OnSpaceDelete) @@ -123,7 +123,7 @@ func NewSettingsObject(deps Deps, spaceId string) (obj SettingsObject) { spaceId: spaceId, account: deps.Account, deletionState: deps.DeletionState, - treeGetter: deps.TreeGetter, + treeManager: deps.TreeManager, store: deps.Store, buildFunc: deps.BuildFunc, builder: builder, diff --git a/commonspace/settings/settings_test.go b/commonspace/settings/settings_test.go index 166cf284..b4879f53 100644 --- a/commonspace/settings/settings_test.go +++ b/commonspace/settings/settings_test.go @@ -9,7 +9,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/mock_synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/object/treegetter/mock_treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager/mock_treemanager" "github.com/anytypeio/any-sync/commonspace/settings/mock_settings" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate" "github.com/anytypeio/any-sync/commonspace/settings/settingsstate/mock_settingsstate" @@ -45,7 +45,7 @@ type settingsFixture struct { docId string doc *settingsObject ctrl *gomock.Controller - treeGetter *mock_treegetter.MockTreeGetter + treeManager *mock_treemanager.MockTreeManager spaceStorage *mock_spacestorage.MockSpaceStorage stateBuilder *mock_settingsstate.MockStateBuilder deletionManager *mock_settings.MockDeletionManager @@ -62,7 +62,7 @@ func newSettingsFixture(t *testing.T) *settingsFixture { ctrl := gomock.NewController(t) acc := mock_accountservice.NewMockService(ctrl) - treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) + treeManager := mock_treemanager.NewMockTreeManager(ctrl) st := mock_spacestorage.NewMockSpaceStorage(ctrl) delState := mock_settingsstate.NewMockObjectDeletionState(ctrl) delManager := mock_settings.NewMockDeletionManager(ctrl) @@ -81,7 +81,7 @@ func newSettingsFixture(t *testing.T) *settingsFixture { deps := Deps{ BuildFunc: buildFunc, Account: acc, - TreeGetter: treeGetter, + TreeManager: treeManager, Store: st, DeletionState: delState, delManager: delManager, @@ -95,7 +95,7 @@ func newSettingsFixture(t *testing.T) *settingsFixture { docId: objectId, doc: doc, ctrl: ctrl, - treeGetter: treeGetter, + treeManager: treeManager, spaceStorage: st, stateBuilder: stateBuilder, changeFactory: changeFactory, diff --git a/commonspace/space.go b/commonspace/space.go index 8b4375fb..c68ff8e3 100644 --- a/commonspace/space.go +++ b/commonspace/space.go @@ -118,7 +118,7 @@ type space struct { headSync headsync.HeadSync syncStatus syncstatus.StatusUpdater storage spacestorage.SpaceStorage - cache *commonGetter + treeManager *commonGetter account accountservice.Service aclList *syncacl.SyncAcl configuration nodeconf.Configuration @@ -179,7 +179,7 @@ func (s *space) Init(ctx context.Context) (err error) { return } s.aclList = syncacl.NewSyncAcl(aclList, s.objectSync.MessagePool()) - s.cache.AddObject(s.aclList) + s.treeManager.AddObject(s.aclList) deletionState := settingsstate.NewObjectDeletionState(s.storage) deps := settings.Deps{ @@ -196,7 +196,7 @@ func (s *space) Init(ctx context.Context) (err error) { return }, Account: s.account, - TreeGetter: s.cache, + TreeManager: s.treeManager, Store: s.storage, DeletionState: deletionState, Provider: s.headSync, @@ -210,7 +210,7 @@ func (s *space) Init(ctx context.Context) (err error) { if err != nil { return } - s.cache.AddObject(s.settingsObject) + s.treeManager.AddObject(s.settingsObject) s.syncStatus.Run() s.handleQueue = multiqueue.New[HandleMessage](s.handleMessage, 100) return nil @@ -283,16 +283,17 @@ func (s *space) PutTree(ctx context.Context, payload treestorage.TreeStorageCrea return } deps := synctree.BuildDeps{ - SpaceId: s.id, - ObjectSync: s.objectSync, - Configuration: s.configuration, - HeadNotifiable: s.headSync, - Listener: listener, - AclList: s.aclList, - SpaceStorage: s.storage, - OnClose: s.onObjectClose, - SyncStatus: s.syncStatus, - PeerGetter: s.peerManager, + SpaceId: s.id, + ObjectSync: s.objectSync, + Configuration: s.configuration, + HeadNotifiable: s.headSync, + Listener: listener, + AclList: s.aclList, + SpaceStorage: s.storage, + OnClose: s.onObjectClose, + SyncStatus: s.syncStatus, + PeerGetter: s.peerManager, + BuildObjectTree: s.treeManager.ObjectTreeBuilder(), } t, err = synctree.PutSyncTree(ctx, payload, deps) if err != nil { @@ -331,6 +332,7 @@ func (s *space) BuildTree(ctx context.Context, id string, opts BuildTreeOpts) (t SyncStatus: s.syncStatus, WaitTreeRemoteSync: opts.WaitTreeRemoteSync, PeerGetter: s.peerManager, + BuildObjectTree: s.treeManager.ObjectTreeBuilder(), } if t, err = synctree.BuildSyncTreeOrGetRemote(ctx, id, deps); err != nil { return nil, err diff --git a/commonspace/spaceservice.go b/commonspace/spaceservice.go index cf4a2710..cfe33692 100644 --- a/commonspace/spaceservice.go +++ b/commonspace/spaceservice.go @@ -9,7 +9,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/headsync" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/objectsync" "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/spacestorage" @@ -49,7 +49,7 @@ type spaceService struct { storageProvider spacestorage.SpaceStorageProvider peermanagerProvider peermanager.PeerManagerProvider credentialProvider credentialprovider.CredentialProvider - treeGetter treegetter.TreeGetter + treeManager treemanager.TreeManager pool pool.Pool } @@ -58,7 +58,7 @@ func (s *spaceService) Init(a *app.App) (err error) { s.account = a.MustComponent(accountservice.CName).(accountservice.Service) s.storageProvider = a.MustComponent(spacestorage.CName).(spacestorage.SpaceStorageProvider) s.configurationService = a.MustComponent(nodeconf.CName).(nodeconf.Service) - s.treeGetter = a.MustComponent(treegetter.CName).(treegetter.TreeGetter) + s.treeManager = a.MustComponent(treemanager.CName).(treemanager.TreeManager) s.peermanagerProvider = a.MustComponent(peermanager.CName).(peermanager.PeerManagerProvider) credProvider := a.Component(credentialprovider.CName) if credProvider != nil { @@ -145,7 +145,7 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) { return nil, err } spaceIsDeleted.Swap(isDeleted) - getter := newCommonGetter(st.Id(), s.treeGetter, spaceIsClosed) + getter := newCommonGetter(st.Id(), s.treeManager, spaceIsClosed) syncStatus := syncstatus.NewNoOpSyncStatus() // this will work only for clients, not the best solution, but... if !lastConfiguration.IsResponsible(st.Id()) { @@ -165,7 +165,7 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) { objectSync: objectSync, headSync: headSync, syncStatus: syncStatus, - cache: getter, + treeManager: getter, account: s.account, configuration: lastConfiguration, peerManager: peerManager, From 99bad9ce9d5e0ad53e511ece48b59bc1d26026ee Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 18 Apr 2023 12:09:25 +0200 Subject: [PATCH 12/45] Send error response on sync (wip) --- .../object/tree/synctree/syncclient.go | 2 +- commonspace/object/tree/synctree/synctree.go | 2 +- .../object/tree/synctree/treeremotegetter.go | 10 +- .../object/tree/synctree/utils_test.go | 4 +- .../treechangeproto/protos/treechange.proto | 2 +- .../tree/treechangeproto/treechange.pb.go | 130 ++++++++---------- commonspace/objectsync/objectsync.go | 51 ++++--- commonspace/space.go | 1 - 8 files changed, 94 insertions(+), 108 deletions(-) diff --git a/commonspace/object/tree/synctree/syncclient.go b/commonspace/object/tree/synctree/syncclient.go index f6e6b990..5967441b 100644 --- a/commonspace/object/tree/synctree/syncclient.go +++ b/commonspace/object/tree/synctree/syncclient.go @@ -20,7 +20,7 @@ type syncClient struct { spaceId string } -func newSyncClient( +func NewSyncClient( spaceId string, peerManager peermanager.PeerManager, factory RequestFactory) SyncClient { diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index 00b91846..89787597 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -54,7 +54,7 @@ type syncTree struct { var log = logger.NewNamed("common.commonspace.synctree") -var createSyncClient = newSyncClient +var createSyncClient = NewSyncClient type ResponsiblePeersGetter interface { GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) diff --git a/commonspace/object/tree/synctree/treeremotegetter.go b/commonspace/object/tree/synctree/treeremotegetter.go index 561d850c..9fd0677a 100644 --- a/commonspace/object/tree/synctree/treeremotegetter.go +++ b/commonspace/object/tree/synctree/treeremotegetter.go @@ -8,6 +8,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/net/peer" + "github.com/anytypeio/any-sync/net/rpc/rpcerr" "github.com/gogo/protobuf/proto" "go.uber.org/zap" "time" @@ -117,9 +118,16 @@ func (t treeRemoteGetter) getTree(ctx context.Context) (treeStorage treestorage. if err != nil { return } - if resp.GetContent().GetFullSyncResponse() == nil { + switch { + case resp.GetContent().GetErrorResponse() != nil: + errResp := resp.GetContent().GetErrorResponse() + err = rpcerr.Err(errResp.ErrCode) + return + case resp.GetContent().GetFullSyncResponse() == nil: err = fmt.Errorf("expected to get full sync response, but got something else") return + default: + break } fullSyncResp := resp.GetContent().GetFullSyncResponse() diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index ff2f5c06..524ab1d2 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -244,7 +244,7 @@ func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.Ra func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *processSyncHandler { factory := GetRequestFactory() - syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) + syncClient := NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) netTree := &broadcastTree{ ObjectTree: objTree, SyncClient: syncClient, @@ -255,7 +255,7 @@ func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, lo func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *processSyncHandler { factory := GetRequestFactory() - syncClient := newSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) + syncClient := NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) batcher := mb.New[processMsg](0) return &processSyncHandler{ diff --git a/commonspace/object/tree/treechangeproto/protos/treechange.proto b/commonspace/object/tree/treechangeproto/protos/treechange.proto index dfc74a42..1ecd9bd6 100644 --- a/commonspace/object/tree/treechangeproto/protos/treechange.proto +++ b/commonspace/object/tree/treechangeproto/protos/treechange.proto @@ -94,7 +94,7 @@ message TreeFullSyncResponse { // TreeErrorResponse is an error sent as a response for a full sync request message TreeErrorResponse { - string error = 1; + uint64 errCode = 1; } // TreeChangeInfo is used internally in Tree implementation for ease of marshalling diff --git a/commonspace/object/tree/treechangeproto/treechange.pb.go b/commonspace/object/tree/treechangeproto/treechange.pb.go index 60eb619b..14815df1 100644 --- a/commonspace/object/tree/treechangeproto/treechange.pb.go +++ b/commonspace/object/tree/treechangeproto/treechange.pb.go @@ -691,7 +691,7 @@ func (m *TreeFullSyncResponse) GetSnapshotPath() []string { // TreeErrorResponse is an error sent as a response for a full sync request type TreeErrorResponse struct { - Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` + ErrCode uint64 `protobuf:"varint,1,opt,name=errCode,proto3" json:"errCode,omitempty"` } func (m *TreeErrorResponse) Reset() { *m = TreeErrorResponse{} } @@ -727,11 +727,11 @@ func (m *TreeErrorResponse) XXX_DiscardUnknown() { var xxx_messageInfo_TreeErrorResponse proto.InternalMessageInfo -func (m *TreeErrorResponse) GetError() string { +func (m *TreeErrorResponse) GetErrCode() uint64 { if m != nil { - return m.Error + return m.ErrCode } - return "" + return 0 } // TreeChangeInfo is used internally in Tree implementation for ease of marshalling @@ -806,50 +806,50 @@ func init() { } var fileDescriptor_5033f0301ef9b772 = []byte{ - // 677 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0xcf, 0x4f, 0xd4, 0x40, - 0x14, 0xee, 0x74, 0x81, 0xb2, 0x8f, 0x05, 0x74, 0xe0, 0xd0, 0x10, 0xad, 0x4d, 0x63, 0x74, 0xbd, - 0x40, 0x82, 0x27, 0x8d, 0x09, 0x11, 0x04, 0x77, 0x43, 0x34, 0x64, 0x40, 0x4c, 0xbc, 0x0d, 0xed, - 0xc0, 0xd6, 0xec, 0x76, 0x6a, 0x67, 0x56, 0xb2, 0x7f, 0x80, 0x17, 0x4d, 0x88, 0xff, 0x92, 0x37, - 0x8f, 0x1c, 0x39, 0x1a, 0xf6, 0x1f, 0x31, 0x9d, 0x69, 0xb7, 0x3f, 0x76, 0x0f, 0xdc, 0xb8, 0x74, - 0xf7, 0x7d, 0x7d, 0xef, 0x7b, 0xdf, 0xfb, 0xe6, 0x47, 0x61, 0xc7, 0xe7, 0x83, 0x01, 0x8f, 0x44, - 0x4c, 0x7d, 0xb6, 0xc5, 0xcf, 0xbe, 0x32, 0x5f, 0x6e, 0xc9, 0x84, 0x31, 0xf5, 0xf0, 0x7b, 0x34, - 0xba, 0x60, 0x71, 0xc2, 0x25, 0xdf, 0x52, 0x4f, 0x51, 0x82, 0x37, 0x15, 0x82, 0xa1, 0x40, 0xbc, - 0x1b, 0x04, 0x40, 0x38, 0x97, 0x7b, 0x2a, 0xc4, 0x8f, 0xa0, 0x49, 0xfd, 0x7e, 0x87, 0xd1, 0xa0, - 0x1b, 0xd8, 0xc8, 0x45, 0xed, 0x26, 0x29, 0x00, 0x6c, 0x83, 0xa5, 0xba, 0x76, 0x03, 0xdb, 0x54, - 0xef, 0xf2, 0x10, 0x3b, 0x00, 0x9a, 0xf0, 0x64, 0x14, 0x33, 0xbb, 0xa1, 0x5e, 0x96, 0x90, 0x94, - 0x57, 0x86, 0x03, 0x26, 0x24, 0x1d, 0xc4, 0xf6, 0x9c, 0x8b, 0xda, 0x0d, 0x52, 0x00, 0x18, 0xc3, - 0x9c, 0x60, 0x2c, 0xb0, 0xe7, 0x5d, 0xd4, 0x6e, 0x11, 0xf5, 0x1f, 0x6f, 0xc0, 0x62, 0x18, 0xb0, - 0x48, 0x86, 0x72, 0x64, 0x2f, 0x28, 0x7c, 0x12, 0xe3, 0xa7, 0xb0, 0xac, 0xb9, 0x8f, 0xe8, 0xa8, - 0xcf, 0x69, 0x60, 0x5b, 0x2a, 0xa1, 0x0a, 0x7a, 0x57, 0x26, 0xc0, 0x49, 0xc2, 0x58, 0x36, 0x9a, - 0x0b, 0x4b, 0xe9, 0xdc, 0x7a, 0x14, 0x61, 0x23, 0xb7, 0xd1, 0x6e, 0x92, 0x32, 0x54, 0x1d, 0xde, - 0xac, 0x0f, 0xff, 0x0c, 0x56, 0x44, 0x44, 0x63, 0xd1, 0xe3, 0x72, 0x97, 0x8a, 0xd4, 0x03, 0x3d, - 0x66, 0x0d, 0x4d, 0xfb, 0x68, 0x1d, 0xe2, 0x1d, 0x95, 0x54, 0x0d, 0xdb, 0x22, 0x65, 0x28, 0xed, - 0x93, 0x30, 0x1a, 0x1c, 0xb2, 0x51, 0x57, 0xcf, 0xdc, 0x24, 0x05, 0x50, 0xb5, 0x6a, 0xa1, 0x6e, - 0x55, 0xd9, 0x16, 0xab, 0x66, 0x8b, 0x03, 0x10, 0x8a, 0xe3, 0x4c, 0x8d, 0xbd, 0xe8, 0xa2, 0xf6, - 0x22, 0x29, 0x21, 0xde, 0x7b, 0x58, 0x26, 0xf4, 0xb2, 0x64, 0x89, 0x0d, 0x56, 0x9c, 0x39, 0x88, - 0x14, 0x57, 0x1e, 0xa6, 0x22, 0x44, 0x78, 0x11, 0x51, 0x39, 0x4c, 0x98, 0xb2, 0xa2, 0x45, 0x0a, - 0xc0, 0xdb, 0x83, 0xb5, 0x0a, 0xd1, 0xe7, 0x50, 0xf6, 0xb4, 0xf2, 0x84, 0x5e, 0x6a, 0x28, 0x23, - 0x2c, 0x00, 0xbc, 0x02, 0x66, 0x98, 0xdb, 0x6a, 0x86, 0x81, 0x77, 0x85, 0x60, 0x35, 0xa5, 0x38, - 0x1e, 0x45, 0xfe, 0x07, 0x26, 0x04, 0xbd, 0x60, 0xf8, 0x35, 0x58, 0x3e, 0x8f, 0x24, 0x8b, 0xa4, - 0xaa, 0x5f, 0xda, 0x76, 0x37, 0x4b, 0xbb, 0x37, 0xcf, 0xde, 0xd3, 0x29, 0xa7, 0xb4, 0x3f, 0x64, - 0x24, 0x2f, 0xc0, 0x3b, 0x00, 0xc9, 0x64, 0x23, 0xab, 0x3e, 0x4b, 0xdb, 0x4f, 0xca, 0xe5, 0x33, - 0x24, 0x93, 0x52, 0x89, 0xf7, 0xc7, 0x84, 0xf5, 0x59, 0x2d, 0xf0, 0x1b, 0x80, 0x1e, 0xa3, 0xc1, - 0xa7, 0x38, 0xa0, 0x92, 0x65, 0xc2, 0x36, 0xea, 0xc2, 0x3a, 0x93, 0x8c, 0x8e, 0x41, 0x4a, 0xf9, - 0xf8, 0x10, 0x56, 0xcf, 0x87, 0xfd, 0x7e, 0xca, 0x4a, 0xd8, 0xb7, 0x21, 0x13, 0x72, 0x96, 0xb8, - 0x94, 0xe2, 0xa0, 0x9a, 0xd6, 0x31, 0x48, 0xbd, 0x12, 0x7f, 0x84, 0x07, 0x05, 0x24, 0x62, 0x1e, - 0x09, 0x7d, 0xda, 0x66, 0x38, 0x75, 0x50, 0xcb, 0xeb, 0x18, 0x64, 0xaa, 0x16, 0xef, 0xc3, 0x32, - 0x4b, 0x12, 0x9e, 0x4c, 0xc8, 0xe6, 0x14, 0xd9, 0xe3, 0x3a, 0xd9, 0x7e, 0x39, 0xa9, 0x63, 0x90, - 0x6a, 0xd5, 0xae, 0x05, 0xf3, 0xdf, 0x53, 0xab, 0xbc, 0x1f, 0x08, 0x56, 0xaa, 0x6e, 0xe0, 0x75, - 0x98, 0x4f, 0xdd, 0xc8, 0x4f, 0x9c, 0x0e, 0xf0, 0x2b, 0xb0, 0xb2, 0x23, 0x61, 0x9b, 0x6e, 0xe3, - 0x2e, 0x4b, 0x95, 0xe7, 0x63, 0x0f, 0x5a, 0xf9, 0x91, 0x3b, 0xa2, 0xb2, 0x67, 0x37, 0x14, 0x6f, - 0x05, 0xf3, 0x7e, 0x22, 0x58, 0x9b, 0x61, 0xe9, 0xfd, 0x88, 0xf9, 0x85, 0xf4, 0xc6, 0xaa, 0xaf, - 0xc8, 0xfd, 0xa8, 0x79, 0x01, 0x0f, 0xa7, 0x56, 0x34, 0x55, 0xa2, 0x56, 0x34, 0xbb, 0xf3, 0x75, - 0xe0, 0x9d, 0xea, 0xc5, 0xd4, 0xbd, 0xba, 0xd1, 0x39, 0xaf, 0xdd, 0xf3, 0x68, 0xea, 0x9e, 0x9f, - 0xba, 0x99, 0xcd, 0x19, 0x37, 0xf3, 0xee, 0xdb, 0xbf, 0xb7, 0x0e, 0xba, 0xbe, 0x75, 0xd0, 0xbf, - 0x5b, 0x07, 0xfd, 0x1e, 0x3b, 0xc6, 0xf5, 0xd8, 0x31, 0x6e, 0xc6, 0x8e, 0xf1, 0xe5, 0xf9, 0x1d, - 0xbf, 0x6d, 0x67, 0x0b, 0xea, 0xe7, 0xe5, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0xbc, 0xc0, 0xf7, - 0x30, 0x0d, 0x07, 0x00, 0x00, + // 681 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0x4d, 0x6f, 0xd3, 0x40, + 0x10, 0xf5, 0x3a, 0x69, 0xdd, 0x4c, 0xd3, 0x16, 0xb6, 0x3d, 0x58, 0x15, 0x18, 0xcb, 0x42, 0x90, + 0x0b, 0xad, 0x54, 0x4e, 0x20, 0xa4, 0x8a, 0x86, 0x96, 0x44, 0x15, 0xa8, 0xda, 0x96, 0x22, 0x71, + 0xdb, 0xda, 0xdb, 0xc6, 0x28, 0xf1, 0x1a, 0xef, 0x86, 0x2a, 0x3f, 0x80, 0x0b, 0x48, 0x15, 0x7f, + 0x89, 0x1b, 0xc7, 0x1e, 0x7b, 0x44, 0xcd, 0x1f, 0x41, 0xde, 0x8d, 0xe3, 0x8f, 0xe4, 0xd0, 0x5b, + 0x2f, 0x4e, 0xe6, 0x79, 0xe6, 0xcd, 0x9b, 0xb7, 0x1f, 0x86, 0x5d, 0x9f, 0x0f, 0x06, 0x3c, 0x12, + 0x31, 0xf5, 0xd9, 0x36, 0x3f, 0xfb, 0xca, 0x7c, 0xb9, 0x2d, 0x13, 0xc6, 0xd4, 0xc3, 0xef, 0xd1, + 0xe8, 0x82, 0xc5, 0x09, 0x97, 0x7c, 0x5b, 0x3d, 0x45, 0x01, 0xde, 0x52, 0x08, 0x86, 0x1c, 0xf1, + 0x6e, 0x10, 0x00, 0xe1, 0x5c, 0xb6, 0x55, 0x88, 0x1f, 0x41, 0x83, 0xfa, 0xfd, 0x0e, 0xa3, 0x41, + 0x37, 0xb0, 0x91, 0x8b, 0x5a, 0x0d, 0x92, 0x03, 0xd8, 0x06, 0x4b, 0x75, 0xed, 0x06, 0xb6, 0xa9, + 0xde, 0x65, 0x21, 0x76, 0x00, 0x34, 0xe1, 0xc9, 0x28, 0x66, 0x76, 0x4d, 0xbd, 0x2c, 0x20, 0x29, + 0xaf, 0x0c, 0x07, 0x4c, 0x48, 0x3a, 0x88, 0xed, 0xba, 0x8b, 0x5a, 0x35, 0x92, 0x03, 0x18, 0x43, + 0x5d, 0x30, 0x16, 0xd8, 0x0b, 0x2e, 0x6a, 0x35, 0x89, 0xfa, 0x8f, 0x37, 0x61, 0x29, 0x0c, 0x58, + 0x24, 0x43, 0x39, 0xb2, 0x17, 0x15, 0x3e, 0x8d, 0xf1, 0x53, 0x58, 0xd1, 0xdc, 0x47, 0x74, 0xd4, + 0xe7, 0x34, 0xb0, 0x2d, 0x95, 0x50, 0x06, 0xbd, 0x2b, 0x13, 0xe0, 0x24, 0x61, 0x6c, 0x32, 0x9a, + 0x0b, 0xcb, 0xe9, 0xdc, 0x7a, 0x14, 0x61, 0x23, 0xb7, 0xd6, 0x6a, 0x90, 0x22, 0x54, 0x1e, 0xde, + 0xac, 0x0e, 0xff, 0x0c, 0x56, 0x45, 0x44, 0x63, 0xd1, 0xe3, 0x72, 0x8f, 0x8a, 0xd4, 0x03, 0x3d, + 0x66, 0x05, 0x4d, 0xfb, 0x68, 0x1d, 0xe2, 0x1d, 0x95, 0x54, 0x0d, 0xdb, 0x24, 0x45, 0x28, 0xed, + 0x93, 0x30, 0x1a, 0x1c, 0xb2, 0x51, 0x57, 0xcf, 0xdc, 0x20, 0x39, 0x50, 0xb6, 0x6a, 0xb1, 0x6a, + 0x55, 0xd1, 0x16, 0xab, 0x62, 0x8b, 0x03, 0x10, 0x8a, 0xe3, 0x89, 0x1a, 0x7b, 0xc9, 0x45, 0xad, + 0x25, 0x52, 0x40, 0xbc, 0xf7, 0xb0, 0x42, 0xe8, 0x65, 0xc1, 0x12, 0x1b, 0xac, 0x78, 0xe2, 0x20, + 0x52, 0x5c, 0x59, 0x98, 0x8a, 0x10, 0xe1, 0x45, 0x44, 0xe5, 0x30, 0x61, 0xca, 0x8a, 0x26, 0xc9, + 0x01, 0xaf, 0x0d, 0xeb, 0x25, 0xa2, 0xcf, 0xa1, 0xec, 0x69, 0xe5, 0x09, 0xbd, 0xd4, 0xd0, 0x84, + 0x30, 0x07, 0xf0, 0x2a, 0x98, 0x61, 0x66, 0xab, 0x19, 0x06, 0xde, 0x15, 0x82, 0xb5, 0x94, 0xe2, + 0x78, 0x14, 0xf9, 0x1f, 0x98, 0x10, 0xf4, 0x82, 0xe1, 0xd7, 0x60, 0xf9, 0x3c, 0x92, 0x2c, 0x92, + 0xaa, 0x7e, 0x79, 0xc7, 0xdd, 0x2a, 0xec, 0xde, 0x2c, 0xbb, 0xad, 0x53, 0x4e, 0x69, 0x7f, 0xc8, + 0x48, 0x56, 0x80, 0x77, 0x01, 0x92, 0xe9, 0x46, 0x56, 0x7d, 0x96, 0x77, 0x9e, 0x14, 0xcb, 0xe7, + 0x48, 0x26, 0x85, 0x12, 0xef, 0x8f, 0x09, 0x1b, 0xf3, 0x5a, 0xe0, 0x37, 0x00, 0x3d, 0x46, 0x83, + 0x4f, 0x71, 0x40, 0x25, 0x9b, 0x08, 0xdb, 0xac, 0x0a, 0xeb, 0x4c, 0x33, 0x3a, 0x06, 0x29, 0xe4, + 0xe3, 0x43, 0x58, 0x3b, 0x1f, 0xf6, 0xfb, 0x29, 0x2b, 0x61, 0xdf, 0x86, 0x4c, 0xc8, 0x79, 0xe2, + 0x52, 0x8a, 0x83, 0x72, 0x5a, 0xc7, 0x20, 0xd5, 0x4a, 0xfc, 0x11, 0x1e, 0xe4, 0x90, 0x88, 0x79, + 0x24, 0xf4, 0x69, 0x9b, 0xe3, 0xd4, 0x41, 0x25, 0xaf, 0x63, 0x90, 0x99, 0x5a, 0xbc, 0x0f, 0x2b, + 0x2c, 0x49, 0x78, 0x32, 0x25, 0xab, 0x2b, 0xb2, 0xc7, 0x55, 0xb2, 0xfd, 0x62, 0x52, 0xc7, 0x20, + 0xe5, 0xaa, 0x3d, 0x0b, 0x16, 0xbe, 0xa7, 0x56, 0x79, 0x3f, 0x10, 0xac, 0x96, 0xdd, 0xc0, 0x1b, + 0xb0, 0x90, 0xba, 0x91, 0x9d, 0x38, 0x1d, 0xe0, 0x57, 0x60, 0x4d, 0x8e, 0x84, 0x6d, 0xba, 0xb5, + 0xbb, 0x2c, 0x55, 0x96, 0x8f, 0x3d, 0x68, 0x66, 0x47, 0xee, 0x88, 0xca, 0x9e, 0x5d, 0x53, 0xbc, + 0x25, 0xcc, 0xfb, 0x89, 0x60, 0x7d, 0x8e, 0xa5, 0xf7, 0x23, 0xe6, 0x17, 0xd2, 0x1b, 0xab, 0xba, + 0x22, 0xf7, 0xa3, 0xe6, 0x05, 0x3c, 0x9c, 0x59, 0xd1, 0xf4, 0x26, 0x60, 0x49, 0xd2, 0xe6, 0x81, + 0xde, 0xdf, 0x75, 0x92, 0x85, 0xde, 0xa9, 0x5e, 0x50, 0xdd, 0xaf, 0x1b, 0x9d, 0xf3, 0xca, 0x5d, + 0x8f, 0x66, 0xee, 0xfa, 0x99, 0xdb, 0xd9, 0x9c, 0x73, 0x3b, 0xef, 0xbd, 0xfd, 0x7b, 0xeb, 0xa0, + 0xeb, 0x5b, 0x07, 0xfd, 0xbb, 0x75, 0xd0, 0xef, 0xb1, 0x63, 0x5c, 0x8f, 0x1d, 0xe3, 0x66, 0xec, + 0x18, 0x5f, 0x9e, 0xdf, 0xf1, 0xfb, 0x76, 0xb6, 0xa8, 0x7e, 0x5e, 0xfe, 0x0f, 0x00, 0x00, 0xff, + 0xff, 0x28, 0x34, 0x11, 0x90, 0x11, 0x07, 0x00, 0x00, } func (m *RootChange) Marshal() (dAtA []byte, err error) { @@ -1426,12 +1426,10 @@ func (m *TreeErrorResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Error) > 0 { - i -= len(m.Error) - copy(dAtA[i:], m.Error) - i = encodeVarintTreechange(dAtA, i, uint64(len(m.Error))) + if m.ErrCode != 0 { + i = encodeVarintTreechange(dAtA, i, uint64(m.ErrCode)) i-- - dAtA[i] = 0xa + dAtA[i] = 0x8 } return len(dAtA) - i, nil } @@ -1759,9 +1757,8 @@ func (m *TreeErrorResponse) Size() (n int) { } var l int _ = l - l = len(m.Error) - if l > 0 { - n += 1 + l + sovTreechange(uint64(l)) + if m.ErrCode != 0 { + n += 1 + sovTreechange(uint64(m.ErrCode)) } return n } @@ -3361,10 +3358,10 @@ func (m *TreeErrorResponse) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ErrCode", wireType) } - var stringLen uint64 + m.ErrCode = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowTreechange @@ -3374,24 +3371,11 @@ func (m *TreeErrorResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + m.ErrCode |= uint64(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthTreechange - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthTreechange - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Error = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipTreechange(dAtA[iNdEx:]) diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index 74b3f7fa..450a1ce5 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -2,6 +2,9 @@ package objectsync import ( "context" + "github.com/anytypeio/any-sync/commonspace/object/tree/synctree" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/gogo/protobuf/proto" "sync/atomic" "time" @@ -23,7 +26,6 @@ type ObjectSync interface { synchandler.SyncHandler MessagePool() MessagePool - Init() Close() (err error) } @@ -31,6 +33,7 @@ type objectSync struct { spaceId string messagePool MessagePool + syncClient synctree.SyncClient objectGetter syncobjectgetter.SyncObjectGetter configuration nodeconf.Configuration spaceStorage spacestorage.SpaceStorage @@ -48,40 +51,18 @@ func NewObjectSync( objectGetter syncobjectgetter.SyncObjectGetter, storage spacestorage.SpaceStorage) ObjectSync { syncCtx, cancel := context.WithCancel(context.Background()) - os := newObjectSync( - spaceId, - spaceIsDeleted, - configuration, - objectGetter, - storage, - syncCtx, - cancel) - msgPool := newMessagePool(peerManager, os.handleMessage) - os.messagePool = msgPool - return os -} - -func newObjectSync( - spaceId string, - spaceIsDeleted *atomic.Bool, - configuration nodeconf.Configuration, - objectGetter syncobjectgetter.SyncObjectGetter, - spaceStorage spacestorage.SpaceStorage, - syncCtx context.Context, - cancel context.CancelFunc, -) *objectSync { - return &objectSync{ + os := &objectSync{ objectGetter: objectGetter, - spaceStorage: spaceStorage, + spaceStorage: storage, spaceId: spaceId, syncCtx: syncCtx, cancelSync: cancel, spaceIsDeleted: spaceIsDeleted, configuration: configuration, + syncClient: synctree.NewSyncClient(spaceId, peerManager, synctree.GetRequestFactory()), } -} - -func (s *objectSync) Init() { + os.messagePool = newMessagePool(peerManager, os.handleMessage) + return os } func (s *objectSync) Close() (err error) { @@ -109,6 +90,10 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp log.With(zap.String("objectId", msg.ObjectId), zap.String("replyId", msg.ReplyId)).DebugCtx(ctx, "handling message") obj, err := s.objectGetter.GetObject(ctx, msg.ObjectId) if err != nil { + respErr := s.sendErrorResponse(ctx, msg, senderId, err) + if respErr != nil { + log.Debug("failed to send error response", zap.Error(respErr)) + } return } return obj.HandleMessage(ctx, senderId, msg) @@ -117,3 +102,13 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp func (s *objectSync) MessagePool() MessagePool { return s.messagePool } + +func (s *objectSync) sendErrorResponse(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, senderId string, respErr error) (err error) { + unmarshalled := &treechangeproto.TreeSyncMessage{} + err = proto.Unmarshal(msg.Payload, unmarshalled) + if err != nil { + return + } + resp := treechangeproto.WrapError(respErr, unmarshalled.RootChange) + return s.syncClient.SendWithReply(ctx, senderId, resp, msg.ReplyId) +} diff --git a/commonspace/space.go b/commonspace/space.go index c68ff8e3..f69b4aaa 100644 --- a/commonspace/space.go +++ b/commonspace/space.go @@ -204,7 +204,6 @@ func (s *space) Init(ctx context.Context) (err error) { OnSpaceDelete: s.onSpaceDelete, } s.settingsObject = settings.NewSettingsObject(deps, s.id) - s.objectSync.Init() s.headSync.Init(initialIds, deletionState) err = s.settingsObject.Init(ctx) if err != nil { From 58f3c4ef8c482c0c4482dddd93e9363f69eb9204 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 18 Apr 2023 22:51:12 +0200 Subject: [PATCH 13/45] Fix objectsync etc --- commonspace/config.go | 5 +- .../synctree/mock_synctree/mock_synctree.go | 111 +------------- commonspace/object/tree/synctree/synctree.go | 7 +- .../object/tree/synctree/synctree_test.go | 10 +- .../object/tree/synctree/synctreehandler.go | 7 +- .../tree/synctree/synctreehandler_test.go | 29 ++-- .../object/tree/synctree/treeremotegetter.go | 5 +- .../object/tree/synctree/utils_test.go | 25 ++-- .../object/tree/treechangeproto/errors.go | 14 ++ .../treechangeproto/protos/treechange.proto | 7 + .../object/tree/treechangeproto/treechange.go | 4 +- .../tree/treechangeproto/treechange.pb.go | 123 ++++++++++------ .../mock_treemanager/mock_treemanager.go | 30 ---- commonspace/object/treemanager/treemanager.go | 2 - .../mock_objectsync/mock_objectsync.go | 138 ++++++++++++++++++ commonspace/objectsync/objectsync.go | 12 +- .../synctree => objectsync}/requestfactory.go | 2 +- .../synctree => objectsync}/syncclient.go | 24 +-- commonspace/settings/deletionmanager.go | 4 - commonspace/settings/deletionmanager_test.go | 2 - commonspace/space.go | 5 +- commonspace/spaceservice.go | 8 + 22 files changed, 320 insertions(+), 254 deletions(-) create mode 100644 commonspace/object/tree/treechangeproto/errors.go create mode 100644 commonspace/objectsync/mock_objectsync/mock_objectsync.go rename commonspace/{object/tree/synctree => objectsync}/requestfactory.go (99%) rename commonspace/{object/tree/synctree => objectsync}/syncclient.go (70%) diff --git a/commonspace/config.go b/commonspace/config.go index befa3ac8..f7444fdd 100644 --- a/commonspace/config.go +++ b/commonspace/config.go @@ -5,6 +5,7 @@ type ConfigGetter interface { } type Config struct { - GCTTL int `yaml:"gcTTL"` - SyncPeriod int `yaml:"syncPeriod"` + GCTTL int `yaml:"gcTTL"` + SyncPeriod int `yaml:"syncPeriod"` + TreeInMemoryData bool `yaml:"treeInMemoryData"` } diff --git a/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go b/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go index 3ab6cf1b..07eb8878 100644 --- a/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go +++ b/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/anytypeio/any-sync/commonspace/object/tree/synctree (interfaces: SyncClient,SyncTree,ReceiveQueue,HeadNotifiable) +// Source: github.com/anytypeio/any-sync/commonspace/object/tree/synctree (interfaces: SyncTree,ReceiveQueue,HeadNotifiable) // Package mock_synctree is a generated GoMock package. package mock_synctree @@ -18,115 +18,6 @@ import ( gomock "github.com/golang/mock/gomock" ) -// MockSyncClient is a mock of SyncClient interface. -type MockSyncClient struct { - ctrl *gomock.Controller - recorder *MockSyncClientMockRecorder -} - -// MockSyncClientMockRecorder is the mock recorder for MockSyncClient. -type MockSyncClientMockRecorder struct { - mock *MockSyncClient -} - -// NewMockSyncClient creates a new mock instance. -func NewMockSyncClient(ctrl *gomock.Controller) *MockSyncClient { - mock := &MockSyncClient{ctrl: ctrl} - mock.recorder = &MockSyncClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockSyncClient) EXPECT() *MockSyncClientMockRecorder { - return m.recorder -} - -// Broadcast mocks base method. -func (m *MockSyncClient) Broadcast(arg0 context.Context, arg1 *treechangeproto.TreeSyncMessage) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Broadcast", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// Broadcast indicates an expected call of Broadcast. -func (mr *MockSyncClientMockRecorder) Broadcast(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Broadcast", reflect.TypeOf((*MockSyncClient)(nil).Broadcast), arg0, arg1) -} - -// CreateFullSyncRequest mocks base method. -func (m *MockSyncClient) CreateFullSyncRequest(arg0 objecttree.ObjectTree, arg1, arg2 []string) (*treechangeproto.TreeSyncMessage, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateFullSyncRequest", arg0, arg1, arg2) - ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateFullSyncRequest indicates an expected call of CreateFullSyncRequest. -func (mr *MockSyncClientMockRecorder) CreateFullSyncRequest(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFullSyncRequest", reflect.TypeOf((*MockSyncClient)(nil).CreateFullSyncRequest), arg0, arg1, arg2) -} - -// CreateFullSyncResponse mocks base method. -func (m *MockSyncClient) CreateFullSyncResponse(arg0 objecttree.ObjectTree, arg1, arg2 []string) (*treechangeproto.TreeSyncMessage, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateFullSyncResponse", arg0, arg1, arg2) - ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateFullSyncResponse indicates an expected call of CreateFullSyncResponse. -func (mr *MockSyncClientMockRecorder) CreateFullSyncResponse(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFullSyncResponse", reflect.TypeOf((*MockSyncClient)(nil).CreateFullSyncResponse), arg0, arg1, arg2) -} - -// CreateHeadUpdate mocks base method. -func (m *MockSyncClient) CreateHeadUpdate(arg0 objecttree.ObjectTree, arg1 []*treechangeproto.RawTreeChangeWithId) *treechangeproto.TreeSyncMessage { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateHeadUpdate", arg0, arg1) - ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) - return ret0 -} - -// CreateHeadUpdate indicates an expected call of CreateHeadUpdate. -func (mr *MockSyncClientMockRecorder) CreateHeadUpdate(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateHeadUpdate", reflect.TypeOf((*MockSyncClient)(nil).CreateHeadUpdate), arg0, arg1) -} - -// CreateNewTreeRequest mocks base method. -func (m *MockSyncClient) CreateNewTreeRequest() *treechangeproto.TreeSyncMessage { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateNewTreeRequest") - ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) - return ret0 -} - -// CreateNewTreeRequest indicates an expected call of CreateNewTreeRequest. -func (mr *MockSyncClientMockRecorder) CreateNewTreeRequest() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateNewTreeRequest", reflect.TypeOf((*MockSyncClient)(nil).CreateNewTreeRequest)) -} - -// SendWithReply mocks base method. -func (m *MockSyncClient) SendWithReply(arg0 context.Context, arg1 string, arg2 *treechangeproto.TreeSyncMessage, arg3 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendWithReply", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(error) - return ret0 -} - -// SendWithReply indicates an expected call of SendWithReply. -func (mr *MockSyncClientMockRecorder) SendWithReply(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendWithReply", reflect.TypeOf((*MockSyncClient)(nil).SendWithReply), arg0, arg1, arg2, arg3) -} - // MockSyncTree is a mock of SyncTree interface. type MockSyncTree struct { ctrl *gomock.Controller diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index 89787597..683079f5 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -1,3 +1,4 @@ +//go:generate mockgen -destination mock_synctree/mock_synctree.go github.com/anytypeio/any-sync/commonspace/object/tree/synctree SyncTree,ReceiveQueue,HeadNotifiable package synctree import ( @@ -43,7 +44,7 @@ type SyncTree interface { type syncTree struct { objecttree.ObjectTree synchandler.SyncHandler - syncClient SyncClient + syncClient objectsync.SyncClient syncStatus syncstatus.StatusUpdater notifiable HeadNotifiable listener updatelistener.UpdateListener @@ -54,7 +55,7 @@ type syncTree struct { var log = logger.NewNamed("common.commonspace.synctree") -var createSyncClient = NewSyncClient +var createSyncClient = objectsync.NewSyncClient type ResponsiblePeersGetter interface { GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) @@ -101,7 +102,7 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t Sy syncClient := createSyncClient( deps.SpaceId, deps.ObjectSync.MessagePool(), - sharedFactory) + objectsync.GetRequestFactory()) syncTree := &syncTree{ ObjectTree: objTree, syncClient: syncClient, diff --git a/commonspace/object/tree/synctree/synctree_test.go b/commonspace/object/tree/synctree/synctree_test.go index 8854a702..cfd56a8d 100644 --- a/commonspace/object/tree/synctree/synctree_test.go +++ b/commonspace/object/tree/synctree/synctree_test.go @@ -4,11 +4,11 @@ import ( "context" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree/mock_objecttree" - "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/mock_synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener" "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener/mock_updatelistener" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/objectsync" + "github.com/anytypeio/any-sync/commonspace/objectsync/mock_objectsync" "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/anytypeio/any-sync/nodeconf" "github.com/golang/mock/gomock" @@ -18,7 +18,7 @@ import ( type syncTreeMatcher struct { objTree objecttree.ObjectTree - client SyncClient + client objectsync.SyncClient listener updatelistener.UpdateListener } @@ -34,8 +34,8 @@ func (s syncTreeMatcher) String() string { return "" } -func syncClientFuncCreator(client SyncClient) func(spaceId string, factory RequestFactory, objectSync objectsync.ObjectSync, configuration nodeconf.Configuration) SyncClient { - return func(spaceId string, factory RequestFactory, objectSync objectsync.ObjectSync, configuration nodeconf.Configuration) SyncClient { +func syncClientFuncCreator(client objectsync.SyncClient) func(spaceId string, factory objectsync.RequestFactory, objectSync objectsync.ObjectSync, configuration nodeconf.Configuration) objectsync.SyncClient { + return func(spaceId string, factory objectsync.RequestFactory, objectSync objectsync.ObjectSync, configuration nodeconf.Configuration) objectsync.SyncClient { return client } } @@ -46,7 +46,7 @@ func Test_BuildSyncTree(t *testing.T) { defer ctrl.Finish() updateListenerMock := mock_updatelistener.NewMockUpdateListener(ctrl) - syncClientMock := mock_synctree.NewMockSyncClient(ctrl) + syncClientMock := mock_objectsync.NewMockSyncClient(ctrl) objTreeMock := newTestObjMock(mock_objecttree.NewMockObjectTree(ctrl)) tr := &syncTree{ ObjectTree: objTreeMock, diff --git a/commonspace/object/tree/synctree/synctreehandler.go b/commonspace/object/tree/synctree/synctreehandler.go index 63b708d9..7865f2a7 100644 --- a/commonspace/object/tree/synctree/synctreehandler.go +++ b/commonspace/object/tree/synctree/synctreehandler.go @@ -4,6 +4,7 @@ import ( "context" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/commonspace/objectsync" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/commonspace/syncstatus" @@ -15,7 +16,7 @@ import ( type syncTreeHandler struct { objTree objecttree.ObjectTree - syncClient SyncClient + syncClient objectsync.SyncClient syncStatus syncstatus.StatusUpdater handlerLock sync.Mutex spaceId string @@ -24,7 +25,7 @@ type syncTreeHandler struct { const maxQueueSize = 5 -func newSyncTreeHandler(spaceId string, objTree objecttree.ObjectTree, syncClient SyncClient, syncStatus syncstatus.StatusUpdater) synchandler.SyncHandler { +func newSyncTreeHandler(spaceId string, objTree objecttree.ObjectTree, syncClient objectsync.SyncClient, syncStatus syncstatus.StatusUpdater) synchandler.SyncHandler { return &syncTreeHandler{ objTree: objTree, syncClient: syncClient, @@ -166,7 +167,7 @@ func (s *syncTreeHandler) handleFullSyncRequest( defer func() { if err != nil { log.With(zap.Error(err)).DebugCtx(ctx, "full sync request finished with error") - s.syncClient.SendWithReply(ctx, senderId, treechangeproto.WrapError(err, header), replyId) + s.syncClient.SendWithReply(ctx, senderId, treechangeproto.WrapError(treechangeproto.ErrorCodes_FullSyncRequestError, header), replyId) return } else if fullResponse != nil { cnt := fullResponse.Content.GetFullSyncResponse() diff --git a/commonspace/object/tree/synctree/synctreehandler_test.go b/commonspace/object/tree/synctree/synctreehandler_test.go index f655b34a..b95ffc82 100644 --- a/commonspace/object/tree/synctree/synctreehandler_test.go +++ b/commonspace/object/tree/synctree/synctreehandler_test.go @@ -3,13 +3,14 @@ package synctree import ( "context" "fmt" + "github.com/anytypeio/any-sync/commonspace/objectsync" + "github.com/anytypeio/any-sync/commonspace/objectsync/mock_objectsync" "sync" "testing" "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree/mock_objecttree" - "github.com/anytypeio/any-sync/commonspace/object/tree/synctree/mock_synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/syncstatus" "github.com/golang/mock/gomock" @@ -54,7 +55,7 @@ func (t *testObjTreeMock) TryRLock() bool { type syncHandlerFixture struct { ctrl *gomock.Controller - syncClientMock *mock_synctree.MockSyncClient + syncClientMock *mock_objectsync.MockSyncClient objectTreeMock *testObjTreeMock receiveQueueMock ReceiveQueue @@ -63,7 +64,7 @@ type syncHandlerFixture struct { func newSyncHandlerFixture(t *testing.T) *syncHandlerFixture { ctrl := gomock.NewController(t) - syncClientMock := mock_synctree.NewMockSyncClient(ctrl) + syncClientMock := mock_objectsync.NewMockSyncClient(ctrl) objectTreeMock := newTestObjMock(mock_objecttree.NewMockObjectTree(ctrl)) receiveQueue := newReceiveQueue(5) @@ -109,7 +110,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2"}).Times(2) @@ -138,7 +139,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2"}).AnyTimes() @@ -171,7 +172,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h1"}).AnyTimes() @@ -192,7 +193,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h2"}).AnyTimes() @@ -217,7 +218,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapHeadUpdate(headUpdate, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Heads().Return([]string{"h1"}).AnyTimes() @@ -250,7 +251,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT().Header().Return(nil) @@ -283,7 +284,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT(). Id().AnyTimes().Return(treeId) @@ -312,7 +313,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") objectMsg.RequestId = replyId fx.objectTreeMock.EXPECT(). @@ -339,7 +340,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullRequest(fullSyncRequest, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, "") + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, "") fx.objectTreeMock.EXPECT(). Id().AnyTimes().Return(treeId) @@ -380,7 +381,7 @@ func TestSyncHandler_HandleFullSyncResponse(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullResponse(fullSyncResponse, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, replyId) + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, replyId) fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT(). @@ -413,7 +414,7 @@ func TestSyncHandler_HandleFullSyncResponse(t *testing.T) { SnapshotPath: []string{"h1"}, } treeMsg := treechangeproto.WrapFullResponse(fullSyncResponse, chWithId) - objectMsg, _ := marshallTreeMessage(treeMsg, "spaceId", treeId, replyId) + objectMsg, _ := objectsync.MarshallTreeMessage(treeMsg, "spaceId", treeId, replyId) fx.objectTreeMock.EXPECT().Id().AnyTimes().Return(treeId) fx.objectTreeMock.EXPECT(). diff --git a/commonspace/object/tree/synctree/treeremotegetter.go b/commonspace/object/tree/synctree/treeremotegetter.go index 9fd0677a..9f819c41 100644 --- a/commonspace/object/tree/synctree/treeremotegetter.go +++ b/commonspace/object/tree/synctree/treeremotegetter.go @@ -6,6 +6,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/commonspace/objectsync" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/net/peer" "github.com/anytypeio/any-sync/net/rpc/rpcerr" @@ -46,8 +47,8 @@ func (t treeRemoteGetter) getPeers(ctx context.Context) (peerIds []string, err e } func (t treeRemoteGetter) treeRequest(ctx context.Context, peerId string) (msg *treechangeproto.TreeSyncMessage, err error) { - newTreeRequest := GetRequestFactory().CreateNewTreeRequest() - objMsg, err := marshallTreeMessage(newTreeRequest, t.deps.SpaceId, t.treeId, "") + newTreeRequest := objectsync.GetRequestFactory().CreateNewTreeRequest() + objMsg, err := objectsync.MarshallTreeMessage(newTreeRequest, t.deps.SpaceId, t.treeId, "") if err != nil { return } diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index 524ab1d2..8e2ffbe9 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -7,6 +7,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" + "github.com/anytypeio/any-sync/commonspace/objectsync" "github.com/anytypeio/any-sync/commonspace/objectsync/synchandler" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/commonspace/syncstatus" @@ -84,7 +85,7 @@ type processSyncHandler struct { peerId string aclList list.AclList log *messageLog - syncClient SyncClient + syncClient objectsync.SyncClient } func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { @@ -97,9 +98,9 @@ func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, return } if unmarshalled.Content.GetFullSyncResponse() == nil { - newTreeRequest := GetRequestFactory().CreateNewTreeRequest() + newTreeRequest := objectsync.GetRequestFactory().CreateNewTreeRequest() var objMsg *spacesyncproto.ObjectSyncMessage - objMsg, err = marshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") + objMsg, err = objectsync.MarshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") if err != nil { return } @@ -124,8 +125,8 @@ func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, } p.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, p.syncClient, syncstatus.NewNoOpSyncStatus()) var objMsg *spacesyncproto.ObjectSyncMessage - newTreeRequest := GetRequestFactory().CreateHeadUpdate(netTree, res.Added) - objMsg, err = marshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") + newTreeRequest := objectsync.GetRequestFactory().CreateHeadUpdate(netTree, res.Added) + objMsg, err = objectsync.MarshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") if err != nil { return } @@ -143,9 +144,9 @@ func newProcessSyncHandler(peerId string, syncHandler synchandler.SyncHandler) * func (p *processSyncHandler) manager() *processPeerManager { if p.SyncHandler != nil { - return p.SyncHandler.(*syncTreeHandler).syncClient.(*syncClient).PeerManager.(*processPeerManager) + return p.SyncHandler.(*syncTreeHandler).syncClient.PeerManager().(*processPeerManager) } - return p.syncClient.(*syncClient).PeerManager.(*processPeerManager) + return p.syncClient.PeerManager().(*processPeerManager) } func (p *processSyncHandler) tree() *broadcastTree { @@ -229,7 +230,7 @@ func (m *processPeerManager) GetResponsiblePeers(ctx context.Context) (peers []p type broadcastTree struct { objecttree.ObjectTree - SyncClient + objectsync.SyncClient } func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) (objecttree.AddResult, error) { @@ -243,8 +244,8 @@ func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.Ra } func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *processSyncHandler { - factory := GetRequestFactory() - syncClient := NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) + factory := objectsync.GetRequestFactory() + syncClient := objectsync.NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) netTree := &broadcastTree{ ObjectTree: objTree, SyncClient: syncClient, @@ -254,8 +255,8 @@ func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, lo } func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *processSyncHandler { - factory := GetRequestFactory() - syncClient := NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) + factory := objectsync.GetRequestFactory() + syncClient := objectsync.NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) batcher := mb.New[processMsg](0) return &processSyncHandler{ diff --git a/commonspace/object/tree/treechangeproto/errors.go b/commonspace/object/tree/treechangeproto/errors.go new file mode 100644 index 00000000..375ebedb --- /dev/null +++ b/commonspace/object/tree/treechangeproto/errors.go @@ -0,0 +1,14 @@ +package treechangeproto + +import ( + "errors" + "github.com/anytypeio/any-sync/net/rpc/rpcerr" +) + +var ( + errGroup = rpcerr.ErrGroup(ErrorCodes_ErrorOffset) + + ErrUnexpected = errGroup.Register(errors.New("unexpected error"), uint64(ErrorCodes_Unexpected)) + ErrGetTree = errGroup.Register(errors.New("tree not found"), uint64(ErrorCodes_GetTreeError)) + ErrFullSync = errGroup.Register(errors.New("full sync request error"), uint64(ErrorCodes_FullSyncRequestError)) +) diff --git a/commonspace/object/tree/treechangeproto/protos/treechange.proto b/commonspace/object/tree/treechangeproto/protos/treechange.proto index 1ecd9bd6..ae7cebd5 100644 --- a/commonspace/object/tree/treechangeproto/protos/treechange.proto +++ b/commonspace/object/tree/treechangeproto/protos/treechange.proto @@ -56,6 +56,13 @@ message RawTreeChangeWithId { string id = 2; } +enum ErrorCodes { + Unexpected = 0; + GetTreeError = 1; + FullSyncRequestError = 2; + ErrorOffset = 400; +} + message TreeSyncMessage { TreeSyncContentValue content = 1; RawTreeChangeWithId rootChange = 2; diff --git a/commonspace/object/tree/treechangeproto/treechange.go b/commonspace/object/tree/treechangeproto/treechange.go index 3aba1fb0..0dc5d668 100644 --- a/commonspace/object/tree/treechangeproto/treechange.go +++ b/commonspace/object/tree/treechangeproto/treechange.go @@ -27,10 +27,10 @@ func WrapFullResponse(response *TreeFullSyncResponse, rootChange *RawTreeChangeW } } -func WrapError(err error, rootChange *RawTreeChangeWithId) *TreeSyncMessage { +func WrapError(code ErrorCodes, rootChange *RawTreeChangeWithId) *TreeSyncMessage { return &TreeSyncMessage{ Content: &TreeSyncContentValue{ - Value: &TreeSyncContentValue_ErrorResponse{ErrorResponse: &TreeErrorResponse{Error: err.Error()}}, + Value: &TreeSyncContentValue_ErrorResponse{ErrorResponse: &TreeErrorResponse{ErrCode: uint64(code)}}, }, RootChange: rootChange, } diff --git a/commonspace/object/tree/treechangeproto/treechange.pb.go b/commonspace/object/tree/treechangeproto/treechange.pb.go index 14815df1..f4098fc6 100644 --- a/commonspace/object/tree/treechangeproto/treechange.pb.go +++ b/commonspace/object/tree/treechangeproto/treechange.pb.go @@ -22,6 +22,37 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package +type ErrorCodes int32 + +const ( + ErrorCodes_Unexpected ErrorCodes = 0 + ErrorCodes_GetTreeError ErrorCodes = 1 + ErrorCodes_FullSyncRequestError ErrorCodes = 2 + ErrorCodes_ErrorOffset ErrorCodes = 400 +) + +var ErrorCodes_name = map[int32]string{ + 0: "Unexpected", + 1: "GetTreeError", + 2: "FullSyncRequestError", + 400: "ErrorOffset", +} + +var ErrorCodes_value = map[string]int32{ + "Unexpected": 0, + "GetTreeError": 1, + "FullSyncRequestError": 2, + "ErrorOffset": 400, +} + +func (x ErrorCodes) String() string { + return proto.EnumName(ErrorCodes_name, int32(x)) +} + +func (ErrorCodes) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_5033f0301ef9b772, []int{0} +} + // RootChange is a root of a tree type RootChange struct { // AclHeadId is a cid of latest acl record at the time of tree creation @@ -788,6 +819,7 @@ func (m *TreeChangeInfo) GetChangePayload() []byte { } func init() { + proto.RegisterEnum("treechange.ErrorCodes", ErrorCodes_name, ErrorCodes_value) proto.RegisterType((*RootChange)(nil), "treechange.RootChange") proto.RegisterType((*TreeChange)(nil), "treechange.TreeChange") proto.RegisterType((*RawTreeChange)(nil), "treechange.RawTreeChange") @@ -806,50 +838,53 @@ func init() { } var fileDescriptor_5033f0301ef9b772 = []byte{ - // 681 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0x4d, 0x6f, 0xd3, 0x40, - 0x10, 0xf5, 0x3a, 0x69, 0xdd, 0x4c, 0xd3, 0x16, 0xb6, 0x3d, 0x58, 0x15, 0x18, 0xcb, 0x42, 0x90, - 0x0b, 0xad, 0x54, 0x4e, 0x20, 0xa4, 0x8a, 0x86, 0x96, 0x44, 0x15, 0xa8, 0xda, 0x96, 0x22, 0x71, - 0xdb, 0xda, 0xdb, 0xc6, 0x28, 0xf1, 0x1a, 0xef, 0x86, 0x2a, 0x3f, 0x80, 0x0b, 0x48, 0x15, 0x7f, - 0x89, 0x1b, 0xc7, 0x1e, 0x7b, 0x44, 0xcd, 0x1f, 0x41, 0xde, 0x8d, 0xe3, 0x8f, 0xe4, 0xd0, 0x5b, - 0x2f, 0x4e, 0xe6, 0x79, 0xe6, 0xcd, 0x9b, 0xb7, 0x1f, 0x86, 0x5d, 0x9f, 0x0f, 0x06, 0x3c, 0x12, - 0x31, 0xf5, 0xd9, 0x36, 0x3f, 0xfb, 0xca, 0x7c, 0xb9, 0x2d, 0x13, 0xc6, 0xd4, 0xc3, 0xef, 0xd1, - 0xe8, 0x82, 0xc5, 0x09, 0x97, 0x7c, 0x5b, 0x3d, 0x45, 0x01, 0xde, 0x52, 0x08, 0x86, 0x1c, 0xf1, - 0x6e, 0x10, 0x00, 0xe1, 0x5c, 0xb6, 0x55, 0x88, 0x1f, 0x41, 0x83, 0xfa, 0xfd, 0x0e, 0xa3, 0x41, - 0x37, 0xb0, 0x91, 0x8b, 0x5a, 0x0d, 0x92, 0x03, 0xd8, 0x06, 0x4b, 0x75, 0xed, 0x06, 0xb6, 0xa9, - 0xde, 0x65, 0x21, 0x76, 0x00, 0x34, 0xe1, 0xc9, 0x28, 0x66, 0x76, 0x4d, 0xbd, 0x2c, 0x20, 0x29, - 0xaf, 0x0c, 0x07, 0x4c, 0x48, 0x3a, 0x88, 0xed, 0xba, 0x8b, 0x5a, 0x35, 0x92, 0x03, 0x18, 0x43, - 0x5d, 0x30, 0x16, 0xd8, 0x0b, 0x2e, 0x6a, 0x35, 0x89, 0xfa, 0x8f, 0x37, 0x61, 0x29, 0x0c, 0x58, - 0x24, 0x43, 0x39, 0xb2, 0x17, 0x15, 0x3e, 0x8d, 0xf1, 0x53, 0x58, 0xd1, 0xdc, 0x47, 0x74, 0xd4, - 0xe7, 0x34, 0xb0, 0x2d, 0x95, 0x50, 0x06, 0xbd, 0x2b, 0x13, 0xe0, 0x24, 0x61, 0x6c, 0x32, 0x9a, - 0x0b, 0xcb, 0xe9, 0xdc, 0x7a, 0x14, 0x61, 0x23, 0xb7, 0xd6, 0x6a, 0x90, 0x22, 0x54, 0x1e, 0xde, - 0xac, 0x0e, 0xff, 0x0c, 0x56, 0x45, 0x44, 0x63, 0xd1, 0xe3, 0x72, 0x8f, 0x8a, 0xd4, 0x03, 0x3d, - 0x66, 0x05, 0x4d, 0xfb, 0x68, 0x1d, 0xe2, 0x1d, 0x95, 0x54, 0x0d, 0xdb, 0x24, 0x45, 0x28, 0xed, - 0x93, 0x30, 0x1a, 0x1c, 0xb2, 0x51, 0x57, 0xcf, 0xdc, 0x20, 0x39, 0x50, 0xb6, 0x6a, 0xb1, 0x6a, - 0x55, 0xd1, 0x16, 0xab, 0x62, 0x8b, 0x03, 0x10, 0x8a, 0xe3, 0x89, 0x1a, 0x7b, 0xc9, 0x45, 0xad, - 0x25, 0x52, 0x40, 0xbc, 0xf7, 0xb0, 0x42, 0xe8, 0x65, 0xc1, 0x12, 0x1b, 0xac, 0x78, 0xe2, 0x20, - 0x52, 0x5c, 0x59, 0x98, 0x8a, 0x10, 0xe1, 0x45, 0x44, 0xe5, 0x30, 0x61, 0xca, 0x8a, 0x26, 0xc9, - 0x01, 0xaf, 0x0d, 0xeb, 0x25, 0xa2, 0xcf, 0xa1, 0xec, 0x69, 0xe5, 0x09, 0xbd, 0xd4, 0xd0, 0x84, - 0x30, 0x07, 0xf0, 0x2a, 0x98, 0x61, 0x66, 0xab, 0x19, 0x06, 0xde, 0x15, 0x82, 0xb5, 0x94, 0xe2, - 0x78, 0x14, 0xf9, 0x1f, 0x98, 0x10, 0xf4, 0x82, 0xe1, 0xd7, 0x60, 0xf9, 0x3c, 0x92, 0x2c, 0x92, - 0xaa, 0x7e, 0x79, 0xc7, 0xdd, 0x2a, 0xec, 0xde, 0x2c, 0xbb, 0xad, 0x53, 0x4e, 0x69, 0x7f, 0xc8, - 0x48, 0x56, 0x80, 0x77, 0x01, 0x92, 0xe9, 0x46, 0x56, 0x7d, 0x96, 0x77, 0x9e, 0x14, 0xcb, 0xe7, - 0x48, 0x26, 0x85, 0x12, 0xef, 0x8f, 0x09, 0x1b, 0xf3, 0x5a, 0xe0, 0x37, 0x00, 0x3d, 0x46, 0x83, - 0x4f, 0x71, 0x40, 0x25, 0x9b, 0x08, 0xdb, 0xac, 0x0a, 0xeb, 0x4c, 0x33, 0x3a, 0x06, 0x29, 0xe4, - 0xe3, 0x43, 0x58, 0x3b, 0x1f, 0xf6, 0xfb, 0x29, 0x2b, 0x61, 0xdf, 0x86, 0x4c, 0xc8, 0x79, 0xe2, - 0x52, 0x8a, 0x83, 0x72, 0x5a, 0xc7, 0x20, 0xd5, 0x4a, 0xfc, 0x11, 0x1e, 0xe4, 0x90, 0x88, 0x79, - 0x24, 0xf4, 0x69, 0x9b, 0xe3, 0xd4, 0x41, 0x25, 0xaf, 0x63, 0x90, 0x99, 0x5a, 0xbc, 0x0f, 0x2b, - 0x2c, 0x49, 0x78, 0x32, 0x25, 0xab, 0x2b, 0xb2, 0xc7, 0x55, 0xb2, 0xfd, 0x62, 0x52, 0xc7, 0x20, - 0xe5, 0xaa, 0x3d, 0x0b, 0x16, 0xbe, 0xa7, 0x56, 0x79, 0x3f, 0x10, 0xac, 0x96, 0xdd, 0xc0, 0x1b, - 0xb0, 0x90, 0xba, 0x91, 0x9d, 0x38, 0x1d, 0xe0, 0x57, 0x60, 0x4d, 0x8e, 0x84, 0x6d, 0xba, 0xb5, - 0xbb, 0x2c, 0x55, 0x96, 0x8f, 0x3d, 0x68, 0x66, 0x47, 0xee, 0x88, 0xca, 0x9e, 0x5d, 0x53, 0xbc, - 0x25, 0xcc, 0xfb, 0x89, 0x60, 0x7d, 0x8e, 0xa5, 0xf7, 0x23, 0xe6, 0x17, 0xd2, 0x1b, 0xab, 0xba, - 0x22, 0xf7, 0xa3, 0xe6, 0x05, 0x3c, 0x9c, 0x59, 0xd1, 0xf4, 0x26, 0x60, 0x49, 0xd2, 0xe6, 0x81, - 0xde, 0xdf, 0x75, 0x92, 0x85, 0xde, 0xa9, 0x5e, 0x50, 0xdd, 0xaf, 0x1b, 0x9d, 0xf3, 0xca, 0x5d, - 0x8f, 0x66, 0xee, 0xfa, 0x99, 0xdb, 0xd9, 0x9c, 0x73, 0x3b, 0xef, 0xbd, 0xfd, 0x7b, 0xeb, 0xa0, - 0xeb, 0x5b, 0x07, 0xfd, 0xbb, 0x75, 0xd0, 0xef, 0xb1, 0x63, 0x5c, 0x8f, 0x1d, 0xe3, 0x66, 0xec, - 0x18, 0x5f, 0x9e, 0xdf, 0xf1, 0xfb, 0x76, 0xb6, 0xa8, 0x7e, 0x5e, 0xfe, 0x0f, 0x00, 0x00, 0xff, - 0xff, 0x28, 0x34, 0x11, 0x90, 0x11, 0x07, 0x00, 0x00, + // 732 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0x4f, 0x4f, 0xfb, 0x46, + 0x10, 0xf5, 0x3a, 0x81, 0x90, 0x49, 0x08, 0xe9, 0xc2, 0xc1, 0x42, 0xad, 0x6b, 0x59, 0x55, 0x1b, + 0x55, 0x2a, 0x48, 0xf4, 0xd4, 0xaa, 0x12, 0x2a, 0x29, 0x90, 0x08, 0xb5, 0x45, 0xcb, 0x9f, 0x4a, + 0xdc, 0x16, 0x7b, 0x42, 0x5c, 0x25, 0xb6, 0xeb, 0xdd, 0x94, 0xe6, 0x03, 0xf4, 0xd2, 0x4a, 0x88, + 0xaf, 0xd4, 0x5b, 0x8f, 0x1c, 0x39, 0xfe, 0x04, 0x5f, 0xe4, 0x27, 0xef, 0xc6, 0x89, 0xed, 0xe4, + 0xc0, 0x8d, 0x8b, 0xe3, 0x79, 0x9e, 0x79, 0xf3, 0xe6, 0xed, 0x9f, 0xc0, 0xa1, 0x17, 0x8d, 0xc7, + 0x51, 0x28, 0x62, 0xee, 0xe1, 0x7e, 0x74, 0xfb, 0x3b, 0x7a, 0x72, 0x5f, 0x26, 0x88, 0xea, 0xe1, + 0x0d, 0x79, 0x78, 0x87, 0x71, 0x12, 0xc9, 0x68, 0x5f, 0x3d, 0x45, 0x0e, 0xde, 0x53, 0x08, 0x85, + 0x05, 0xe2, 0x3e, 0x13, 0x00, 0x16, 0x45, 0xb2, 0xab, 0x42, 0xfa, 0x29, 0xd4, 0xb9, 0x37, 0xea, + 0x21, 0xf7, 0xfb, 0xbe, 0x45, 0x1c, 0xd2, 0xa9, 0xb3, 0x05, 0x40, 0x2d, 0xa8, 0xa9, 0xae, 0x7d, + 0xdf, 0x32, 0xd5, 0xb7, 0x2c, 0xa4, 0x36, 0x80, 0x26, 0xbc, 0x9c, 0xc6, 0x68, 0x55, 0xd4, 0xc7, + 0x1c, 0x92, 0xf2, 0xca, 0x60, 0x8c, 0x42, 0xf2, 0x71, 0x6c, 0x55, 0x1d, 0xd2, 0xa9, 0xb0, 0x05, + 0x40, 0x29, 0x54, 0x05, 0xa2, 0x6f, 0xad, 0x39, 0xa4, 0xd3, 0x64, 0xea, 0x9d, 0xee, 0xc2, 0x46, + 0xe0, 0x63, 0x28, 0x03, 0x39, 0xb5, 0xd6, 0x15, 0x3e, 0x8f, 0xe9, 0x17, 0xb0, 0xa9, 0xb9, 0xcf, + 0xf9, 0x74, 0x14, 0x71, 0xdf, 0xaa, 0xa9, 0x84, 0x22, 0xe8, 0x3e, 0x98, 0x00, 0x97, 0x09, 0xe2, + 0x6c, 0x34, 0x07, 0x1a, 0xe9, 0xdc, 0x7a, 0x14, 0x61, 0x11, 0xa7, 0xd2, 0xa9, 0xb3, 0x3c, 0x54, + 0x1c, 0xde, 0x2c, 0x0f, 0xff, 0x25, 0xb4, 0x44, 0xc8, 0x63, 0x31, 0x8c, 0xe4, 0x11, 0x17, 0xa9, + 0x07, 0x7a, 0xcc, 0x12, 0x9a, 0xf6, 0xd1, 0x3a, 0xc4, 0x4f, 0x5c, 0x72, 0x35, 0x6c, 0x93, 0xe5, + 0xa1, 0xb4, 0x4f, 0x82, 0xdc, 0x3f, 0xc3, 0x69, 0x5f, 0xcf, 0x5c, 0x67, 0x0b, 0xa0, 0x68, 0xd5, + 0x7a, 0xd9, 0xaa, 0xbc, 0x2d, 0xb5, 0x92, 0x2d, 0x36, 0x40, 0x20, 0x2e, 0x66, 0x6a, 0xac, 0x0d, + 0x87, 0x74, 0x36, 0x58, 0x0e, 0x71, 0x4f, 0x61, 0x93, 0xf1, 0xfb, 0x9c, 0x25, 0x16, 0xd4, 0xe2, + 0x99, 0x83, 0x44, 0x71, 0x65, 0x61, 0x2a, 0x42, 0x04, 0x77, 0x21, 0x97, 0x93, 0x04, 0x95, 0x15, + 0x4d, 0xb6, 0x00, 0xdc, 0x2e, 0x6c, 0x17, 0x88, 0x7e, 0x0b, 0xe4, 0x50, 0x2b, 0x4f, 0xf8, 0xbd, + 0x86, 0x66, 0x84, 0x0b, 0x80, 0xb6, 0xc0, 0x0c, 0x32, 0x5b, 0xcd, 0xc0, 0x77, 0x1f, 0x08, 0x6c, + 0xa5, 0x14, 0x17, 0xd3, 0xd0, 0xfb, 0x19, 0x85, 0xe0, 0x77, 0x48, 0xbf, 0x87, 0x9a, 0x17, 0x85, + 0x12, 0x43, 0xa9, 0xea, 0x1b, 0x07, 0xce, 0x5e, 0x6e, 0xf7, 0x66, 0xd9, 0x5d, 0x9d, 0x72, 0xcd, + 0x47, 0x13, 0x64, 0x59, 0x01, 0x3d, 0x04, 0x48, 0xe6, 0x1b, 0x59, 0xf5, 0x69, 0x1c, 0x7c, 0x9e, + 0x2f, 0x5f, 0x21, 0x99, 0xe5, 0x4a, 0xdc, 0xff, 0x4c, 0xd8, 0x59, 0xd5, 0x82, 0xfe, 0x00, 0x30, + 0x44, 0xee, 0x5f, 0xc5, 0x3e, 0x97, 0x38, 0x13, 0xb6, 0x5b, 0x16, 0xd6, 0x9b, 0x67, 0xf4, 0x0c, + 0x96, 0xcb, 0xa7, 0x67, 0xb0, 0x35, 0x98, 0x8c, 0x46, 0x29, 0x2b, 0xc3, 0x3f, 0x26, 0x28, 0xe4, + 0x2a, 0x71, 0x29, 0xc5, 0x49, 0x31, 0xad, 0x67, 0xb0, 0x72, 0x25, 0xfd, 0x05, 0xda, 0x0b, 0x48, + 0xc4, 0x51, 0x28, 0xf4, 0x69, 0x5b, 0xe1, 0xd4, 0x49, 0x29, 0xaf, 0x67, 0xb0, 0xa5, 0x5a, 0x7a, + 0x0c, 0x9b, 0x98, 0x24, 0x51, 0x32, 0x27, 0xab, 0x2a, 0xb2, 0xcf, 0xca, 0x64, 0xc7, 0xf9, 0xa4, + 0x9e, 0xc1, 0x8a, 0x55, 0x47, 0x35, 0x58, 0xfb, 0x33, 0xb5, 0xca, 0xfd, 0x9b, 0x40, 0xab, 0xe8, + 0x06, 0xdd, 0x81, 0xb5, 0xd4, 0x8d, 0xec, 0xc4, 0xe9, 0x80, 0x7e, 0x07, 0xb5, 0xd9, 0x91, 0xb0, + 0x4c, 0xa7, 0xf2, 0x96, 0xa5, 0xca, 0xf2, 0xa9, 0x0b, 0xcd, 0xec, 0xc8, 0x9d, 0x73, 0x39, 0xb4, + 0x2a, 0x8a, 0xb7, 0x80, 0xb9, 0xff, 0x10, 0xd8, 0x5e, 0x61, 0xe9, 0xfb, 0x88, 0xf9, 0x97, 0xe8, + 0x8d, 0x55, 0x5e, 0x91, 0xf7, 0x51, 0xf3, 0x0d, 0x7c, 0xb2, 0xb4, 0xa2, 0xe9, 0x4d, 0x80, 0x49, + 0xd2, 0x8d, 0x7c, 0xbd, 0xbf, 0xab, 0x2c, 0x0b, 0xdd, 0x6b, 0xbd, 0xa0, 0xba, 0x5f, 0x3f, 0x1c, + 0x44, 0xa5, 0xbb, 0x9e, 0x2c, 0xdd, 0xf5, 0x4b, 0xb7, 0xb3, 0xb9, 0xe2, 0x76, 0xfe, 0xfa, 0x06, + 0x40, 0x49, 0x48, 0x9b, 0x08, 0xda, 0x02, 0xb8, 0x0a, 0xf1, 0xaf, 0x18, 0x3d, 0x89, 0x7e, 0xdb, + 0xa0, 0x6d, 0x68, 0x9e, 0xa2, 0x9c, 0xeb, 0x6c, 0x13, 0x6a, 0xc1, 0x4e, 0x69, 0x31, 0xf5, 0x17, + 0x93, 0xb6, 0xa1, 0xa1, 0x5e, 0x7f, 0x1d, 0x0c, 0x04, 0xca, 0xf6, 0x63, 0xe5, 0xe8, 0xc7, 0xff, + 0x5f, 0x6c, 0xf2, 0xf4, 0x62, 0x93, 0x0f, 0x2f, 0x36, 0x79, 0x7c, 0xb5, 0x8d, 0xa7, 0x57, 0xdb, + 0x78, 0x7e, 0xb5, 0x8d, 0x9b, 0xaf, 0xde, 0xf8, 0xdf, 0x79, 0xbb, 0xae, 0x7e, 0xbe, 0xfd, 0x18, + 0x00, 0x00, 0xff, 0xff, 0xb5, 0x68, 0xf7, 0x3f, 0x6d, 0x07, 0x00, 0x00, } func (m *RootChange) Marshal() (dAtA []byte, err error) { diff --git a/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go b/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go index f378bc43..68f1d579 100644 --- a/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go +++ b/commonspace/object/treemanager/mock_treemanager/mock_treemanager.go @@ -9,9 +9,7 @@ import ( reflect "reflect" app "github.com/anytypeio/any-sync/app" - list "github.com/anytypeio/any-sync/commonspace/object/acl/list" objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" - treestorage "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" gomock "github.com/golang/mock/gomock" ) @@ -52,20 +50,6 @@ func (mr *MockTreeManagerMockRecorder) Close(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTreeManager)(nil).Close), arg0) } -// DeleteSpace mocks base method. -func (m *MockTreeManager) DeleteSpace(arg0 context.Context, arg1 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteSpace", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteSpace indicates an expected call of DeleteSpace. -func (mr *MockTreeManagerMockRecorder) DeleteSpace(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSpace", reflect.TypeOf((*MockTreeManager)(nil).DeleteSpace), arg0, arg1) -} - // DeleteTree mocks base method. func (m *MockTreeManager) DeleteTree(arg0 context.Context, arg1, arg2 string) error { m.ctrl.T.Helper() @@ -123,20 +107,6 @@ func (mr *MockTreeManagerMockRecorder) Name() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockTreeManager)(nil).Name)) } -// ObjectTreeBuilder mocks base method. -func (m *MockTreeManager) ObjectTreeBuilder() func(treestorage.TreeStorage, list.AclList) (objecttree.ObjectTree, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ObjectTreeBuilder") - ret0, _ := ret[0].(func(treestorage.TreeStorage, list.AclList) (objecttree.ObjectTree, error)) - return ret0 -} - -// ObjectTreeBuilder indicates an expected call of ObjectTreeBuilder. -func (mr *MockTreeManagerMockRecorder) ObjectTreeBuilder() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ObjectTreeBuilder", reflect.TypeOf((*MockTreeManager)(nil).ObjectTreeBuilder)) -} - // Run mocks base method. func (m *MockTreeManager) Run(arg0 context.Context) error { m.ctrl.T.Helper() diff --git a/commonspace/object/treemanager/treemanager.go b/commonspace/object/treemanager/treemanager.go index 00685ff5..0573471e 100644 --- a/commonspace/object/treemanager/treemanager.go +++ b/commonspace/object/treemanager/treemanager.go @@ -13,6 +13,4 @@ type TreeManager interface { app.ComponentRunnable GetTree(ctx context.Context, spaceId, treeId string) (objecttree.ObjectTree, error) DeleteTree(ctx context.Context, spaceId, treeId string) error - DeleteSpace(ctx context.Context, spaceId string) error - ObjectTreeBuilder() objecttree.BuildObjectTreeFunc } diff --git a/commonspace/objectsync/mock_objectsync/mock_objectsync.go b/commonspace/objectsync/mock_objectsync/mock_objectsync.go new file mode 100644 index 00000000..b82360db --- /dev/null +++ b/commonspace/objectsync/mock_objectsync/mock_objectsync.go @@ -0,0 +1,138 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/anytypeio/any-sync/commonspace/objectsync (interfaces: SyncClient) + +// Package mock_objectsync is a generated GoMock package. +package mock_objectsync + +import ( + context "context" + reflect "reflect" + + objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + treechangeproto "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + peermanager "github.com/anytypeio/any-sync/commonspace/peermanager" + gomock "github.com/golang/mock/gomock" +) + +// MockSyncClient is a mock of SyncClient interface. +type MockSyncClient struct { + ctrl *gomock.Controller + recorder *MockSyncClientMockRecorder +} + +// MockSyncClientMockRecorder is the mock recorder for MockSyncClient. +type MockSyncClientMockRecorder struct { + mock *MockSyncClient +} + +// NewMockSyncClient creates a new mock instance. +func NewMockSyncClient(ctrl *gomock.Controller) *MockSyncClient { + mock := &MockSyncClient{ctrl: ctrl} + mock.recorder = &MockSyncClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSyncClient) EXPECT() *MockSyncClientMockRecorder { + return m.recorder +} + +// Broadcast mocks base method. +func (m *MockSyncClient) Broadcast(arg0 context.Context, arg1 *treechangeproto.TreeSyncMessage) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Broadcast", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Broadcast indicates an expected call of Broadcast. +func (mr *MockSyncClientMockRecorder) Broadcast(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Broadcast", reflect.TypeOf((*MockSyncClient)(nil).Broadcast), arg0, arg1) +} + +// CreateFullSyncRequest mocks base method. +func (m *MockSyncClient) CreateFullSyncRequest(arg0 objecttree.ObjectTree, arg1, arg2 []string) (*treechangeproto.TreeSyncMessage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateFullSyncRequest", arg0, arg1, arg2) + ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateFullSyncRequest indicates an expected call of CreateFullSyncRequest. +func (mr *MockSyncClientMockRecorder) CreateFullSyncRequest(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFullSyncRequest", reflect.TypeOf((*MockSyncClient)(nil).CreateFullSyncRequest), arg0, arg1, arg2) +} + +// CreateFullSyncResponse mocks base method. +func (m *MockSyncClient) CreateFullSyncResponse(arg0 objecttree.ObjectTree, arg1, arg2 []string) (*treechangeproto.TreeSyncMessage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateFullSyncResponse", arg0, arg1, arg2) + ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateFullSyncResponse indicates an expected call of CreateFullSyncResponse. +func (mr *MockSyncClientMockRecorder) CreateFullSyncResponse(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateFullSyncResponse", reflect.TypeOf((*MockSyncClient)(nil).CreateFullSyncResponse), arg0, arg1, arg2) +} + +// CreateHeadUpdate mocks base method. +func (m *MockSyncClient) CreateHeadUpdate(arg0 objecttree.ObjectTree, arg1 []*treechangeproto.RawTreeChangeWithId) *treechangeproto.TreeSyncMessage { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateHeadUpdate", arg0, arg1) + ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) + return ret0 +} + +// CreateHeadUpdate indicates an expected call of CreateHeadUpdate. +func (mr *MockSyncClientMockRecorder) CreateHeadUpdate(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateHeadUpdate", reflect.TypeOf((*MockSyncClient)(nil).CreateHeadUpdate), arg0, arg1) +} + +// CreateNewTreeRequest mocks base method. +func (m *MockSyncClient) CreateNewTreeRequest() *treechangeproto.TreeSyncMessage { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateNewTreeRequest") + ret0, _ := ret[0].(*treechangeproto.TreeSyncMessage) + return ret0 +} + +// CreateNewTreeRequest indicates an expected call of CreateNewTreeRequest. +func (mr *MockSyncClientMockRecorder) CreateNewTreeRequest() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateNewTreeRequest", reflect.TypeOf((*MockSyncClient)(nil).CreateNewTreeRequest)) +} + +// PeerManager mocks base method. +func (m *MockSyncClient) PeerManager() peermanager.PeerManager { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PeerManager") + ret0, _ := ret[0].(peermanager.PeerManager) + return ret0 +} + +// PeerManager indicates an expected call of PeerManager. +func (mr *MockSyncClientMockRecorder) PeerManager() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PeerManager", reflect.TypeOf((*MockSyncClient)(nil).PeerManager)) +} + +// SendWithReply mocks base method. +func (m *MockSyncClient) SendWithReply(arg0 context.Context, arg1 string, arg2 *treechangeproto.TreeSyncMessage, arg3 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendWithReply", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendWithReply indicates an expected call of SendWithReply. +func (mr *MockSyncClientMockRecorder) SendWithReply(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendWithReply", reflect.TypeOf((*MockSyncClient)(nil).SendWithReply), arg0, arg1, arg2, arg3) +} diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index 450a1ce5..6cd39695 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -1,8 +1,8 @@ +//go:generate mockgen -destination mock_objectsync/mock_objectsync.go github.com/anytypeio/any-sync/commonspace/objectsync SyncClient package objectsync import ( "context" - "github.com/anytypeio/any-sync/commonspace/object/tree/synctree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/gogo/protobuf/proto" "sync/atomic" @@ -33,7 +33,7 @@ type objectSync struct { spaceId string messagePool MessagePool - syncClient synctree.SyncClient + syncClient SyncClient objectGetter syncobjectgetter.SyncObjectGetter configuration nodeconf.Configuration spaceStorage spacestorage.SpaceStorage @@ -59,7 +59,7 @@ func NewObjectSync( cancelSync: cancel, spaceIsDeleted: spaceIsDeleted, configuration: configuration, - syncClient: synctree.NewSyncClient(spaceId, peerManager, synctree.GetRequestFactory()), + syncClient: NewSyncClient(spaceId, peerManager, GetRequestFactory()), } os.messagePool = newMessagePool(peerManager, os.handleMessage) return os @@ -90,7 +90,7 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp log.With(zap.String("objectId", msg.ObjectId), zap.String("replyId", msg.ReplyId)).DebugCtx(ctx, "handling message") obj, err := s.objectGetter.GetObject(ctx, msg.ObjectId) if err != nil { - respErr := s.sendErrorResponse(ctx, msg, senderId, err) + respErr := s.sendErrorResponse(ctx, msg, senderId) if respErr != nil { log.Debug("failed to send error response", zap.Error(respErr)) } @@ -103,12 +103,12 @@ func (s *objectSync) MessagePool() MessagePool { return s.messagePool } -func (s *objectSync) sendErrorResponse(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, senderId string, respErr error) (err error) { +func (s *objectSync) sendErrorResponse(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, senderId string) (err error) { unmarshalled := &treechangeproto.TreeSyncMessage{} err = proto.Unmarshal(msg.Payload, unmarshalled) if err != nil { return } - resp := treechangeproto.WrapError(respErr, unmarshalled.RootChange) + resp := treechangeproto.WrapError(treechangeproto.ErrorCodes_GetTreeError, unmarshalled.RootChange) return s.syncClient.SendWithReply(ctx, senderId, resp, msg.ReplyId) } diff --git a/commonspace/object/tree/synctree/requestfactory.go b/commonspace/objectsync/requestfactory.go similarity index 99% rename from commonspace/object/tree/synctree/requestfactory.go rename to commonspace/objectsync/requestfactory.go index 6fc5202c..0608ad33 100644 --- a/commonspace/object/tree/synctree/requestfactory.go +++ b/commonspace/objectsync/requestfactory.go @@ -1,4 +1,4 @@ -package synctree +package objectsync import ( "fmt" diff --git a/commonspace/object/tree/synctree/syncclient.go b/commonspace/objectsync/syncclient.go similarity index 70% rename from commonspace/object/tree/synctree/syncclient.go rename to commonspace/objectsync/syncclient.go index 5967441b..6ef203c0 100644 --- a/commonspace/object/tree/synctree/syncclient.go +++ b/commonspace/objectsync/syncclient.go @@ -1,5 +1,4 @@ -//go:generate mockgen -destination mock_synctree/mock_synctree.go github.com/anytypeio/any-sync/commonspace/object/tree/synctree SyncClient,SyncTree,ReceiveQueue,HeadNotifiable -package synctree +package objectsync import ( "context" @@ -12,12 +11,13 @@ type SyncClient interface { RequestFactory Broadcast(ctx context.Context, msg *treechangeproto.TreeSyncMessage) (err error) SendWithReply(ctx context.Context, peerId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) + PeerManager() peermanager.PeerManager } type syncClient struct { - peermanager.PeerManager RequestFactory - spaceId string + spaceId string + peerManager peermanager.PeerManager } func NewSyncClient( @@ -25,29 +25,33 @@ func NewSyncClient( peerManager peermanager.PeerManager, factory RequestFactory) SyncClient { return &syncClient{ - PeerManager: peerManager, + peerManager: peerManager, RequestFactory: factory, spaceId: spaceId, } } func (s *syncClient) Broadcast(ctx context.Context, msg *treechangeproto.TreeSyncMessage) (err error) { - objMsg, err := marshallTreeMessage(msg, s.spaceId, msg.RootChange.Id, "") + objMsg, err := MarshallTreeMessage(msg, s.spaceId, msg.RootChange.Id, "") if err != nil { return } - return s.PeerManager.Broadcast(ctx, objMsg) + return s.peerManager.Broadcast(ctx, objMsg) } func (s *syncClient) SendWithReply(ctx context.Context, peerId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) { - objMsg, err := marshallTreeMessage(msg, s.spaceId, msg.RootChange.Id, replyId) + objMsg, err := MarshallTreeMessage(msg, s.spaceId, msg.RootChange.Id, replyId) if err != nil { return } - return s.PeerManager.SendPeer(ctx, peerId, objMsg) + return s.peerManager.SendPeer(ctx, peerId, objMsg) } -func marshallTreeMessage(message *treechangeproto.TreeSyncMessage, spaceId, objectId, replyId string) (objMsg *spacesyncproto.ObjectSyncMessage, err error) { +func (s *syncClient) PeerManager() peermanager.PeerManager { + return s.peerManager +} + +func MarshallTreeMessage(message *treechangeproto.TreeSyncMessage, spaceId, objectId, replyId string) (objMsg *spacesyncproto.ObjectSyncMessage, err error) { payload, err := message.Marshal() if err != nil { return diff --git a/commonspace/settings/deletionmanager.go b/commonspace/settings/deletionmanager.go index 366c2686..1e357cdf 100644 --- a/commonspace/settings/deletionmanager.go +++ b/commonspace/settings/deletionmanager.go @@ -55,10 +55,6 @@ func (d *deletionManager) UpdateState(ctx context.Context, state *settingsstate. return nil } log.Debug("deleting space") - err = d.treeManager.DeleteSpace(ctx, d.spaceId) - if err != nil { - log.Debug("failed to notify on space deletion", zap.Error(err)) - } if d.isResponsible { allIds := slice.DiscardFromSlice(d.provider.AllIds(), func(id string) bool { return id == d.settingsId diff --git a/commonspace/settings/deletionmanager_test.go b/commonspace/settings/deletionmanager_test.go index 990efc07..31aff516 100644 --- a/commonspace/settings/deletionmanager_test.go +++ b/commonspace/settings/deletionmanager_test.go @@ -30,7 +30,6 @@ func TestDeletionManager_UpdateState_NotResponsible(t *testing.T) { treeManager := mock_treemanager.NewMockTreeManager(ctrl) delState.EXPECT().Add(state.DeletedIds).Return(nil) - treeManager.EXPECT().DeleteSpace(ctx, spaceId).Return(nil) delManager := newDeletionManager(spaceId, settingsId, @@ -64,7 +63,6 @@ func TestDeletionManager_UpdateState_Responsible(t *testing.T) { provider := mock_settings.NewMockSpaceIdsProvider(ctrl) delState.EXPECT().Add(state.DeletedIds).Return(nil) - treeManager.EXPECT().DeleteSpace(ctx, spaceId).Return(nil) provider.EXPECT().AllIds().Return([]string{"id", "otherId", settingsId}) delState.EXPECT().Add([]string{"id", "otherId"}).Return(nil) delManager := newDeletionManager(spaceId, diff --git a/commonspace/space.go b/commonspace/space.go index f69b4aaa..de8c8f76 100644 --- a/commonspace/space.go +++ b/commonspace/space.go @@ -124,6 +124,7 @@ type space struct { configuration nodeconf.Configuration settingsObject settings.SettingsObject peerManager peermanager.PeerManager + treeBuilder objecttree.BuildObjectTreeFunc handleQueue multiqueue.MultiQueue[HandleMessage] @@ -292,7 +293,7 @@ func (s *space) PutTree(ctx context.Context, payload treestorage.TreeStorageCrea OnClose: s.onObjectClose, SyncStatus: s.syncStatus, PeerGetter: s.peerManager, - BuildObjectTree: s.treeManager.ObjectTreeBuilder(), + BuildObjectTree: s.treeBuilder, } t, err = synctree.PutSyncTree(ctx, payload, deps) if err != nil { @@ -331,7 +332,7 @@ func (s *space) BuildTree(ctx context.Context, id string, opts BuildTreeOpts) (t SyncStatus: s.syncStatus, WaitTreeRemoteSync: opts.WaitTreeRemoteSync, PeerGetter: s.peerManager, - BuildObjectTree: s.treeManager.ObjectTreeBuilder(), + BuildObjectTree: s.treeBuilder, } if t, err = synctree.BuildSyncTreeOrGetRemote(ctx, id, deps); err != nil { return nil, err diff --git a/commonspace/spaceservice.go b/commonspace/spaceservice.go index cfe33692..69daef67 100644 --- a/commonspace/spaceservice.go +++ b/commonspace/spaceservice.go @@ -8,6 +8,7 @@ import ( "github.com/anytypeio/any-sync/commonspace/credentialprovider" "github.com/anytypeio/any-sync/commonspace/headsync" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/treemanager" "github.com/anytypeio/any-sync/commonspace/objectsync" @@ -152,6 +153,12 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) { // TODO: move it to the client package and add possibility to inject StatusProvider from the client syncStatus = syncstatus.NewSyncStatusProvider(st.Id(), syncstatus.DefaultDeps(lastConfiguration, st)) } + var builder objecttree.BuildObjectTreeFunc + if s.config.TreeInMemoryData { + builder = objecttree.BuildEmptyDataObjectTree + } else { + builder = objecttree.BuildObjectTree + } peerManager, err := s.peermanagerProvider.NewPeerManager(ctx, id) if err != nil { @@ -171,6 +178,7 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) { peerManager: peerManager, storage: st, treesUsed: &atomic.Int32{}, + treeBuilder: builder, isClosed: spaceIsClosed, isDeleted: spaceIsDeleted, } From 9acec773faafc572d018153dc84a6762c9d07f4a Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 19 Apr 2023 11:49:04 +0200 Subject: [PATCH 14/45] Add comments to test --- .../object/tree/synctree/syncprotocol_test.go | 4 +- .../object/tree/synctree/utils_test.go | 180 ++++++++++-------- 2 files changed, 99 insertions(+), 85 deletions(-) diff --git a/commonspace/object/tree/synctree/syncprotocol_test.go b/commonspace/object/tree/synctree/syncprotocol_test.go index 765e3235..24a3e535 100644 --- a/commonspace/object/tree/synctree/syncprotocol_test.go +++ b/commonspace/object/tree/synctree/syncprotocol_test.go @@ -32,7 +32,7 @@ func TestEmptyClientGetsFullHistory(t *testing.T) { }, emptyTrees: []string{"peer2"}, } - fx := newProcessFixture(t, spaceId, deps) + fx := newProtocolFixture(t, spaceId, deps) fx.run(t) fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ NewHeads: nil, @@ -111,7 +111,7 @@ func testTreeMerge(t *testing.T, levels, perlevel int, isSnapshot func() bool) { }, emptyTrees: []string{"peer2", "node1"}, } - fx := newProcessFixture(t, spaceId, deps) + fx := newProtocolFixture(t, spaceId, deps) fx.run(t) fx.handlers["peer1"].sendRawChanges(context.Background(), objecttree.RawChangesPayload{ NewHeads: initialRes.heads, diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index 8e2ffbe9..9d2c4f68 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -22,13 +22,15 @@ import ( "time" ) -type processMsg struct { +// protocolMsg is a message used in sync protocol tests +type protocolMsg struct { msg *spacesyncproto.ObjectSyncMessage senderId string receiverId string userMsg *objecttree.RawChangesPayload } +// msgDescription is a representation of message used for checking the results of the test type msgDescription struct { name string from string @@ -37,7 +39,7 @@ type msgDescription struct { changes []*treechangeproto.RawTreeChangeWithId } -func (p *processMsg) description() (descr msgDescription) { +func (p *protocolMsg) description() (descr msgDescription) { unmarshalled := &treechangeproto.TreeSyncMessage{} err := proto.Unmarshal(p.msg.Payload, unmarshalled) if err != nil { @@ -67,30 +69,68 @@ func (p *processMsg) description() (descr msgDescription) { return } +// messageLog saves all messages that were sent during sync test type messageLog struct { - batcher *mb.MB[processMsg] + batcher *mb.MB[protocolMsg] } func newMessageLog() *messageLog { - return &messageLog{batcher: mb.New[processMsg](0)} + return &messageLog{batcher: mb.New[protocolMsg](0)} } -func (m *messageLog) addMessage(msg processMsg) { +func (m *messageLog) addMessage(msg protocolMsg) { m.batcher.Add(context.Background(), msg) } -type processSyncHandler struct { +// testSyncHandler is the wrapper around individual tree to test sync protocol +type testSyncHandler struct { synchandler.SyncHandler - batcher *mb.MB[processMsg] + batcher *mb.MB[protocolMsg] peerId string aclList list.AclList log *messageLog syncClient objectsync.SyncClient } -func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { - if p.SyncHandler != nil { - return p.SyncHandler.HandleMessage(ctx, senderId, request) +// createSyncHandler creates a sync handler when a tree is already created +func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *testSyncHandler { + factory := objectsync.GetRequestFactory() + syncClient := objectsync.NewSyncClient(spaceId, newTestPeerManager(peerId, log), factory) + netTree := &broadcastTree{ + ObjectTree: objTree, + SyncClient: syncClient, + } + handler := newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) + return newTestSyncHandler(peerId, handler) +} + +// createEmptySyncHandler creates a sync handler when the tree will be provided later (this emulates the situation when we have no tree) +func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *testSyncHandler { + factory := objectsync.GetRequestFactory() + syncClient := objectsync.NewSyncClient(spaceId, newTestPeerManager(peerId, log), factory) + + batcher := mb.New[protocolMsg](0) + return &testSyncHandler{ + batcher: batcher, + peerId: peerId, + aclList: aclList, + log: log, + syncClient: syncClient, + } +} + +func newTestSyncHandler(peerId string, syncHandler synchandler.SyncHandler) *testSyncHandler { + batcher := mb.New[protocolMsg](0) + return &testSyncHandler{ + SyncHandler: syncHandler, + batcher: batcher, + peerId: peerId, + } +} + +func (h *testSyncHandler) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { + if h.SyncHandler != nil { + return h.SyncHandler.HandleMessage(ctx, senderId, request) } unmarshalled := &treechangeproto.TreeSyncMessage{} err = proto.Unmarshal(request.Payload, unmarshalled) @@ -104,17 +144,17 @@ func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, if err != nil { return } - return p.manager().SendPeer(context.Background(), senderId, objMsg) + return h.manager().SendPeer(context.Background(), senderId, objMsg) } fullSyncResponse := unmarshalled.Content.GetFullSyncResponse() treeStorage, _ := treestorage.NewInMemoryTreeStorage(unmarshalled.RootChange, []string{unmarshalled.RootChange.Id}, nil) - tree, err := createTestTree(p.aclList, treeStorage) + tree, err := createTestTree(h.aclList, treeStorage) if err != nil { return } netTree := &broadcastTree{ ObjectTree: tree, - SyncClient: p.syncClient, + SyncClient: h.syncClient, } res, err := netTree.AddRawChanges(context.Background(), objecttree.RawChangesPayload{ NewHeads: fullSyncResponse.Heads, @@ -123,62 +163,53 @@ func (p *processSyncHandler) HandleMessage(ctx context.Context, senderId string, if err != nil { return } - p.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, p.syncClient, syncstatus.NewNoOpSyncStatus()) + h.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, h.syncClient, syncstatus.NewNoOpSyncStatus()) var objMsg *spacesyncproto.ObjectSyncMessage newTreeRequest := objectsync.GetRequestFactory().CreateHeadUpdate(netTree, res.Added) objMsg, err = objectsync.MarshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") if err != nil { return } - return p.manager().Broadcast(context.Background(), objMsg) + return h.manager().Broadcast(context.Background(), objMsg) } -func newProcessSyncHandler(peerId string, syncHandler synchandler.SyncHandler) *processSyncHandler { - batcher := mb.New[processMsg](0) - return &processSyncHandler{ - SyncHandler: syncHandler, - batcher: batcher, - peerId: peerId, +func (h *testSyncHandler) manager() *testPeerManager { + if h.SyncHandler != nil { + return h.SyncHandler.(*syncTreeHandler).syncClient.PeerManager().(*testPeerManager) } + return h.syncClient.PeerManager().(*testPeerManager) } -func (p *processSyncHandler) manager() *processPeerManager { - if p.SyncHandler != nil { - return p.SyncHandler.(*syncTreeHandler).syncClient.PeerManager().(*processPeerManager) - } - return p.syncClient.PeerManager().(*processPeerManager) +func (h *testSyncHandler) tree() *broadcastTree { + return h.SyncHandler.(*syncTreeHandler).objTree.(*broadcastTree) } -func (p *processSyncHandler) tree() *broadcastTree { - return p.SyncHandler.(*syncTreeHandler).objTree.(*broadcastTree) +func (h *testSyncHandler) send(ctx context.Context, msg protocolMsg) (err error) { + return h.batcher.Add(ctx, msg) } -func (p *processSyncHandler) send(ctx context.Context, msg processMsg) (err error) { - return p.batcher.Add(ctx, msg) +func (h *testSyncHandler) sendRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) { + h.batcher.Add(ctx, protocolMsg{userMsg: &changes}) } -func (p *processSyncHandler) sendRawChanges(ctx context.Context, changes objecttree.RawChangesPayload) { - p.batcher.Add(ctx, processMsg{userMsg: &changes}) -} - -func (p *processSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.WaitGroup) { +func (h *testSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.WaitGroup) { wg.Add(1) go func() { defer wg.Done() for { - res, err := p.batcher.WaitOne(ctx) + res, err := h.batcher.WaitOne(ctx) if err != nil { return } if res.userMsg != nil { - p.tree().Lock() - userRes, err := p.tree().AddRawChanges(ctx, *res.userMsg) + h.tree().Lock() + userRes, err := h.tree().AddRawChanges(ctx, *res.userMsg) require.NoError(t, err) fmt.Println("user add result", userRes.Heads) - p.tree().Unlock() + h.tree().Unlock() continue } - err = p.HandleMessage(ctx, res.senderId, res.msg) + err = h.HandleMessage(ctx, res.senderId, res.msg) if err != nil { fmt.Println("error handling message", err.Error()) continue @@ -187,22 +218,23 @@ func (p *processSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.Wai }() } -type processPeerManager struct { +// testPeerManager captures all other handlers and sends messages to them +type testPeerManager struct { peerId string - handlers map[string]*processSyncHandler + handlers map[string]*testSyncHandler log *messageLog } -func newProcessPeerManager(peerId string, log *messageLog) *processPeerManager { - return &processPeerManager{handlers: map[string]*processSyncHandler{}, peerId: peerId, log: log} +func newTestPeerManager(peerId string, log *messageLog) *testPeerManager { + return &testPeerManager{handlers: map[string]*testSyncHandler{}, peerId: peerId, log: log} } -func (m *processPeerManager) addHandler(peerId string, handler *processSyncHandler) { +func (m *testPeerManager) addHandler(peerId string, handler *testSyncHandler) { m.handlers[peerId] = handler } -func (m *processPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { - pMsg := processMsg{ +func (m *testPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { + pMsg := protocolMsg{ msg: msg, senderId: m.peerId, receiverId: peerId, @@ -211,9 +243,9 @@ func (m *processPeerManager) SendPeer(ctx context.Context, peerId string, msg *s return m.handlers[peerId].send(context.Background(), pMsg) } -func (m *processPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { +func (m *testPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { for _, handler := range m.handlers { - pMsg := processMsg{ + pMsg := protocolMsg{ msg: msg, senderId: m.peerId, receiverId: handler.peerId, @@ -224,10 +256,12 @@ func (m *processPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto. return } -func (m *processPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { +func (m *testPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { panic("should not be called") } +// broadcastTree is the tree that broadcasts changes to everyone when changes are added +// it is a simplified version of SyncTree which is easier to use in the test environment type broadcastTree struct { objecttree.ObjectTree objectsync.SyncClient @@ -243,31 +277,6 @@ func (b *broadcastTree) AddRawChanges(ctx context.Context, changes objecttree.Ra return res, nil } -func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *processSyncHandler { - factory := objectsync.GetRequestFactory() - syncClient := objectsync.NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) - netTree := &broadcastTree{ - ObjectTree: objTree, - SyncClient: syncClient, - } - handler := newSyncTreeHandler(spaceId, netTree, syncClient, syncstatus.NewNoOpSyncStatus()) - return newProcessSyncHandler(peerId, handler) -} - -func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *processSyncHandler { - factory := objectsync.GetRequestFactory() - syncClient := objectsync.NewSyncClient(spaceId, newProcessPeerManager(peerId, log), factory) - - batcher := mb.New[processMsg](0) - return &processSyncHandler{ - batcher: batcher, - peerId: peerId, - aclList: aclList, - log: log, - syncClient: syncClient, - } -} - func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage { changeCreator := objecttree.NewMockChangeCreator() st := changeCreator.CreateNewTreeStorage(treeId, aclList.Head().Id) @@ -285,24 +294,25 @@ type fixtureDeps struct { emptyTrees []string } -type processFixture struct { - handlers map[string]*processSyncHandler +// protocolFixture is the test environment for sync protocol tests +type protocolFixture struct { + handlers map[string]*testSyncHandler log *messageLog wg *sync.WaitGroup ctx context.Context cancel context.CancelFunc } -func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processFixture { +func newProtocolFixture(t *testing.T, spaceId string, deps fixtureDeps) *protocolFixture { var ( - handlers = map[string]*processSyncHandler{} + handlers = map[string]*testSyncHandler{} log = newMessageLog() wg = sync.WaitGroup{} ctx, cancel = context.WithCancel(context.Background()) ) for peerId := range deps.connectionMap { - var handler *processSyncHandler + var handler *testSyncHandler if slices.Contains(deps.emptyTrees, peerId) { handler = createEmptySyncHandler(peerId, spaceId, deps.aclList, log) } else { @@ -320,7 +330,7 @@ func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processF manager.addHandler(connectionId, handlers[connectionId]) } } - return &processFixture{ + return &protocolFixture{ handlers: handlers, log: log, wg: &wg, @@ -329,18 +339,20 @@ func newProcessFixture(t *testing.T, spaceId string, deps fixtureDeps) *processF } } -func (p *processFixture) run(t *testing.T) { +func (p *protocolFixture) run(t *testing.T) { for _, handler := range p.handlers { handler.run(p.ctx, t, p.wg) } } -func (p *processFixture) stop() { +func (p *protocolFixture) stop() { p.cancel() p.wg.Wait() } +// genParams is the parameters for genChanges type genParams struct { + // prefix is the prefix which is added to change id prefix string aclId string startIdx int @@ -351,12 +363,14 @@ type genParams struct { isSnapshot func() bool } +// genResult is the result of genChanges type genResult struct { changes []*treechangeproto.RawTreeChangeWithId heads []string snapshotId string } +// genChanges generates several levels of tree changes where each level is connected with only previous one func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res genResult) { src := rand.NewSource(time.Now().Unix()) rnd := rand.New(src) From 34865d0e686e182fdb1d8ca62ed32f5104e9b17a Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 19 Apr 2023 11:52:11 +0200 Subject: [PATCH 15/45] Further test comments and changes --- commonspace/object/tree/synctree/syncprotocol_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/commonspace/object/tree/synctree/syncprotocol_test.go b/commonspace/object/tree/synctree/syncprotocol_test.go index 24a3e535..cc7ebec4 100644 --- a/commonspace/object/tree/synctree/syncprotocol_test.go +++ b/commonspace/object/tree/synctree/syncprotocol_test.go @@ -44,7 +44,7 @@ func TestEmptyClientGetsFullHistory(t *testing.T) { fx.stop() firstHeads := fx.handlers["peer1"].tree().Heads() secondHeads := fx.handlers["peer2"].tree().Heads() - require.True(t, slice.SortedEquals(firstHeads, secondHeads)) + require.True(t, slice.UnsortedEquals(firstHeads, secondHeads)) require.Equal(t, []string{"1"}, firstHeads) logMsgs := fx.log.batcher.GetAll() @@ -98,6 +98,7 @@ func testTreeMerge(t *testing.T, levels, perlevel int, isSnapshot func() bool) { prevHeads: []string{treeId}, isSnapshot: isSnapshot, } + // generating initial tree initialRes := genChanges(changeCreator, params) err = storage.TransactionAdd(initialRes.changes, initialRes.heads) require.NoError(t, err) @@ -132,6 +133,7 @@ func testTreeMerge(t *testing.T, levels, perlevel int, isSnapshot func() bool) { prevHeads: initialRes.heads, isSnapshot: isSnapshot, } + // generating different additions to the tree for different peers peer1Res := genChanges(changeCreator, params) params.prefix = "peer2" peer2Res := genChanges(changeCreator, params) From aeaad8ab3596f53444d132d60401e7897f158cdf Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 19 Apr 2023 19:34:31 +0200 Subject: [PATCH 16/45] Add logs if failed to get object --- commonspace/objectsync/objectsync.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index ec931eaa..a6d52d73 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -87,12 +87,13 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp return spacesyncproto.ErrSpaceIsDeleted } } - log.With(zap.String("objectId", msg.ObjectId), zap.String("replyId", msg.ReplyId)).DebugCtx(ctx, "handling message") + log.DebugCtx(ctx, "handling message") obj, err := s.objectGetter.GetObject(ctx, msg.ObjectId) if err != nil { + log.DebugCtx(ctx, "failed to get object") respErr := s.sendErrorResponse(ctx, msg, senderId) if respErr != nil { - log.Debug("failed to send error response", zap.Error(respErr)) + log.DebugCtx(ctx, "failed to send error response", zap.Error(respErr)) } return } From 3edda33db6044a221aca8e10cffb930e11bd2a1b Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 19 Apr 2023 20:42:47 +0200 Subject: [PATCH 17/45] Update object sync error handling logic --- .../object/tree/synctree/synctreehandler.go | 2 +- .../object/tree/treechangeproto/treechange.go | 6 ++- commonspace/objectsync/objectsync.go | 38 ++++++++++++++----- .../mock_spacestorage/mock_spacestorage.go | 15 ++++++++ commonspace/spacestorage/spacestorage.go | 1 + net/rpc/rpcerr/registry.go | 4 ++ 6 files changed, 54 insertions(+), 12 deletions(-) diff --git a/commonspace/object/tree/synctree/synctreehandler.go b/commonspace/object/tree/synctree/synctreehandler.go index 7865f2a7..7b4cb2a8 100644 --- a/commonspace/object/tree/synctree/synctreehandler.go +++ b/commonspace/object/tree/synctree/synctreehandler.go @@ -167,7 +167,7 @@ func (s *syncTreeHandler) handleFullSyncRequest( defer func() { if err != nil { log.With(zap.Error(err)).DebugCtx(ctx, "full sync request finished with error") - s.syncClient.SendWithReply(ctx, senderId, treechangeproto.WrapError(treechangeproto.ErrorCodes_FullSyncRequestError, header), replyId) + s.syncClient.SendWithReply(ctx, senderId, treechangeproto.WrapError(treechangeproto.ErrFullSync, header), replyId) return } else if fullResponse != nil { cnt := fullResponse.Content.GetFullSyncResponse() diff --git a/commonspace/object/tree/treechangeproto/treechange.go b/commonspace/object/tree/treechangeproto/treechange.go index 0dc5d668..92ef3f16 100644 --- a/commonspace/object/tree/treechangeproto/treechange.go +++ b/commonspace/object/tree/treechangeproto/treechange.go @@ -1,5 +1,7 @@ package treechangeproto +import "github.com/anytypeio/any-sync/net/rpc/rpcerr" + func WrapHeadUpdate(update *TreeHeadUpdate, rootChange *RawTreeChangeWithId) *TreeSyncMessage { return &TreeSyncMessage{ Content: &TreeSyncContentValue{ @@ -27,10 +29,10 @@ func WrapFullResponse(response *TreeFullSyncResponse, rootChange *RawTreeChangeW } } -func WrapError(code ErrorCodes, rootChange *RawTreeChangeWithId) *TreeSyncMessage { +func WrapError(err error, rootChange *RawTreeChangeWithId) *TreeSyncMessage { return &TreeSyncMessage{ Content: &TreeSyncContentValue{ - Value: &TreeSyncContentValue_ErrorResponse{ErrorResponse: &TreeErrorResponse{ErrCode: uint64(code)}}, + Value: &TreeSyncContentValue_ErrorResponse{ErrorResponse: &TreeErrorResponse{ErrCode: rpcerr.Code(err)}}, }, RootChange: rootChange, } diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index a6d52d73..f6e311ac 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -84,18 +84,30 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp log = log.With(zap.Bool("isDeleted", true)) // preventing sync with other clients if they are not just syncing the settings tree if !slices.Contains(s.configuration.NodeIds(s.spaceId), senderId) && msg.ObjectId != s.spaceStorage.SpaceSettingsId() { - return spacesyncproto.ErrSpaceIsDeleted + return s.unmarshallSendError(ctx, msg, spacesyncproto.ErrSpaceIsDeleted, senderId) } } log.DebugCtx(ctx, "handling message") + hasTree, err := s.spaceStorage.HasTree(msg.ObjectId) + if err != nil { + return s.unmarshallSendError(ctx, msg, spacesyncproto.ErrUnexpected, senderId) + } + // in this case we will try to get it from remote, unless the sender also sent us the same request :-) + if !hasTree { + treeMsg := &treechangeproto.TreeSyncMessage{} + err = proto.Unmarshal(msg.Payload, treeMsg) + if err != nil { + return s.sendError(ctx, nil, spacesyncproto.ErrUnexpected, senderId, msg.ReplyId) + } + // this means that we don't have the tree locally and therefore can't return it + if s.isEmptyFullSyncRequest(treeMsg) { + return s.sendError(ctx, treeMsg.RootChange, treechangeproto.ErrGetTree, senderId, msg.ReplyId) + } + } obj, err := s.objectGetter.GetObject(ctx, msg.ObjectId) if err != nil { log.DebugCtx(ctx, "failed to get object") - respErr := s.sendErrorResponse(ctx, msg, senderId) - if respErr != nil { - log.DebugCtx(ctx, "failed to send error response", zap.Error(respErr)) - } - return + return s.unmarshallSendError(ctx, msg, err, senderId) } return obj.HandleMessage(ctx, senderId, msg) } @@ -104,12 +116,20 @@ func (s *objectSync) MessagePool() MessagePool { return s.messagePool } -func (s *objectSync) sendErrorResponse(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, senderId string) (err error) { +func (s *objectSync) unmarshallSendError(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, respErr error, senderId string) (err error) { unmarshalled := &treechangeproto.TreeSyncMessage{} err = proto.Unmarshal(msg.Payload, unmarshalled) if err != nil { return } - resp := treechangeproto.WrapError(treechangeproto.ErrorCodes_GetTreeError, unmarshalled.RootChange) - return s.syncClient.SendWithReply(ctx, senderId, resp, msg.ReplyId) + return s.sendError(ctx, unmarshalled.RootChange, respErr, senderId, msg.ReplyId) +} + +func (s *objectSync) sendError(ctx context.Context, root *treechangeproto.RawTreeChangeWithId, respErr error, senderId, replyId string) (err error) { + resp := treechangeproto.WrapError(respErr, root) + return s.syncClient.SendWithReply(ctx, senderId, resp, replyId) +} + +func (s *objectSync) isEmptyFullSyncRequest(msg *treechangeproto.TreeSyncMessage) bool { + return len(msg.GetContent().GetFullSyncRequest().GetHeads()) == 0 } diff --git a/commonspace/spacestorage/mock_spacestorage/mock_spacestorage.go b/commonspace/spacestorage/mock_spacestorage/mock_spacestorage.go index 82a14be2..93baa877 100644 --- a/commonspace/spacestorage/mock_spacestorage/mock_spacestorage.go +++ b/commonspace/spacestorage/mock_spacestorage/mock_spacestorage.go @@ -81,6 +81,21 @@ func (mr *MockSpaceStorageMockRecorder) CreateTreeStorage(arg0 interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTreeStorage", reflect.TypeOf((*MockSpaceStorage)(nil).CreateTreeStorage), arg0) } +// HasTree mocks base method. +func (m *MockSpaceStorage) HasTree(arg0 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HasTree", arg0) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// HasTree indicates an expected call of HasTree. +func (mr *MockSpaceStorageMockRecorder) HasTree(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasTree", reflect.TypeOf((*MockSpaceStorage)(nil).HasTree), arg0) +} + // Id mocks base method. func (m *MockSpaceStorage) Id() string { m.ctrl.T.Helper() diff --git a/commonspace/spacestorage/spacestorage.go b/commonspace/spacestorage/spacestorage.go index ebd4b68d..3f21c22e 100644 --- a/commonspace/spacestorage/spacestorage.go +++ b/commonspace/spacestorage/spacestorage.go @@ -47,6 +47,7 @@ type SpaceStorage interface { StoredIds() ([]string, error) TreeRoot(id string) (*treechangeproto.RawTreeChangeWithId, error) TreeStorage(id string) (treestorage.TreeStorage, error) + HasTree(id string) (bool, error) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (treestorage.TreeStorage, error) WriteSpaceHash(hash string) error ReadSpaceHash() (hash string, err error) diff --git a/net/rpc/rpcerr/registry.go b/net/rpc/rpcerr/registry.go index 5160fec3..e8a749e6 100644 --- a/net/rpc/rpcerr/registry.go +++ b/net/rpc/rpcerr/registry.go @@ -24,6 +24,10 @@ func RegisterErr(err error, code uint64) error { return errWithCode } +func Code(err error) uint64 { + return drpcerr.Code(err) +} + func Err(code uint64) error { err, ok := errsMap[code] if !ok { From ed16abcedeb3a9ff943287dd11198850f8125b9c Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 19 Apr 2023 21:16:32 +0200 Subject: [PATCH 18/45] Fix sending reply with empty rootchange --- commonspace/object/tree/synctree/synctree.go | 2 +- .../object/tree/synctree/synctreehandler.go | 17 ++++++++++------- .../tree/synctree/synctreehandler_test.go | 12 ++++++------ .../mock_objectsync/mock_objectsync.go | 8 ++++---- commonspace/objectsync/objectsync.go | 18 +++++++++--------- commonspace/objectsync/syncclient.go | 6 +++--- 6 files changed, 33 insertions(+), 30 deletions(-) diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index 8b3295df..906ba5cf 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -243,7 +243,7 @@ func (s *syncTree) SyncWithPeer(ctx context.Context, peerId string) (err error) s.Lock() defer s.Unlock() headUpdate := s.syncClient.CreateHeadUpdate(s, nil) - return s.syncClient.SendWithReply(ctx, peerId, headUpdate, "") + return s.syncClient.SendWithReply(ctx, peerId, headUpdate.RootChange.Id, headUpdate, "") } func (s *syncTree) afterBuild() { diff --git a/commonspace/object/tree/synctree/synctreehandler.go b/commonspace/object/tree/synctree/synctreehandler.go index 7b4cb2a8..dab63e92 100644 --- a/commonspace/object/tree/synctree/synctreehandler.go +++ b/commonspace/object/tree/synctree/synctreehandler.go @@ -82,10 +82,11 @@ func (s *syncTreeHandler) handleHeadUpdate( fullRequest *treechangeproto.TreeSyncMessage isEmptyUpdate = len(update.Changes) == 0 objTree = s.objTree + treeId = objTree.Id() ) log := log.With( zap.Strings("update heads", update.Heads), - zap.String("treeId", objTree.Id()), + zap.String("treeId", treeId), zap.String("spaceId", s.spaceId), zap.Int("len(update changes)", len(update.Changes))) log.DebugCtx(ctx, "received head update message") @@ -118,7 +119,7 @@ func (s *syncTreeHandler) handleHeadUpdate( return } - return s.syncClient.SendWithReply(ctx, senderId, fullRequest, replyId) + return s.syncClient.SendWithReply(ctx, senderId, treeId, fullRequest, replyId) } if s.alreadyHasHeads(objTree, update.Heads) { @@ -142,7 +143,7 @@ func (s *syncTreeHandler) handleHeadUpdate( return } - return s.syncClient.SendWithReply(ctx, senderId, fullRequest, replyId) + return s.syncClient.SendWithReply(ctx, senderId, treeId, fullRequest, replyId) } func (s *syncTreeHandler) handleFullSyncRequest( @@ -154,11 +155,12 @@ func (s *syncTreeHandler) handleFullSyncRequest( fullResponse *treechangeproto.TreeSyncMessage header = s.objTree.Header() objTree = s.objTree + treeId = s.objTree.Id() ) log := log.With(zap.String("senderId", senderId), zap.Strings("request heads", request.Heads), - zap.String("treeId", s.objTree.Id()), + zap.String("treeId", treeId), zap.String("replyId", replyId), zap.String("spaceId", s.spaceId), zap.Int("len(request changes)", len(request.Changes))) @@ -167,7 +169,7 @@ func (s *syncTreeHandler) handleFullSyncRequest( defer func() { if err != nil { log.With(zap.Error(err)).DebugCtx(ctx, "full sync request finished with error") - s.syncClient.SendWithReply(ctx, senderId, treechangeproto.WrapError(treechangeproto.ErrFullSync, header), replyId) + s.syncClient.SendWithReply(ctx, senderId, treeId, treechangeproto.WrapError(treechangeproto.ErrFullSync, header), replyId) return } else if fullResponse != nil { cnt := fullResponse.Content.GetFullSyncResponse() @@ -190,7 +192,7 @@ func (s *syncTreeHandler) handleFullSyncRequest( return } - return s.syncClient.SendWithReply(ctx, senderId, fullResponse, replyId) + return s.syncClient.SendWithReply(ctx, senderId, treeId, fullResponse, replyId) } func (s *syncTreeHandler) handleFullSyncResponse( @@ -199,10 +201,11 @@ func (s *syncTreeHandler) handleFullSyncResponse( response *treechangeproto.TreeFullSyncResponse) (err error) { var ( objTree = s.objTree + treeId = s.objTree.Id() ) log := log.With( zap.Strings("heads", response.Heads), - zap.String("treeId", s.objTree.Id()), + zap.String("treeId", treeId), zap.String("spaceId", s.spaceId), zap.Int("len(changes)", len(response.Changes))) log.DebugCtx(ctx, "received full sync response message") diff --git a/commonspace/object/tree/synctree/synctreehandler_test.go b/commonspace/object/tree/synctree/synctreehandler_test.go index b95ffc82..401f9989 100644 --- a/commonspace/object/tree/synctree/synctreehandler_test.go +++ b/commonspace/object/tree/synctree/synctreehandler_test.go @@ -154,7 +154,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { fx.syncClientMock.EXPECT(). CreateFullSyncRequest(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullRequest, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullRequest), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullRequest), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -200,7 +200,7 @@ func TestSyncHandler_HandleHeadUpdate(t *testing.T) { fx.syncClientMock.EXPECT(). CreateFullSyncRequest(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullRequest, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullRequest), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullRequest), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -266,7 +266,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { fx.syncClientMock.EXPECT(). CreateFullSyncResponse(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullResponse, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullResponse), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullResponse), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -295,7 +295,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { fx.syncClientMock.EXPECT(). CreateFullSyncResponse(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullResponse, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullResponse), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullResponse), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -322,7 +322,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { fx.syncClientMock.EXPECT(). CreateFullSyncResponse(gomock.Eq(fx.objectTreeMock), gomock.Eq([]string{"h1"}), gomock.Eq([]string{"h1"})). Return(fullResponse, nil) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(fullResponse), gomock.Eq(replyId)) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Eq(fullResponse), gomock.Eq(replyId)) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.NoError(t, err) @@ -357,7 +357,7 @@ func TestSyncHandler_HandleFullSyncRequest(t *testing.T) { RawChanges: []*treechangeproto.RawTreeChangeWithId{chWithId}, })). Return(objecttree.AddResult{}, fmt.Errorf("")) - fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Any(), gomock.Eq("")) + fx.syncClientMock.EXPECT().SendWithReply(gomock.Any(), gomock.Eq(senderId), gomock.Eq(treeId), gomock.Any(), gomock.Eq("")) err := fx.syncHandler.HandleMessage(ctx, senderId, objectMsg) require.Error(t, err) diff --git a/commonspace/objectsync/mock_objectsync/mock_objectsync.go b/commonspace/objectsync/mock_objectsync/mock_objectsync.go index b82360db..aaca46a5 100644 --- a/commonspace/objectsync/mock_objectsync/mock_objectsync.go +++ b/commonspace/objectsync/mock_objectsync/mock_objectsync.go @@ -124,15 +124,15 @@ func (mr *MockSyncClientMockRecorder) PeerManager() *gomock.Call { } // SendWithReply mocks base method. -func (m *MockSyncClient) SendWithReply(arg0 context.Context, arg1 string, arg2 *treechangeproto.TreeSyncMessage, arg3 string) error { +func (m *MockSyncClient) SendWithReply(arg0 context.Context, arg1, arg2 string, arg3 *treechangeproto.TreeSyncMessage, arg4 string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendWithReply", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "SendWithReply", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(error) return ret0 } // SendWithReply indicates an expected call of SendWithReply. -func (mr *MockSyncClientMockRecorder) SendWithReply(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +func (mr *MockSyncClientMockRecorder) SendWithReply(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendWithReply", reflect.TypeOf((*MockSyncClient)(nil).SendWithReply), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendWithReply", reflect.TypeOf((*MockSyncClient)(nil).SendWithReply), arg0, arg1, arg2, arg3, arg4) } diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index f6e311ac..8bb9b5f2 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -84,30 +84,30 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp log = log.With(zap.Bool("isDeleted", true)) // preventing sync with other clients if they are not just syncing the settings tree if !slices.Contains(s.configuration.NodeIds(s.spaceId), senderId) && msg.ObjectId != s.spaceStorage.SpaceSettingsId() { - return s.unmarshallSendError(ctx, msg, spacesyncproto.ErrSpaceIsDeleted, senderId) + return s.unmarshallSendError(ctx, msg, spacesyncproto.ErrSpaceIsDeleted, senderId, msg.ObjectId) } } log.DebugCtx(ctx, "handling message") hasTree, err := s.spaceStorage.HasTree(msg.ObjectId) if err != nil { - return s.unmarshallSendError(ctx, msg, spacesyncproto.ErrUnexpected, senderId) + return s.unmarshallSendError(ctx, msg, spacesyncproto.ErrUnexpected, senderId, msg.ObjectId) } // in this case we will try to get it from remote, unless the sender also sent us the same request :-) if !hasTree { treeMsg := &treechangeproto.TreeSyncMessage{} err = proto.Unmarshal(msg.Payload, treeMsg) if err != nil { - return s.sendError(ctx, nil, spacesyncproto.ErrUnexpected, senderId, msg.ReplyId) + return s.sendError(ctx, nil, spacesyncproto.ErrUnexpected, senderId, msg.ObjectId, msg.ReplyId) } // this means that we don't have the tree locally and therefore can't return it if s.isEmptyFullSyncRequest(treeMsg) { - return s.sendError(ctx, treeMsg.RootChange, treechangeproto.ErrGetTree, senderId, msg.ReplyId) + return s.sendError(ctx, nil, treechangeproto.ErrGetTree, senderId, msg.ObjectId, msg.ReplyId) } } obj, err := s.objectGetter.GetObject(ctx, msg.ObjectId) if err != nil { log.DebugCtx(ctx, "failed to get object") - return s.unmarshallSendError(ctx, msg, err, senderId) + return s.unmarshallSendError(ctx, msg, err, msg.ObjectId, senderId) } return obj.HandleMessage(ctx, senderId, msg) } @@ -116,18 +116,18 @@ func (s *objectSync) MessagePool() MessagePool { return s.messagePool } -func (s *objectSync) unmarshallSendError(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, respErr error, senderId string) (err error) { +func (s *objectSync) unmarshallSendError(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, respErr error, senderId, objectId string) (err error) { unmarshalled := &treechangeproto.TreeSyncMessage{} err = proto.Unmarshal(msg.Payload, unmarshalled) if err != nil { return } - return s.sendError(ctx, unmarshalled.RootChange, respErr, senderId, msg.ReplyId) + return s.sendError(ctx, unmarshalled.RootChange, respErr, senderId, objectId, msg.ReplyId) } -func (s *objectSync) sendError(ctx context.Context, root *treechangeproto.RawTreeChangeWithId, respErr error, senderId, replyId string) (err error) { +func (s *objectSync) sendError(ctx context.Context, root *treechangeproto.RawTreeChangeWithId, respErr error, senderId, objectId, replyId string) (err error) { resp := treechangeproto.WrapError(respErr, root) - return s.syncClient.SendWithReply(ctx, senderId, resp, replyId) + return s.syncClient.SendWithReply(ctx, senderId, objectId, resp, replyId) } func (s *objectSync) isEmptyFullSyncRequest(msg *treechangeproto.TreeSyncMessage) bool { diff --git a/commonspace/objectsync/syncclient.go b/commonspace/objectsync/syncclient.go index 6ef203c0..4a95cae1 100644 --- a/commonspace/objectsync/syncclient.go +++ b/commonspace/objectsync/syncclient.go @@ -10,7 +10,7 @@ import ( type SyncClient interface { RequestFactory Broadcast(ctx context.Context, msg *treechangeproto.TreeSyncMessage) (err error) - SendWithReply(ctx context.Context, peerId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) + SendWithReply(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) PeerManager() peermanager.PeerManager } @@ -39,8 +39,8 @@ func (s *syncClient) Broadcast(ctx context.Context, msg *treechangeproto.TreeSyn return s.peerManager.Broadcast(ctx, objMsg) } -func (s *syncClient) SendWithReply(ctx context.Context, peerId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) { - objMsg, err := MarshallTreeMessage(msg, s.spaceId, msg.RootChange.Id, replyId) +func (s *syncClient) SendWithReply(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) { + objMsg, err := MarshallTreeMessage(msg, s.spaceId, objectId, replyId) if err != nil { return } From e79d319925eef63368811783670024777d108198 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 19 Apr 2023 22:43:46 +0200 Subject: [PATCH 19/45] Change sync client message pool interactions and request-reply --- commonspace/object/tree/synctree/synctree.go | 9 +---- .../object/tree/synctree/treeremotegetter.go | 7 +--- .../object/tree/synctree/utils_test.go | 38 ++++++++++++------- .../mock_objectsync/mock_objectsync.go | 32 ++++++++++++---- commonspace/objectsync/objectsync.go | 19 ++++++---- commonspace/objectsync/syncclient.go | 26 ++++++++----- commonspace/space.go | 6 +-- 7 files changed, 83 insertions(+), 54 deletions(-) diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index 906ba5cf..dc87a3c4 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -55,15 +55,13 @@ type syncTree struct { var log = logger.NewNamed("common.commonspace.synctree") -var createSyncClient = objectsync.NewSyncClient - type ResponsiblePeersGetter interface { GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) } type BuildDeps struct { SpaceId string - ObjectSync objectsync.ObjectSync + SyncClient objectsync.SyncClient Configuration nodeconf.NodeConf HeadNotifiable HeadNotifiable Listener updatelistener.UpdateListener @@ -99,10 +97,7 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t Sy if err != nil { return } - syncClient := createSyncClient( - deps.SpaceId, - deps.ObjectSync.MessagePool(), - objectsync.GetRequestFactory()) + syncClient := deps.SyncClient syncTree := &syncTree{ ObjectTree: objTree, syncClient: syncClient, diff --git a/commonspace/object/tree/synctree/treeremotegetter.go b/commonspace/object/tree/synctree/treeremotegetter.go index 9f819c41..bcf547b0 100644 --- a/commonspace/object/tree/synctree/treeremotegetter.go +++ b/commonspace/object/tree/synctree/treeremotegetter.go @@ -48,12 +48,7 @@ func (t treeRemoteGetter) getPeers(ctx context.Context) (peerIds []string, err e func (t treeRemoteGetter) treeRequest(ctx context.Context, peerId string) (msg *treechangeproto.TreeSyncMessage, err error) { newTreeRequest := objectsync.GetRequestFactory().CreateNewTreeRequest() - objMsg, err := objectsync.MarshallTreeMessage(newTreeRequest, t.deps.SpaceId, t.treeId, "") - if err != nil { - return - } - - resp, err := t.deps.ObjectSync.MessagePool().SendSync(ctx, peerId, objMsg) + resp, err := t.deps.SyncClient.SendSync(ctx, peerId, t.treeId, newTreeRequest) if err != nil { return } diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index 9d2c4f68..451f7536 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -95,7 +95,7 @@ type testSyncHandler struct { // createSyncHandler creates a sync handler when a tree is already created func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *testSyncHandler { factory := objectsync.GetRequestFactory() - syncClient := objectsync.NewSyncClient(spaceId, newTestPeerManager(peerId, log), factory) + syncClient := objectsync.NewSyncClient(spaceId, newTestMessagePool(peerId, log), factory) netTree := &broadcastTree{ ObjectTree: objTree, SyncClient: syncClient, @@ -107,7 +107,7 @@ func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, lo // createEmptySyncHandler creates a sync handler when the tree will be provided later (this emulates the situation when we have no tree) func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *testSyncHandler { factory := objectsync.GetRequestFactory() - syncClient := objectsync.NewSyncClient(spaceId, newTestPeerManager(peerId, log), factory) + syncClient := objectsync.NewSyncClient(spaceId, newTestMessagePool(peerId, log), factory) batcher := mb.New[protocolMsg](0) return &testSyncHandler{ @@ -173,11 +173,11 @@ func (h *testSyncHandler) HandleMessage(ctx context.Context, senderId string, re return h.manager().Broadcast(context.Background(), objMsg) } -func (h *testSyncHandler) manager() *testPeerManager { +func (h *testSyncHandler) manager() *testMessagePool { if h.SyncHandler != nil { - return h.SyncHandler.(*syncTreeHandler).syncClient.PeerManager().(*testPeerManager) + return h.SyncHandler.(*syncTreeHandler).syncClient.MessagePool().(*testMessagePool) } - return h.syncClient.PeerManager().(*testPeerManager) + return h.syncClient.MessagePool().(*testMessagePool) } func (h *testSyncHandler) tree() *broadcastTree { @@ -218,22 +218,22 @@ func (h *testSyncHandler) run(ctx context.Context, t *testing.T, wg *sync.WaitGr }() } -// testPeerManager captures all other handlers and sends messages to them -type testPeerManager struct { +// testMessagePool captures all other handlers and sends messages to them +type testMessagePool struct { peerId string handlers map[string]*testSyncHandler log *messageLog } -func newTestPeerManager(peerId string, log *messageLog) *testPeerManager { - return &testPeerManager{handlers: map[string]*testSyncHandler{}, peerId: peerId, log: log} +func newTestMessagePool(peerId string, log *messageLog) *testMessagePool { + return &testMessagePool{handlers: map[string]*testSyncHandler{}, peerId: peerId, log: log} } -func (m *testPeerManager) addHandler(peerId string, handler *testSyncHandler) { +func (m *testMessagePool) addHandler(peerId string, handler *testSyncHandler) { m.handlers[peerId] = handler } -func (m *testPeerManager) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { +func (m *testMessagePool) SendPeer(ctx context.Context, peerId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { pMsg := protocolMsg{ msg: msg, senderId: m.peerId, @@ -243,7 +243,7 @@ func (m *testPeerManager) SendPeer(ctx context.Context, peerId string, msg *spac return m.handlers[peerId].send(context.Background(), pMsg) } -func (m *testPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { +func (m *testMessagePool) Broadcast(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage) (err error) { for _, handler := range m.handlers { pMsg := protocolMsg{ msg: msg, @@ -256,7 +256,19 @@ func (m *testPeerManager) Broadcast(ctx context.Context, msg *spacesyncproto.Obj return } -func (m *testPeerManager) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { +func (m *testMessagePool) GetResponsiblePeers(ctx context.Context) (peers []peer.Peer, err error) { + panic("should not be called") +} + +func (m *testMessagePool) LastUsage() time.Time { + panic("should not be called") +} + +func (m *testMessagePool) HandleMessage(ctx context.Context, senderId string, request *spacesyncproto.ObjectSyncMessage) (err error) { + panic("should not be called") +} + +func (m *testMessagePool) SendSync(ctx context.Context, peerId string, message *spacesyncproto.ObjectSyncMessage) (reply *spacesyncproto.ObjectSyncMessage, err error) { panic("should not be called") } diff --git a/commonspace/objectsync/mock_objectsync/mock_objectsync.go b/commonspace/objectsync/mock_objectsync/mock_objectsync.go index aaca46a5..29ea2717 100644 --- a/commonspace/objectsync/mock_objectsync/mock_objectsync.go +++ b/commonspace/objectsync/mock_objectsync/mock_objectsync.go @@ -10,7 +10,8 @@ import ( objecttree "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" treechangeproto "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - peermanager "github.com/anytypeio/any-sync/commonspace/peermanager" + objectsync "github.com/anytypeio/any-sync/commonspace/objectsync" + spacesyncproto "github.com/anytypeio/any-sync/commonspace/spacesyncproto" gomock "github.com/golang/mock/gomock" ) @@ -109,18 +110,33 @@ func (mr *MockSyncClientMockRecorder) CreateNewTreeRequest() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateNewTreeRequest", reflect.TypeOf((*MockSyncClient)(nil).CreateNewTreeRequest)) } -// PeerManager mocks base method. -func (m *MockSyncClient) PeerManager() peermanager.PeerManager { +// MessagePool mocks base method. +func (m *MockSyncClient) MessagePool() objectsync.MessagePool { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "PeerManager") - ret0, _ := ret[0].(peermanager.PeerManager) + ret := m.ctrl.Call(m, "MessagePool") + ret0, _ := ret[0].(objectsync.MessagePool) return ret0 } -// PeerManager indicates an expected call of PeerManager. -func (mr *MockSyncClientMockRecorder) PeerManager() *gomock.Call { +// MessagePool indicates an expected call of MessagePool. +func (mr *MockSyncClientMockRecorder) MessagePool() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PeerManager", reflect.TypeOf((*MockSyncClient)(nil).PeerManager)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MessagePool", reflect.TypeOf((*MockSyncClient)(nil).MessagePool)) +} + +// SendSync mocks base method. +func (m *MockSyncClient) SendSync(arg0 context.Context, arg1, arg2 string, arg3 *treechangeproto.TreeSyncMessage) (*spacesyncproto.ObjectSyncMessage, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendSync", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(*spacesyncproto.ObjectSyncMessage) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SendSync indicates an expected call of SendSync. +func (mr *MockSyncClientMockRecorder) SendSync(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendSync", reflect.TypeOf((*MockSyncClient)(nil).SendSync), arg0, arg1, arg2, arg3) } // SendWithReply mocks base method. diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index 8bb9b5f2..7a7a0d69 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -24,7 +24,7 @@ var log = logger.NewNamed("common.commonspace.objectsync") type ObjectSync interface { LastUsage synchandler.SyncHandler - MessagePool() MessagePool + SyncClient() SyncClient Close() (err error) } @@ -59,9 +59,9 @@ func NewObjectSync( cancelSync: cancel, spaceIsDeleted: spaceIsDeleted, configuration: configuration, - syncClient: NewSyncClient(spaceId, peerManager, GetRequestFactory()), } os.messagePool = newMessagePool(peerManager, os.handleMessage) + os.syncClient = NewSyncClient(spaceId, os.messagePool, GetRequestFactory()) return os } @@ -79,7 +79,10 @@ func (s *objectSync) HandleMessage(ctx context.Context, senderId string, message } func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *spacesyncproto.ObjectSyncMessage) (err error) { - log := log.With(zap.String("objectId", msg.ObjectId), zap.String("replyId", msg.ReplyId)) + log := log.With( + zap.String("objectId", msg.ObjectId), + zap.String("requestId", msg.RequestId), + zap.String("replyId", msg.ReplyId)) if s.spaceIsDeleted.Load() { log = log.With(zap.Bool("isDeleted", true)) // preventing sync with other clients if they are not just syncing the settings tree @@ -97,11 +100,11 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp treeMsg := &treechangeproto.TreeSyncMessage{} err = proto.Unmarshal(msg.Payload, treeMsg) if err != nil { - return s.sendError(ctx, nil, spacesyncproto.ErrUnexpected, senderId, msg.ObjectId, msg.ReplyId) + return s.sendError(ctx, nil, spacesyncproto.ErrUnexpected, senderId, msg.ObjectId, msg.RequestId) } // this means that we don't have the tree locally and therefore can't return it if s.isEmptyFullSyncRequest(treeMsg) { - return s.sendError(ctx, nil, treechangeproto.ErrGetTree, senderId, msg.ObjectId, msg.ReplyId) + return s.sendError(ctx, nil, treechangeproto.ErrGetTree, senderId, msg.ObjectId, msg.RequestId) } } obj, err := s.objectGetter.GetObject(ctx, msg.ObjectId) @@ -112,8 +115,8 @@ func (s *objectSync) handleMessage(ctx context.Context, senderId string, msg *sp return obj.HandleMessage(ctx, senderId, msg) } -func (s *objectSync) MessagePool() MessagePool { - return s.messagePool +func (s *objectSync) SyncClient() SyncClient { + return s.syncClient } func (s *objectSync) unmarshallSendError(ctx context.Context, msg *spacesyncproto.ObjectSyncMessage, respErr error, senderId, objectId string) (err error) { @@ -122,7 +125,7 @@ func (s *objectSync) unmarshallSendError(ctx context.Context, msg *spacesyncprot if err != nil { return } - return s.sendError(ctx, unmarshalled.RootChange, respErr, senderId, objectId, msg.ReplyId) + return s.sendError(ctx, unmarshalled.RootChange, respErr, senderId, objectId, msg.RequestId) } func (s *objectSync) sendError(ctx context.Context, root *treechangeproto.RawTreeChangeWithId, respErr error, senderId, objectId, replyId string) (err error) { diff --git a/commonspace/objectsync/syncclient.go b/commonspace/objectsync/syncclient.go index 4a95cae1..e6fd515e 100644 --- a/commonspace/objectsync/syncclient.go +++ b/commonspace/objectsync/syncclient.go @@ -3,7 +3,6 @@ package objectsync import ( "context" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/peermanager" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" ) @@ -11,21 +10,22 @@ type SyncClient interface { RequestFactory Broadcast(ctx context.Context, msg *treechangeproto.TreeSyncMessage) (err error) SendWithReply(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) - PeerManager() peermanager.PeerManager + SendSync(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage) (reply *spacesyncproto.ObjectSyncMessage, err error) + MessagePool() MessagePool } type syncClient struct { RequestFactory spaceId string - peerManager peermanager.PeerManager + messagePool MessagePool } func NewSyncClient( spaceId string, - peerManager peermanager.PeerManager, + messagePool MessagePool, factory RequestFactory) SyncClient { return &syncClient{ - peerManager: peerManager, + messagePool: messagePool, RequestFactory: factory, spaceId: spaceId, } @@ -36,7 +36,15 @@ func (s *syncClient) Broadcast(ctx context.Context, msg *treechangeproto.TreeSyn if err != nil { return } - return s.peerManager.Broadcast(ctx, objMsg) + return s.messagePool.Broadcast(ctx, objMsg) +} + +func (s *syncClient) SendSync(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage) (reply *spacesyncproto.ObjectSyncMessage, err error) { + objMsg, err := MarshallTreeMessage(msg, s.spaceId, objectId, "") + if err != nil { + return + } + return s.messagePool.SendSync(ctx, peerId, objMsg) } func (s *syncClient) SendWithReply(ctx context.Context, peerId, objectId string, msg *treechangeproto.TreeSyncMessage, replyId string) (err error) { @@ -44,11 +52,11 @@ func (s *syncClient) SendWithReply(ctx context.Context, peerId, objectId string, if err != nil { return } - return s.peerManager.SendPeer(ctx, peerId, objMsg) + return s.messagePool.SendPeer(ctx, peerId, objMsg) } -func (s *syncClient) PeerManager() peermanager.PeerManager { - return s.peerManager +func (s *syncClient) MessagePool() MessagePool { + return s.messagePool } func MarshallTreeMessage(message *treechangeproto.TreeSyncMessage, spaceId, objectId, replyId string) (objMsg *spacesyncproto.ObjectSyncMessage, err error) { diff --git a/commonspace/space.go b/commonspace/space.go index 88fb5692..10090b95 100644 --- a/commonspace/space.go +++ b/commonspace/space.go @@ -179,7 +179,7 @@ func (s *space) Init(ctx context.Context) (err error) { if err != nil { return } - s.aclList = syncacl.NewSyncAcl(aclList, s.objectSync.MessagePool()) + s.aclList = syncacl.NewSyncAcl(aclList, s.objectSync.SyncClient().MessagePool()) s.treeManager.AddObject(s.aclList) deletionState := settingsstate.NewObjectDeletionState(s.storage) @@ -284,7 +284,7 @@ func (s *space) PutTree(ctx context.Context, payload treestorage.TreeStorageCrea } deps := synctree.BuildDeps{ SpaceId: s.id, - ObjectSync: s.objectSync, + SyncClient: s.objectSync.SyncClient(), Configuration: s.configuration, HeadNotifiable: s.headSync, Listener: listener, @@ -322,7 +322,7 @@ func (s *space) BuildTree(ctx context.Context, id string, opts BuildTreeOpts) (t deps := synctree.BuildDeps{ SpaceId: s.id, - ObjectSync: s.objectSync, + SyncClient: s.objectSync.SyncClient(), Configuration: s.configuration, HeadNotifiable: s.headSync, Listener: opts.Listener, From 4c71a9a1c4b3ed266750bbf130c37833b6f7180f Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 19 Apr 2023 22:52:06 +0200 Subject: [PATCH 20/45] Simplify request factory --- commonspace/object/tree/synctree/treeremotegetter.go | 3 +-- commonspace/object/tree/synctree/utils_test.go | 8 ++++---- commonspace/objectsync/objectsync.go | 2 +- commonspace/objectsync/requestfactory.go | 6 ++---- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/commonspace/object/tree/synctree/treeremotegetter.go b/commonspace/object/tree/synctree/treeremotegetter.go index bcf547b0..3156373f 100644 --- a/commonspace/object/tree/synctree/treeremotegetter.go +++ b/commonspace/object/tree/synctree/treeremotegetter.go @@ -6,7 +6,6 @@ import ( "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" - "github.com/anytypeio/any-sync/commonspace/objectsync" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/net/peer" "github.com/anytypeio/any-sync/net/rpc/rpcerr" @@ -47,7 +46,7 @@ func (t treeRemoteGetter) getPeers(ctx context.Context) (peerIds []string, err e } func (t treeRemoteGetter) treeRequest(ctx context.Context, peerId string) (msg *treechangeproto.TreeSyncMessage, err error) { - newTreeRequest := objectsync.GetRequestFactory().CreateNewTreeRequest() + newTreeRequest := t.deps.SyncClient.CreateNewTreeRequest() resp, err := t.deps.SyncClient.SendSync(ctx, peerId, t.treeId, newTreeRequest) if err != nil { return diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index 451f7536..1d844e90 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -94,7 +94,7 @@ type testSyncHandler struct { // createSyncHandler creates a sync handler when a tree is already created func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, log *messageLog) *testSyncHandler { - factory := objectsync.GetRequestFactory() + factory := objectsync.NewRequestFactory() syncClient := objectsync.NewSyncClient(spaceId, newTestMessagePool(peerId, log), factory) netTree := &broadcastTree{ ObjectTree: objTree, @@ -106,7 +106,7 @@ func createSyncHandler(peerId, spaceId string, objTree objecttree.ObjectTree, lo // createEmptySyncHandler creates a sync handler when the tree will be provided later (this emulates the situation when we have no tree) func createEmptySyncHandler(peerId, spaceId string, aclList list.AclList, log *messageLog) *testSyncHandler { - factory := objectsync.GetRequestFactory() + factory := objectsync.NewRequestFactory() syncClient := objectsync.NewSyncClient(spaceId, newTestMessagePool(peerId, log), factory) batcher := mb.New[protocolMsg](0) @@ -138,7 +138,7 @@ func (h *testSyncHandler) HandleMessage(ctx context.Context, senderId string, re return } if unmarshalled.Content.GetFullSyncResponse() == nil { - newTreeRequest := objectsync.GetRequestFactory().CreateNewTreeRequest() + newTreeRequest := objectsync.NewRequestFactory().CreateNewTreeRequest() var objMsg *spacesyncproto.ObjectSyncMessage objMsg, err = objectsync.MarshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") if err != nil { @@ -165,7 +165,7 @@ func (h *testSyncHandler) HandleMessage(ctx context.Context, senderId string, re } h.SyncHandler = newSyncTreeHandler(request.SpaceId, netTree, h.syncClient, syncstatus.NewNoOpSyncStatus()) var objMsg *spacesyncproto.ObjectSyncMessage - newTreeRequest := objectsync.GetRequestFactory().CreateHeadUpdate(netTree, res.Added) + newTreeRequest := objectsync.NewRequestFactory().CreateHeadUpdate(netTree, res.Added) objMsg, err = objectsync.MarshallTreeMessage(newTreeRequest, request.SpaceId, request.ObjectId, "") if err != nil { return diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index 7a7a0d69..02e6019c 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -61,7 +61,7 @@ func NewObjectSync( configuration: configuration, } os.messagePool = newMessagePool(peerManager, os.handleMessage) - os.syncClient = NewSyncClient(spaceId, os.messagePool, GetRequestFactory()) + os.syncClient = NewSyncClient(spaceId, os.messagePool, NewRequestFactory()) return os } diff --git a/commonspace/objectsync/requestfactory.go b/commonspace/objectsync/requestfactory.go index 0608ad33..4d6a9124 100644 --- a/commonspace/objectsync/requestfactory.go +++ b/commonspace/objectsync/requestfactory.go @@ -14,10 +14,8 @@ type RequestFactory interface { CreateFullSyncResponse(t objecttree.ObjectTree, theirHeads, theirSnapshotPath []string) (*treechangeproto.TreeSyncMessage, error) } -var sharedFactory = &requestFactory{} - -func GetRequestFactory() RequestFactory { - return sharedFactory +func NewRequestFactory() RequestFactory { + return &requestFactory{} } type requestFactory struct{} From 25f90d5f5f767df3037e40f322c4d98fdbd071b9 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 20 Apr 2023 00:04:16 +0200 Subject: [PATCH 21/45] Fix proto --- .../treechangeproto/protos/treechange.proto | 3 +- .../tree/treechangeproto/treechange.pb.go | 146 ++++++++++++------ 2 files changed, 101 insertions(+), 48 deletions(-) diff --git a/commonspace/object/tree/treechangeproto/protos/treechange.proto b/commonspace/object/tree/treechangeproto/protos/treechange.proto index ae7cebd5..eb331567 100644 --- a/commonspace/object/tree/treechangeproto/protos/treechange.proto +++ b/commonspace/object/tree/treechangeproto/protos/treechange.proto @@ -101,7 +101,8 @@ message TreeFullSyncResponse { // TreeErrorResponse is an error sent as a response for a full sync request message TreeErrorResponse { - uint64 errCode = 1; + string error = 1; + uint64 errCode = 2; } // TreeChangeInfo is used internally in Tree implementation for ease of marshalling diff --git a/commonspace/object/tree/treechangeproto/treechange.pb.go b/commonspace/object/tree/treechangeproto/treechange.pb.go index f4098fc6..34d57487 100644 --- a/commonspace/object/tree/treechangeproto/treechange.pb.go +++ b/commonspace/object/tree/treechangeproto/treechange.pb.go @@ -722,7 +722,8 @@ func (m *TreeFullSyncResponse) GetSnapshotPath() []string { // TreeErrorResponse is an error sent as a response for a full sync request type TreeErrorResponse struct { - ErrCode uint64 `protobuf:"varint,1,opt,name=errCode,proto3" json:"errCode,omitempty"` + Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` + ErrCode uint64 `protobuf:"varint,2,opt,name=errCode,proto3" json:"errCode,omitempty"` } func (m *TreeErrorResponse) Reset() { *m = TreeErrorResponse{} } @@ -758,6 +759,13 @@ func (m *TreeErrorResponse) XXX_DiscardUnknown() { var xxx_messageInfo_TreeErrorResponse proto.InternalMessageInfo +func (m *TreeErrorResponse) GetError() string { + if m != nil { + return m.Error + } + return "" +} + func (m *TreeErrorResponse) GetErrCode() uint64 { if m != nil { return m.ErrCode @@ -838,53 +846,54 @@ func init() { } var fileDescriptor_5033f0301ef9b772 = []byte{ - // 732 bytes of a gzipped FileDescriptorProto + // 741 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x55, 0x4f, 0x4f, 0xfb, 0x46, 0x10, 0xf5, 0x3a, 0x81, 0x90, 0x49, 0x08, 0xe9, 0xc2, 0xc1, 0x42, 0xad, 0x6b, 0x59, 0x55, 0x1b, - 0x55, 0x2a, 0x48, 0xf4, 0xd4, 0xaa, 0x12, 0x2a, 0x29, 0x90, 0x08, 0xb5, 0x45, 0xcb, 0x9f, 0x4a, - 0xdc, 0x16, 0x7b, 0x42, 0x5c, 0x25, 0xb6, 0xeb, 0xdd, 0x94, 0xe6, 0x03, 0xf4, 0xd2, 0x4a, 0x88, - 0xaf, 0xd4, 0x5b, 0x8f, 0x1c, 0x39, 0xfe, 0x04, 0x5f, 0xe4, 0x27, 0xef, 0xc6, 0x89, 0xed, 0xe4, - 0xc0, 0x8d, 0x8b, 0xe3, 0x79, 0x9e, 0x79, 0xf3, 0xe6, 0xed, 0x9f, 0xc0, 0xa1, 0x17, 0x8d, 0xc7, - 0x51, 0x28, 0x62, 0xee, 0xe1, 0x7e, 0x74, 0xfb, 0x3b, 0x7a, 0x72, 0x5f, 0x26, 0x88, 0xea, 0xe1, - 0x0d, 0x79, 0x78, 0x87, 0x71, 0x12, 0xc9, 0x68, 0x5f, 0x3d, 0x45, 0x0e, 0xde, 0x53, 0x08, 0x85, - 0x05, 0xe2, 0x3e, 0x13, 0x00, 0x16, 0x45, 0xb2, 0xab, 0x42, 0xfa, 0x29, 0xd4, 0xb9, 0x37, 0xea, - 0x21, 0xf7, 0xfb, 0xbe, 0x45, 0x1c, 0xd2, 0xa9, 0xb3, 0x05, 0x40, 0x2d, 0xa8, 0xa9, 0xae, 0x7d, - 0xdf, 0x32, 0xd5, 0xb7, 0x2c, 0xa4, 0x36, 0x80, 0x26, 0xbc, 0x9c, 0xc6, 0x68, 0x55, 0xd4, 0xc7, - 0x1c, 0x92, 0xf2, 0xca, 0x60, 0x8c, 0x42, 0xf2, 0x71, 0x6c, 0x55, 0x1d, 0xd2, 0xa9, 0xb0, 0x05, - 0x40, 0x29, 0x54, 0x05, 0xa2, 0x6f, 0xad, 0x39, 0xa4, 0xd3, 0x64, 0xea, 0x9d, 0xee, 0xc2, 0x46, - 0xe0, 0x63, 0x28, 0x03, 0x39, 0xb5, 0xd6, 0x15, 0x3e, 0x8f, 0xe9, 0x17, 0xb0, 0xa9, 0xb9, 0xcf, - 0xf9, 0x74, 0x14, 0x71, 0xdf, 0xaa, 0xa9, 0x84, 0x22, 0xe8, 0x3e, 0x98, 0x00, 0x97, 0x09, 0xe2, - 0x6c, 0x34, 0x07, 0x1a, 0xe9, 0xdc, 0x7a, 0x14, 0x61, 0x11, 0xa7, 0xd2, 0xa9, 0xb3, 0x3c, 0x54, - 0x1c, 0xde, 0x2c, 0x0f, 0xff, 0x25, 0xb4, 0x44, 0xc8, 0x63, 0x31, 0x8c, 0xe4, 0x11, 0x17, 0xa9, - 0x07, 0x7a, 0xcc, 0x12, 0x9a, 0xf6, 0xd1, 0x3a, 0xc4, 0x4f, 0x5c, 0x72, 0x35, 0x6c, 0x93, 0xe5, - 0xa1, 0xb4, 0x4f, 0x82, 0xdc, 0x3f, 0xc3, 0x69, 0x5f, 0xcf, 0x5c, 0x67, 0x0b, 0xa0, 0x68, 0xd5, - 0x7a, 0xd9, 0xaa, 0xbc, 0x2d, 0xb5, 0x92, 0x2d, 0x36, 0x40, 0x20, 0x2e, 0x66, 0x6a, 0xac, 0x0d, - 0x87, 0x74, 0x36, 0x58, 0x0e, 0x71, 0x4f, 0x61, 0x93, 0xf1, 0xfb, 0x9c, 0x25, 0x16, 0xd4, 0xe2, - 0x99, 0x83, 0x44, 0x71, 0x65, 0x61, 0x2a, 0x42, 0x04, 0x77, 0x21, 0x97, 0x93, 0x04, 0x95, 0x15, - 0x4d, 0xb6, 0x00, 0xdc, 0x2e, 0x6c, 0x17, 0x88, 0x7e, 0x0b, 0xe4, 0x50, 0x2b, 0x4f, 0xf8, 0xbd, - 0x86, 0x66, 0x84, 0x0b, 0x80, 0xb6, 0xc0, 0x0c, 0x32, 0x5b, 0xcd, 0xc0, 0x77, 0x1f, 0x08, 0x6c, - 0xa5, 0x14, 0x17, 0xd3, 0xd0, 0xfb, 0x19, 0x85, 0xe0, 0x77, 0x48, 0xbf, 0x87, 0x9a, 0x17, 0x85, - 0x12, 0x43, 0xa9, 0xea, 0x1b, 0x07, 0xce, 0x5e, 0x6e, 0xf7, 0x66, 0xd9, 0x5d, 0x9d, 0x72, 0xcd, - 0x47, 0x13, 0x64, 0x59, 0x01, 0x3d, 0x04, 0x48, 0xe6, 0x1b, 0x59, 0xf5, 0x69, 0x1c, 0x7c, 0x9e, - 0x2f, 0x5f, 0x21, 0x99, 0xe5, 0x4a, 0xdc, 0xff, 0x4c, 0xd8, 0x59, 0xd5, 0x82, 0xfe, 0x00, 0x30, - 0x44, 0xee, 0x5f, 0xc5, 0x3e, 0x97, 0x38, 0x13, 0xb6, 0x5b, 0x16, 0xd6, 0x9b, 0x67, 0xf4, 0x0c, - 0x96, 0xcb, 0xa7, 0x67, 0xb0, 0x35, 0x98, 0x8c, 0x46, 0x29, 0x2b, 0xc3, 0x3f, 0x26, 0x28, 0xe4, - 0x2a, 0x71, 0x29, 0xc5, 0x49, 0x31, 0xad, 0x67, 0xb0, 0x72, 0x25, 0xfd, 0x05, 0xda, 0x0b, 0x48, - 0xc4, 0x51, 0x28, 0xf4, 0x69, 0x5b, 0xe1, 0xd4, 0x49, 0x29, 0xaf, 0x67, 0xb0, 0xa5, 0x5a, 0x7a, - 0x0c, 0x9b, 0x98, 0x24, 0x51, 0x32, 0x27, 0xab, 0x2a, 0xb2, 0xcf, 0xca, 0x64, 0xc7, 0xf9, 0xa4, - 0x9e, 0xc1, 0x8a, 0x55, 0x47, 0x35, 0x58, 0xfb, 0x33, 0xb5, 0xca, 0xfd, 0x9b, 0x40, 0xab, 0xe8, - 0x06, 0xdd, 0x81, 0xb5, 0xd4, 0x8d, 0xec, 0xc4, 0xe9, 0x80, 0x7e, 0x07, 0xb5, 0xd9, 0x91, 0xb0, - 0x4c, 0xa7, 0xf2, 0x96, 0xa5, 0xca, 0xf2, 0xa9, 0x0b, 0xcd, 0xec, 0xc8, 0x9d, 0x73, 0x39, 0xb4, - 0x2a, 0x8a, 0xb7, 0x80, 0xb9, 0xff, 0x10, 0xd8, 0x5e, 0x61, 0xe9, 0xfb, 0x88, 0xf9, 0x97, 0xe8, - 0x8d, 0x55, 0x5e, 0x91, 0xf7, 0x51, 0xf3, 0x0d, 0x7c, 0xb2, 0xb4, 0xa2, 0xe9, 0x4d, 0x80, 0x49, - 0xd2, 0x8d, 0x7c, 0xbd, 0xbf, 0xab, 0x2c, 0x0b, 0xdd, 0x6b, 0xbd, 0xa0, 0xba, 0x5f, 0x3f, 0x1c, - 0x44, 0xa5, 0xbb, 0x9e, 0x2c, 0xdd, 0xf5, 0x4b, 0xb7, 0xb3, 0xb9, 0xe2, 0x76, 0xfe, 0xfa, 0x06, - 0x40, 0x49, 0x48, 0x9b, 0x08, 0xda, 0x02, 0xb8, 0x0a, 0xf1, 0xaf, 0x18, 0x3d, 0x89, 0x7e, 0xdb, - 0xa0, 0x6d, 0x68, 0x9e, 0xa2, 0x9c, 0xeb, 0x6c, 0x13, 0x6a, 0xc1, 0x4e, 0x69, 0x31, 0xf5, 0x17, - 0x93, 0xb6, 0xa1, 0xa1, 0x5e, 0x7f, 0x1d, 0x0c, 0x04, 0xca, 0xf6, 0x63, 0xe5, 0xe8, 0xc7, 0xff, - 0x5f, 0x6c, 0xf2, 0xf4, 0x62, 0x93, 0x0f, 0x2f, 0x36, 0x79, 0x7c, 0xb5, 0x8d, 0xa7, 0x57, 0xdb, - 0x78, 0x7e, 0xb5, 0x8d, 0x9b, 0xaf, 0xde, 0xf8, 0xdf, 0x79, 0xbb, 0xae, 0x7e, 0xbe, 0xfd, 0x18, - 0x00, 0x00, 0xff, 0xff, 0xb5, 0x68, 0xf7, 0x3f, 0x6d, 0x07, 0x00, 0x00, + 0xf5, 0x00, 0x12, 0x3d, 0xb5, 0xaa, 0x84, 0x4a, 0x0a, 0x24, 0x42, 0x6d, 0xd1, 0xf2, 0xa7, 0x12, + 0xb7, 0xc5, 0x9e, 0x10, 0x57, 0x89, 0xed, 0x7a, 0x37, 0xa5, 0xf9, 0x00, 0xbd, 0xb4, 0x12, 0xe2, + 0x2b, 0xf5, 0xf6, 0x3b, 0x72, 0xe4, 0xf8, 0x13, 0x7c, 0x91, 0x9f, 0xbc, 0x1b, 0x27, 0xb6, 0x93, + 0x03, 0x37, 0x2e, 0x89, 0xe7, 0x79, 0xe6, 0xed, 0x9b, 0x37, 0xde, 0x5d, 0x38, 0xf4, 0xa2, 0xf1, + 0x38, 0x0a, 0x45, 0xcc, 0x3d, 0xdc, 0x8f, 0x6e, 0xff, 0x40, 0x4f, 0xee, 0xcb, 0x04, 0x51, 0xfd, + 0x78, 0x43, 0x1e, 0xde, 0x61, 0x9c, 0x44, 0x32, 0xda, 0x57, 0xbf, 0x22, 0x07, 0xef, 0x29, 0x84, + 0xc2, 0x02, 0x71, 0x9f, 0x09, 0x00, 0x8b, 0x22, 0xd9, 0x55, 0x21, 0xfd, 0x1c, 0xea, 0xdc, 0x1b, + 0xf5, 0x90, 0xfb, 0x7d, 0xdf, 0x22, 0x0e, 0xe9, 0xd4, 0xd9, 0x02, 0xa0, 0x16, 0xd4, 0xd4, 0xaa, + 0x7d, 0xdf, 0x32, 0xd5, 0xbb, 0x2c, 0xa4, 0x36, 0x80, 0x26, 0xbc, 0x9c, 0xc6, 0x68, 0x55, 0xd4, + 0xcb, 0x1c, 0x92, 0xf2, 0xca, 0x60, 0x8c, 0x42, 0xf2, 0x71, 0x6c, 0x55, 0x1d, 0xd2, 0xa9, 0xb0, + 0x05, 0x40, 0x29, 0x54, 0x05, 0xa2, 0x6f, 0xad, 0x39, 0xa4, 0xd3, 0x64, 0xea, 0x99, 0xee, 0xc2, + 0x46, 0xe0, 0x63, 0x28, 0x03, 0x39, 0xb5, 0xd6, 0x15, 0x3e, 0x8f, 0xe9, 0x57, 0xb0, 0xa9, 0xb9, + 0xcf, 0xf9, 0x74, 0x14, 0x71, 0xdf, 0xaa, 0xa9, 0x84, 0x22, 0xe8, 0x3e, 0x98, 0x00, 0x97, 0x09, + 0xe2, 0xac, 0x35, 0x07, 0x1a, 0x69, 0xdf, 0xba, 0x15, 0x61, 0x11, 0xa7, 0xd2, 0xa9, 0xb3, 0x3c, + 0x54, 0x6c, 0xde, 0x2c, 0x37, 0xff, 0x35, 0xb4, 0x44, 0xc8, 0x63, 0x31, 0x8c, 0xe4, 0x11, 0x17, + 0xa9, 0x07, 0xba, 0xcd, 0x12, 0x9a, 0xae, 0xa3, 0x75, 0x88, 0x9f, 0xb9, 0xe4, 0xaa, 0xd9, 0x26, + 0xcb, 0x43, 0xe9, 0x3a, 0x09, 0x72, 0xff, 0x0c, 0xa7, 0x7d, 0xdd, 0x73, 0x9d, 0x2d, 0x80, 0xa2, + 0x55, 0xeb, 0x65, 0xab, 0xf2, 0xb6, 0xd4, 0x4a, 0xb6, 0xd8, 0x00, 0x81, 0xb8, 0x98, 0xa9, 0xb1, + 0x36, 0x1c, 0xd2, 0xd9, 0x60, 0x39, 0xc4, 0x3d, 0x85, 0x4d, 0xc6, 0xef, 0x73, 0x96, 0x58, 0x50, + 0x8b, 0x67, 0x0e, 0x12, 0xc5, 0x95, 0x85, 0xa9, 0x08, 0x11, 0xdc, 0x85, 0x5c, 0x4e, 0x12, 0x54, + 0x56, 0x34, 0xd9, 0x02, 0x70, 0xbb, 0xb0, 0x5d, 0x20, 0xfa, 0x3d, 0x90, 0x43, 0xad, 0x3c, 0xe1, + 0xf7, 0x1a, 0x9a, 0x11, 0x2e, 0x00, 0xda, 0x02, 0x33, 0xc8, 0x6c, 0x35, 0x03, 0xdf, 0x7d, 0x20, + 0xb0, 0x95, 0x52, 0x5c, 0x4c, 0x43, 0xef, 0x17, 0x14, 0x82, 0xdf, 0x21, 0xfd, 0x01, 0x6a, 0x5e, + 0x14, 0x4a, 0x0c, 0xa5, 0xaa, 0x6f, 0x1c, 0x38, 0x7b, 0xb9, 0xaf, 0x37, 0xcb, 0xee, 0xea, 0x94, + 0x6b, 0x3e, 0x9a, 0x20, 0xcb, 0x0a, 0xe8, 0x21, 0x40, 0x32, 0xff, 0x90, 0xd5, 0x3a, 0x8d, 0x83, + 0x2f, 0xf3, 0xe5, 0x2b, 0x24, 0xb3, 0x5c, 0x89, 0xfb, 0xbf, 0x09, 0x3b, 0xab, 0x96, 0xa0, 0x3f, + 0x02, 0x0c, 0x91, 0xfb, 0x57, 0xb1, 0xcf, 0x25, 0xce, 0x84, 0xed, 0x96, 0x85, 0xf5, 0xe6, 0x19, + 0x3d, 0x83, 0xe5, 0xf2, 0xe9, 0x19, 0x6c, 0x0d, 0x26, 0xa3, 0x51, 0xca, 0xca, 0xf0, 0xcf, 0x09, + 0x0a, 0xb9, 0x4a, 0x5c, 0x4a, 0x71, 0x52, 0x4c, 0xeb, 0x19, 0xac, 0x5c, 0x49, 0x7f, 0x85, 0xf6, + 0x02, 0x12, 0x71, 0x14, 0x0a, 0xbd, 0xdb, 0x56, 0x38, 0x75, 0x52, 0xca, 0xeb, 0x19, 0x6c, 0xa9, + 0x96, 0x1e, 0xc3, 0x26, 0x26, 0x49, 0x94, 0xcc, 0xc9, 0xaa, 0x8a, 0xec, 0x8b, 0x32, 0xd9, 0x71, + 0x3e, 0xa9, 0x67, 0xb0, 0x62, 0xd5, 0x51, 0x0d, 0xd6, 0xfe, 0x4a, 0xad, 0x72, 0xff, 0x21, 0xd0, + 0x2a, 0xba, 0x41, 0x77, 0x60, 0x2d, 0x75, 0x23, 0xdb, 0x71, 0x3a, 0xa0, 0xdf, 0x43, 0x6d, 0xb6, + 0x25, 0x2c, 0xd3, 0xa9, 0xbc, 0x65, 0x54, 0x59, 0x3e, 0x75, 0xa1, 0x99, 0x6d, 0xb9, 0x73, 0x2e, + 0x87, 0x56, 0x45, 0xf1, 0x16, 0x30, 0xf7, 0x5f, 0x02, 0xdb, 0x2b, 0x2c, 0x7d, 0x1f, 0x31, 0xff, + 0x11, 0xfd, 0x61, 0x95, 0x27, 0xf2, 0x3e, 0x6a, 0xba, 0xf0, 0xd9, 0xd2, 0x44, 0x53, 0x25, 0x6a, + 0xa2, 0xb3, 0x33, 0x5f, 0x07, 0xe9, 0xf9, 0x80, 0x49, 0xd2, 0x8d, 0x7c, 0xbd, 0x9f, 0xaa, 0x2c, + 0x0b, 0xdd, 0x6b, 0x3d, 0x66, 0xad, 0xa2, 0x1f, 0x0e, 0xa2, 0xd2, 0x0d, 0x40, 0x96, 0x6e, 0x80, + 0xa5, 0x33, 0xdb, 0x5c, 0x71, 0x66, 0x7f, 0x7b, 0x03, 0xa0, 0x84, 0xa5, 0x8b, 0x08, 0xda, 0x02, + 0xb8, 0x0a, 0xf1, 0xef, 0x18, 0x3d, 0x89, 0x7e, 0xdb, 0xa0, 0x6d, 0x68, 0x9e, 0xa2, 0x9c, 0xab, + 0x6f, 0x13, 0x6a, 0xc1, 0x4e, 0x69, 0xc4, 0xfa, 0x8d, 0x49, 0xdb, 0xd0, 0x50, 0x8f, 0xbf, 0x0d, + 0x06, 0x02, 0x65, 0xfb, 0xb1, 0x72, 0xf4, 0xd3, 0x87, 0x17, 0x9b, 0x3c, 0xbd, 0xd8, 0xe4, 0xe3, + 0x8b, 0x4d, 0x1e, 0x5f, 0x6d, 0xe3, 0xe9, 0xd5, 0x36, 0x9e, 0x5f, 0x6d, 0xe3, 0xe6, 0x9b, 0x37, + 0xde, 0xa8, 0xb7, 0xeb, 0xea, 0xef, 0xbb, 0x4f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x1b, 0x35, 0x02, + 0x10, 0x83, 0x07, 0x00, 0x00, } func (m *RootChange) Marshal() (dAtA []byte, err error) { @@ -1464,7 +1473,14 @@ func (m *TreeErrorResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { if m.ErrCode != 0 { i = encodeVarintTreechange(dAtA, i, uint64(m.ErrCode)) i-- - dAtA[i] = 0x8 + dAtA[i] = 0x10 + } + if len(m.Error) > 0 { + i -= len(m.Error) + copy(dAtA[i:], m.Error) + i = encodeVarintTreechange(dAtA, i, uint64(len(m.Error))) + i-- + dAtA[i] = 0xa } return len(dAtA) - i, nil } @@ -1792,6 +1808,10 @@ func (m *TreeErrorResponse) Size() (n int) { } var l int _ = l + l = len(m.Error) + if l > 0 { + n += 1 + l + sovTreechange(uint64(l)) + } if m.ErrCode != 0 { n += 1 + sovTreechange(uint64(m.ErrCode)) } @@ -3393,6 +3413,38 @@ func (m *TreeErrorResponse) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTreechange + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTreechange + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthTreechange + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Error = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ErrCode", wireType) } From 6f42dc2f7a318f936857e711104dbdc6244e56ab Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 20 Apr 2023 11:26:53 +0200 Subject: [PATCH 22/45] Refactor genchanges --- .../object/tree/synctree/utils_test.go | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index 1d844e90..d0b207b4 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -382,7 +382,7 @@ type genResult struct { snapshotId string } -// genChanges generates several levels of tree changes where each level is connected with only previous one +// genChanges generates several levels of tree changes where each level is connected only with previous one func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res genResult) { src := rand.NewSource(time.Now().Unix()) rnd := rand.New(src) @@ -393,10 +393,18 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge prevHeads = append(prevHeads, params.prevHeads...) for i := 0; i < params.levels; i++ { - if params.isSnapshot() { - newId := fmt.Sprintf("%s.%d.%d", params.prefix, params.startIdx+i, 0) - newCh := creator.CreateRaw(newId, params.aclId, snapshotId, true, prevHeads...) + var ( + newHeads []string + usedIds = map[string]struct{}{} + ) + newChange := func(isSnapshot bool, idx int, prevIds []string) string { + newId := fmt.Sprintf("%s.%d.%d", params.prefix, params.startIdx+i, idx) + newCh := creator.CreateRaw(newId, params.aclId, snapshotId, isSnapshot, prevIds...) res.changes = append(res.changes, newCh) + return newId + } + if params.isSnapshot() { + newId := newChange(true, 0, prevHeads) prevHeads = []string{newId} snapshotId = newId continue @@ -405,10 +413,6 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge if perLevel == 0 { perLevel = 1 } - var ( - newHeads []string - usedIds = map[string]struct{}{} - ) for j := 0; j < perLevel; j++ { prevConns := rnd.Intn(len(prevHeads)) if prevConns == 0 { @@ -428,14 +432,12 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge prevHeads = unusedIds prevConns = len(prevHeads) } - var prevChId []string + var prevIds []string for k := 0; k < prevConns; k++ { - prevChId = append(prevChId, prevHeads[k]) + prevIds = append(prevIds, prevHeads[k]) usedIds[prevHeads[k]] = struct{}{} } - newId := fmt.Sprintf("%s.%d.%d", params.prefix, params.startIdx+i, j) - newCh := creator.CreateRaw(newId, params.aclId, snapshotId, false, prevChId...) - res.changes = append(res.changes, newCh) + newId := newChange(false, j, prevIds) newHeads = append(newHeads, newId) } prevHeads = newHeads From 2894b8ef665ec2d86aac722b92028681f3948c7b Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 20 Apr 2023 11:46:19 +0200 Subject: [PATCH 23/45] Fix full sync request condition --- commonspace/objectsync/objectsync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commonspace/objectsync/objectsync.go b/commonspace/objectsync/objectsync.go index 02e6019c..e22db4bd 100644 --- a/commonspace/objectsync/objectsync.go +++ b/commonspace/objectsync/objectsync.go @@ -134,5 +134,5 @@ func (s *objectSync) sendError(ctx context.Context, root *treechangeproto.RawTre } func (s *objectSync) isEmptyFullSyncRequest(msg *treechangeproto.TreeSyncMessage) bool { - return len(msg.GetContent().GetFullSyncRequest().GetHeads()) == 0 + return msg.GetContent().GetFullSyncRequest() != nil && len(msg.GetContent().GetFullSyncRequest().GetHeads()) == 0 } From 7979c48ca3e3242184f1a479725da1022628ceb6 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 20 Apr 2023 12:46:36 +0200 Subject: [PATCH 24/45] Change error to typed in treeremotegetter --- commonspace/object/tree/synctree/treeremotegetter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commonspace/object/tree/synctree/treeremotegetter.go b/commonspace/object/tree/synctree/treeremotegetter.go index 3156373f..7c401a5e 100644 --- a/commonspace/object/tree/synctree/treeremotegetter.go +++ b/commonspace/object/tree/synctree/treeremotegetter.go @@ -119,7 +119,7 @@ func (t treeRemoteGetter) getTree(ctx context.Context) (treeStorage treestorage. err = rpcerr.Err(errResp.ErrCode) return case resp.GetContent().GetFullSyncResponse() == nil: - err = fmt.Errorf("expected to get full sync response, but got something else") + err = treechangeproto.ErrUnexpected return default: break From e896a88d29384a1c942c0ae6ae9c1dba3f70a9fd Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 20 Apr 2023 12:48:25 +0200 Subject: [PATCH 25/45] Change test name for fuzzy merge --- commonspace/object/tree/synctree/syncprotocol_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commonspace/object/tree/synctree/syncprotocol_test.go b/commonspace/object/tree/synctree/syncprotocol_test.go index cc7ebec4..d7c7a935 100644 --- a/commonspace/object/tree/synctree/syncprotocol_test.go +++ b/commonspace/object/tree/synctree/syncprotocol_test.go @@ -59,7 +59,7 @@ func TestEmptyClientGetsFullHistory(t *testing.T) { require.Len(t, fullResponseMsg.changes, 2) } -func TestRandomMerge(t *testing.T) { +func TestTreeFuzzyMerge(t *testing.T) { var ( rnd = rand.New(rand.NewSource(time.Now().Unix())) levels = 20 From 5d13fb6a6ebee4ed634afd91fb326030bcf0a18a Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 20 Apr 2023 12:52:25 +0200 Subject: [PATCH 26/45] Change config parameter name --- commonspace/config.go | 6 +++--- commonspace/spaceservice.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/commonspace/config.go b/commonspace/config.go index f7444fdd..f23f8e33 100644 --- a/commonspace/config.go +++ b/commonspace/config.go @@ -5,7 +5,7 @@ type ConfigGetter interface { } type Config struct { - GCTTL int `yaml:"gcTTL"` - SyncPeriod int `yaml:"syncPeriod"` - TreeInMemoryData bool `yaml:"treeInMemoryData"` + GCTTL int `yaml:"gcTTL"` + SyncPeriod int `yaml:"syncPeriod"` + TreeNoInMemoryData bool `yaml:"treeNoInMemoryData"` } diff --git a/commonspace/spaceservice.go b/commonspace/spaceservice.go index ef59e336..d59ae751 100644 --- a/commonspace/spaceservice.go +++ b/commonspace/spaceservice.go @@ -154,7 +154,7 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) { syncStatus = syncstatus.NewSyncStatusProvider(st.Id(), syncstatus.DefaultDeps(lastConfiguration, st)) } var builder objecttree.BuildObjectTreeFunc - if s.config.TreeInMemoryData { + if s.config.TreeNoInMemoryData { builder = objecttree.BuildEmptyDataObjectTree } else { builder = objecttree.BuildObjectTree From 5c954d1bdd6ec733c651f3a5765a7760f9903b98 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 20 Apr 2023 17:12:39 +0200 Subject: [PATCH 27/45] Change inmemory parameter naming --- commonspace/config.go | 6 +++--- commonspace/spaceservice.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/commonspace/config.go b/commonspace/config.go index f23f8e33..e5485068 100644 --- a/commonspace/config.go +++ b/commonspace/config.go @@ -5,7 +5,7 @@ type ConfigGetter interface { } type Config struct { - GCTTL int `yaml:"gcTTL"` - SyncPeriod int `yaml:"syncPeriod"` - TreeNoInMemoryData bool `yaml:"treeNoInMemoryData"` + GCTTL int `yaml:"gcTTL"` + SyncPeriod int `yaml:"syncPeriod"` + KeepTreeDataInMemory bool `yaml:"keepTreeDataInMemory"` } diff --git a/commonspace/spaceservice.go b/commonspace/spaceservice.go index d59ae751..addf1f7f 100644 --- a/commonspace/spaceservice.go +++ b/commonspace/spaceservice.go @@ -154,10 +154,10 @@ func (s *spaceService) NewSpace(ctx context.Context, id string) (Space, error) { syncStatus = syncstatus.NewSyncStatusProvider(st.Id(), syncstatus.DefaultDeps(lastConfiguration, st)) } var builder objecttree.BuildObjectTreeFunc - if s.config.TreeNoInMemoryData { - builder = objecttree.BuildEmptyDataObjectTree - } else { + if s.config.KeepTreeDataInMemory { builder = objecttree.BuildObjectTree + } else { + builder = objecttree.BuildEmptyDataObjectTree } peerManager, err := s.peermanagerProvider.NewPeerManager(ctx, id) From a57fb52b63dd29f09d350e6cad8d9b91c512db8e Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 21 Apr 2023 16:04:30 +0200 Subject: [PATCH 28/45] fix panic for empty node addresses --- nodeconf/nodeconf.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nodeconf/nodeconf.go b/nodeconf/nodeconf.go index 3d673fe8..ac665b9f 100644 --- a/nodeconf/nodeconf.go +++ b/nodeconf/nodeconf.go @@ -85,6 +85,9 @@ func (c *nodeConf) CoordinatorPeers() []string { func (c *nodeConf) PeerAddresses(peerId string) (addrs []string, ok bool) { addrs, ok = c.addrs[peerId] + if ok && len(addrs) == 0 { + return nil, false + } return } From 443ab3c24bfb6e9811675391b7da5d0093abea91 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 21 Apr 2023 17:02:51 +0200 Subject: [PATCH 29/45] update libp2p --- go.mod | 23 ++++++++++++-------- go.sum | 68 +++++++++++++++++++++++++++++++++++++--------------------- 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index 54625d94..69b628e1 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-merkledag v0.10.0 github.com/ipfs/go-unixfs v0.4.5 - github.com/libp2p/go-libp2p v0.25.1 + github.com/libp2p/go-libp2p v0.27.1 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multibase v0.2.0 github.com/multiformats/go-multihash v0.2.1 @@ -34,25 +34,28 @@ require ( github.com/zeebo/errs v1.3.0 go.uber.org/zap v1.24.0 golang.org/x/crypto v0.8.0 - golang.org/x/exp v0.0.0-20230321023759-10a507213a29 + golang.org/x/exp v0.0.0-20230420155640-133eef4313cb golang.org/x/net v0.9.0 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 + gopkg.in/yaml.v3 v3.0.1 storj.io/drpc v0.0.32 ) require ( github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/benbjohnson/clock v1.3.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/fogleman/gg v1.3.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/pprof v0.0.0-20230406165453-00490a63f317 // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/ipfs/bbloom v0.0.4 // indirect @@ -73,15 +76,16 @@ require ( github.com/ipld/go-ipld-prime v0.20.0 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect - github.com/klauspost/cpuid/v2 v2.2.3 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.18 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr v0.8.0 // indirect - github.com/multiformats/go-multicodec v0.8.0 // indirect + github.com/multiformats/go-multiaddr v0.9.0 // indirect + github.com/multiformats/go-multicodec v0.8.1 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect @@ -91,18 +95,19 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect + github.com/quic-go/quic-go v0.34.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect go.opentelemetry.io/otel v1.11.2 // indirect go.opentelemetry.io/otel/trace v1.11.2 // indirect go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.9.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/image v0.6.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.7.0 // indirect + golang.org/x/tools v0.8.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/protobuf v1.30.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect ) diff --git a/go.sum b/go.sum index 90ebebbb..734fcc2f 100644 --- a/go.sum +++ b/go.sum @@ -12,7 +12,8 @@ github.com/anytypeio/go-slip10 v0.0.0-20200330112030-a352ca8495e4/go.mod h1:/8GI github.com/anytypeio/go-slip21 v0.0.0-20200218204727-e2e51e20ab51 h1:3Y+18zBC8LZgcL3l2dgoTEIzIUzCZa/kN0UV3ZWpbuA= github.com/anytypeio/go-slip21 v0.0.0-20200218204727-e2e51e20ab51/go.mod h1:SoKy+W8Mf6v7XBV30xFWkIFMs7UnXwsNGrGV12yVkEs= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.1 h1:Heo0FGXzOxUHquZbraxt+tT7UXVDhesUQH5ISbsOkCQ= +github.com/benbjohnson/clock v1.3.1/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -31,11 +32,13 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= -github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -43,6 +46,7 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= @@ -65,6 +69,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/pprof v0.0.0-20230406165453-00490a63f317 h1:hFhpt7CTmR3DX+b4R19ydQFtofxT0Sv3QsKNMVQYTMQ= +github.com/google/pprof v0.0.0-20230406165453-00490a63f317/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -79,7 +85,7 @@ github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3 github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U= github.com/huandu/skiplist v1.2.0 h1:gox56QD77HzSC0w+Ws3MH3iie755GBJU1OER3h5VsYw= github.com/huandu/skiplist v1.2.0/go.mod h1:7v3iFjLcSAzO4fN5B8dvebvo/qsfumiLiDXMrPiHF9w= -github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= +github.com/huin/goupnp v1.1.0 h1:gEe0Dp/lZmPZiDFzJJaOfUpOvv2MKUkoBX8lDrn9vKU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= @@ -168,12 +174,14 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -185,9 +193,9 @@ github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoR github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= -github.com/libp2p/go-libp2p v0.25.1 h1:YK+YDCHpYyTvitKWVxa5PfElgIpOONU01X5UcLEwJGA= -github.com/libp2p/go-libp2p v0.25.1/go.mod h1:xnK9/1d9+jeQCVvi/f1g12KqtVi/jP/SijtKV1hML3g= -github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= +github.com/libp2p/go-libp2p v0.27.1 h1:k1u6RHsX3hqKnslDjsSgLNURxJ3O1atIZCY4gpMbbus= +github.com/libp2p/go-libp2p v0.27.1/go.mod h1:FAvvfQa/YOShUYdiSS03IR9OXzkcJXwcNA2FUCh9ImE= +github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= @@ -197,11 +205,11 @@ github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rB github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= +github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= +github.com/miekg/dns v1.1.53 h1:ZBkuHr5dxHtB1caEOlZTLPo7D3L3TWckgUUs/RHfDxw= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= @@ -218,16 +226,16 @@ github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYg github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= -github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sggzwC/NcqbDQ= +github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= -github.com/multiformats/go-multicodec v0.8.0 h1:evBmgkbSQux+Ds2IgfhkO38Dl2GDtRW8/Rp6YiSHX/Q= -github.com/multiformats/go-multicodec v0.8.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= +github.com/multiformats/go-multicodec v0.8.1 h1:ycepHwavHafh3grIbR1jIXnKCsFm0fqsfEOsJ8NtKE8= +github.com/multiformats/go-multicodec v0.8.1/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= @@ -242,6 +250,8 @@ github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= @@ -262,6 +272,12 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= +github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U= +github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E= +github.com/quic-go/quic-go v0.34.0 h1:OvOJ9LFjTySgwOTYUZmNoq0FzVicP8YujpV0kB7m2lU= +github.com/quic-go/quic-go v0.34.0/go.mod h1:+4CVgVppm0FNjpG3UcX8Joi/frKOH7/ciD5yGcwOO1g= +github.com/quic-go/webtransport-go v0.5.2 h1:GA6Bl6oZY+g/flt00Pnu0XtivSD8vukOu3lYhJjnGEk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -324,8 +340,8 @@ go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpK go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= @@ -342,8 +358,8 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230420155640-133eef4313cb h1:rhjz/8Mbfa8xROFiH+MQphmAmgqRM0bOMnytznhWEXk= +golang.org/x/exp v0.0.0-20230420155640-133eef4313cb/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.6.0 h1:bR8b5okrPI3g/gyZakLZHeWxAR8Dn5CyxXv1hLH5g/4= golang.org/x/image v0.6.0/go.mod h1:MXLdDR43H7cDJq5GEGXEVeeNhPgi+YYEQ2pC1byI1x0= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -352,8 +368,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -390,8 +406,8 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -403,6 +419,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -416,8 +433,9 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From f0c4ae3f5e473892e6cfe61e097e718359be21bb Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 21 Apr 2023 20:12:47 +0200 Subject: [PATCH 30/45] Move validation to any-sync --- commonspace/payloads.go | 180 +++++ commonspace/payloads_test.go | 655 ++++++++++++++++++ commonspace/spaceservice.go | 16 +- commonspace/spacestorage/spacestorage.go | 181 ----- commonspace/spacestorage/spacestorage_test.go | 653 ----------------- 5 files changed, 847 insertions(+), 838 deletions(-) create mode 100644 commonspace/payloads_test.go diff --git a/commonspace/payloads.go b/commonspace/payloads.go index 46d2515d..4d601155 100644 --- a/commonspace/payloads.go +++ b/commonspace/payloads.go @@ -1,14 +1,20 @@ package commonspace import ( + "fmt" + "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/spacestorage" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/util/cidutil" "github.com/anytypeio/any-sync/util/crypto" + "github.com/gogo/protobuf/proto" "hash/fnv" "math/rand" + "strconv" + "strings" "time" ) @@ -180,3 +186,177 @@ func storagePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload sp } return } + +func validateSpaceStorageCreatePayload(payload spacestorage.SpaceStorageCreatePayload) (err error) { + err = validateCreateSpaceHeaderPayload(payload.SpaceHeaderWithId) + if err != nil { + return + } + aclSpaceId, err := validateCreateSpaceAclPayload(payload.AclWithId) + if err != nil { + return + } + aclHeadId, settingsSpaceId, err := validateCreateSpaceSettingsPayload(payload.SpaceSettingsWithId) + if err != nil { + return + } + if aclSpaceId != payload.SpaceHeaderWithId.Id || aclSpaceId != settingsSpaceId { + err = spacestorage.ErrIncorrectSpaceHeader + return + } + if aclHeadId != payload.AclWithId.Id { + err = spacestorage.ErrIncorrectSpaceHeader + return + } + return +} + +func validateCreateSpaceHeaderPayload(rawHeaderWithId *spacesyncproto.RawSpaceHeaderWithId) (err error) { + var rawSpaceHeader spacesyncproto.RawSpaceHeader + err = proto.Unmarshal(rawHeaderWithId.RawHeader, &rawSpaceHeader) + if err != nil { + return + } + var header spacesyncproto.SpaceHeader + err = proto.Unmarshal(rawSpaceHeader.SpaceHeader, &header) + if err != nil { + return + } + split := strings.Split(rawHeaderWithId.Id, ".") + if len(split) != 2 { + return spacestorage.ErrIncorrectSpaceHeader + } + if !cidutil.VerifyCid(rawHeaderWithId.RawHeader, split[0]) { + err = objecttree.ErrIncorrectCid + return + } + payloadIdentity, err := crypto.UnmarshalEd25519PublicKeyProto(header.Identity) + if err != nil { + return + } + res, err := payloadIdentity.Verify(rawSpaceHeader.SpaceHeader, rawSpaceHeader.Signature) + if err != nil || !res { + err = spacestorage.ErrIncorrectSpaceHeader + return + } + id, err := cidutil.NewCidFromBytes(rawHeaderWithId.RawHeader) + if err != nil { + return + } + requiredSpaceId := fmt.Sprintf("%s.%s", id, strconv.FormatUint(header.ReplicationKey, 36)) + if requiredSpaceId != rawHeaderWithId.Id { + err = spacestorage.ErrIncorrectSpaceHeader + return + } + + return +} + +func validateCreateSpaceAclPayload(rawWithId *aclrecordproto.RawAclRecordWithId) (spaceId string, err error) { + if !cidutil.VerifyCid(rawWithId.Payload, rawWithId.Id) { + err = objecttree.ErrIncorrectCid + return + } + var rawAcl aclrecordproto.RawAclRecord + err = proto.Unmarshal(rawWithId.Payload, &rawAcl) + if err != nil { + return + } + var aclRoot aclrecordproto.AclRoot + err = proto.Unmarshal(rawAcl.Payload, &aclRoot) + if err != nil { + return + } + payloadIdentity, err := crypto.UnmarshalEd25519PublicKeyProto(aclRoot.Identity) + if err != nil { + return + } + res, err := payloadIdentity.Verify(rawAcl.Payload, rawAcl.Signature) + if err != nil || !res { + err = spacestorage.ErrIncorrectSpaceHeader + return + } + masterKey, err := crypto.UnmarshalEd25519PublicKeyProto(aclRoot.MasterKey) + if err != nil { + return + } + rawIdentity, err := payloadIdentity.Raw() + if err != nil { + return + } + res, err = masterKey.Verify(rawIdentity, aclRoot.IdentitySignature) + if err != nil || !res { + err = spacestorage.ErrIncorrectSpaceHeader + return + } + spaceId = aclRoot.SpaceId + + return +} + +func validateCreateSpaceSettingsPayload(rawWithId *treechangeproto.RawTreeChangeWithId) (aclHeadId string, spaceId string, err error) { + var raw treechangeproto.RawTreeChange + err = proto.Unmarshal(rawWithId.RawChange, &raw) + if err != nil { + return + } + var rootChange treechangeproto.RootChange + err = proto.Unmarshal(raw.Payload, &rootChange) + if err != nil { + return + } + payloadIdentity, err := crypto.UnmarshalEd25519PublicKeyProto(rootChange.Identity) + if err != nil { + return + } + res, err := payloadIdentity.Verify(raw.Payload, raw.Signature) + if err != nil || !res { + err = spacestorage.ErrIncorrectSpaceHeader + return + } + id, err := cidutil.NewCidFromBytes(rawWithId.RawChange) + if id != rawWithId.Id { + err = spacestorage.ErrIncorrectSpaceHeader + return + } + spaceId = rootChange.SpaceId + aclHeadId = rootChange.AclHeadId + + return +} + +// ValidateSpaceHeader Used in coordinator +func ValidateSpaceHeader(spaceId string, header []byte, identity crypto.PubKey) (err error) { + split := strings.Split(spaceId, ".") + if len(split) != 2 { + return spacestorage.ErrIncorrectSpaceHeader + } + if !cidutil.VerifyCid(header, split[0]) { + err = objecttree.ErrIncorrectCid + return + } + raw := &spacesyncproto.RawSpaceHeader{} + err = proto.Unmarshal(header, raw) + if err != nil { + return + } + payload := &spacesyncproto.SpaceHeader{} + err = proto.Unmarshal(raw.SpaceHeader, payload) + if err != nil { + return + } + payloadIdentity, err := crypto.UnmarshalEd25519PublicKeyProto(payload.Identity) + if err != nil { + return + } + if identity != nil && !payloadIdentity.Equals(identity) { + err = spacestorage.ErrIncorrectSpaceHeader + return + } + res, err := identity.Verify(raw.SpaceHeader, raw.Signature) + if err != nil || !res { + err = spacestorage.ErrIncorrectSpaceHeader + return + } + return +} diff --git a/commonspace/payloads_test.go b/commonspace/payloads_test.go new file mode 100644 index 00000000..13535b34 --- /dev/null +++ b/commonspace/payloads_test.go @@ -0,0 +1,655 @@ +package commonspace + +import ( + "fmt" + "github.com/anytypeio/any-sync/commonspace/object/accountdata" + "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/commonspace/spacestorage" + "github.com/anytypeio/any-sync/commonspace/spacesyncproto" + "github.com/anytypeio/any-sync/util/cidutil" + "github.com/anytypeio/any-sync/util/crypto" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "math/rand" + "strconv" + "testing" + "time" +) + +func TestSuccessHeaderPayloadForSpaceCreate(t *testing.T) { + accountKeys, err := accountdata.NewRandom() + require.NoError(t, err) + _, rawHeaderWithId, err := rawHeaderWithId(accountKeys) + require.NoError(t, err) + err = validateCreateSpaceHeaderPayload(rawHeaderWithId) + require.NoError(t, err) +} + +func TestFailedHeaderPayloadForSpaceCreate_InvalidFormatSpaceId(t *testing.T) { + accountKeys, err := accountdata.NewRandom() + require.NoError(t, err) + identity, err := accountKeys.SignKey.GetPublic().Marshall() + require.NoError(t, err) + spaceHeaderSeed := make([]byte, 32) + _, err = rand.Read(spaceHeaderSeed) + require.NoError(t, err) + spaceHeaderPayload := make([]byte, 32) + _, err = rand.Read(spaceHeaderPayload) + require.NoError(t, err) + replicationKey := rand.Uint64() + header := &spacesyncproto.SpaceHeader{ + Identity: identity, + Timestamp: time.Now().Unix(), + SpaceType: "SpaceType", + ReplicationKey: replicationKey, + Seed: spaceHeaderSeed, + SpaceHeaderPayload: spaceHeaderPayload, + } + marhalled, err := header.Marshal() + require.NoError(t, err) + signature, err := accountKeys.SignKey.Sign(marhalled) + require.NoError(t, err) + rawHeader := &spacesyncproto.RawSpaceHeader{ + SpaceHeader: marhalled, + Signature: signature, + } + marhalledRawHeader, err := rawHeader.Marshal() + require.NoError(t, err) + id, err := cidutil.NewCidFromBytes(marhalled) + require.NoError(t, err) + spaceId := fmt.Sprintf("%s%s", id, strconv.FormatUint(replicationKey, 36)) + rawHeaderWithId := &spacesyncproto.RawSpaceHeaderWithId{ + RawHeader: marhalledRawHeader, + Id: spaceId, + } + err = validateCreateSpaceHeaderPayload(rawHeaderWithId) + assert.EqualErrorf(t, err, spacestorage.ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", spacestorage.ErrIncorrectSpaceHeader, err) +} + +func TestFailedHeaderPayloadForSpaceCreate_CidIsWrong(t *testing.T) { + accountKeys, err := accountdata.NewRandom() + require.NoError(t, err) + identity, err := accountKeys.SignKey.GetPublic().Marshall() + require.NoError(t, err) + spaceHeaderSeed := make([]byte, 32) + _, err = rand.Read(spaceHeaderSeed) + require.NoError(t, err) + spaceHeaderPayload := make([]byte, 32) + _, err = rand.Read(spaceHeaderPayload) + require.NoError(t, err) + replicationKey := rand.Uint64() + header := &spacesyncproto.SpaceHeader{ + Identity: identity, + Timestamp: time.Now().Unix(), + SpaceType: "SpaceType", + ReplicationKey: replicationKey, + Seed: spaceHeaderSeed, + SpaceHeaderPayload: spaceHeaderPayload, + } + marhalled, err := header.Marshal() + require.NoError(t, err) + signature, err := accountKeys.SignKey.Sign(marhalled) + require.NoError(t, err) + rawHeader := &spacesyncproto.RawSpaceHeader{ + SpaceHeader: marhalled, + Signature: signature, + } + marhalledRawHeader, err := rawHeader.Marshal() + require.NoError(t, err) + id := "faisdfjpiocpoakopkop34" + spaceId := fmt.Sprintf("%s.%s", id, strconv.FormatUint(replicationKey, 36)) + rawHeaderWithId := &spacesyncproto.RawSpaceHeaderWithId{ + RawHeader: marhalledRawHeader, + Id: spaceId, + } + err = validateCreateSpaceHeaderPayload(rawHeaderWithId) + assert.EqualErrorf(t, err, objecttree.ErrIncorrectCid.Error(), "Error should be: %v, got: %v", objecttree.ErrIncorrectCid, err) +} + +func TestFailedHeaderPayloadForSpaceCreate_SignedWithAnotherIdentity(t *testing.T) { + accountKeys, err := accountdata.NewRandom() + require.NoError(t, err) + identity, err := accountKeys.SignKey.GetPublic().Marshall() + require.NoError(t, err) + spaceHeaderSeed := make([]byte, 32) + _, err = rand.Read(spaceHeaderSeed) + require.NoError(t, err) + spaceHeaderPayload := make([]byte, 32) + _, err = rand.Read(spaceHeaderPayload) + require.NoError(t, err) + replicationKey := rand.Uint64() + header := &spacesyncproto.SpaceHeader{ + Identity: identity, + Timestamp: time.Now().Unix(), + SpaceType: "SpaceType", + ReplicationKey: replicationKey, + Seed: spaceHeaderSeed, + SpaceHeaderPayload: spaceHeaderPayload, + } + marhalled, err := header.Marshal() + require.NoError(t, err) + anotherAccountKeys, err := accountdata.NewRandom() + signature, err := anotherAccountKeys.SignKey.Sign(marhalled) + require.NoError(t, err) + rawHeader := &spacesyncproto.RawSpaceHeader{ + SpaceHeader: marhalled, + Signature: signature, + } + marhalledRawHeader, err := rawHeader.Marshal() + require.NoError(t, err) + id := "faisdfjpiocpoakopkop34" + spaceId := fmt.Sprintf("%s.%s", id, strconv.FormatUint(replicationKey, 36)) + rawHeaderWithId := &spacesyncproto.RawSpaceHeaderWithId{ + RawHeader: marhalledRawHeader, + Id: spaceId, + } + err = validateCreateSpaceHeaderPayload(rawHeaderWithId) + assert.EqualErrorf(t, err, objecttree.ErrIncorrectCid.Error(), "Error should be: %v, got: %v", objecttree.ErrIncorrectCid, err) +} + +func TestSuccessAclPayloadSpace(t *testing.T) { + accountKeys, err := accountdata.NewRandom() + spaceId := "AnySpaceId" + _, rawWithId, err := rawAclWithId(accountKeys, spaceId) + require.NoError(t, err) + validationSpaceId, err := validateCreateSpaceAclPayload(rawWithId) + require.Equal(t, validationSpaceId, spaceId) + require.NoError(t, err) +} + +func TestFailAclPayloadSpace_IncorrectCid(t *testing.T) { + accountKeys, err := accountdata.NewRandom() + require.NoError(t, err) + identity, err := accountKeys.SignKey.GetPublic().Marshall() + require.NoError(t, err) + readKeyBytes := make([]byte, 32) + _, err = rand.Read(readKeyBytes) + require.NoError(t, err) + readKey, err := accountKeys.SignKey.GetPublic().Encrypt(readKeyBytes) + require.NoError(t, err) + masterKey, _, err := crypto.GenerateRandomEd25519KeyPair() + require.NoError(t, err) + rawIdentity, err := accountKeys.SignKey.GetPublic().Raw() + require.NoError(t, err) + identitySignature, err := masterKey.Sign(rawIdentity) + require.NoError(t, err) + rawMasterKey, err := masterKey.GetPublic().Marshall() + require.NoError(t, err) + aclRoot := aclrecordproto.AclRoot{ + Identity: identity, + MasterKey: rawMasterKey, + SpaceId: "SpaceId", + EncryptedReadKey: readKey, + Timestamp: time.Now().Unix(), + IdentitySignature: identitySignature, + } + marshalled, err := aclRoot.Marshal() + require.NoError(t, err) + signature, err := accountKeys.SignKey.Sign(marshalled) + rawAclRecord := &aclrecordproto.RawAclRecord{ + Payload: marshalled, + Signature: signature, + } + marshalledRaw, err := rawAclRecord.Marshal() + require.NoError(t, err) + aclHeadId := "rand" + rawWithId := &aclrecordproto.RawAclRecordWithId{ + Payload: marshalledRaw, + Id: aclHeadId, + } + _, err = validateCreateSpaceAclPayload(rawWithId) + assert.EqualErrorf(t, err, objecttree.ErrIncorrectCid.Error(), "Error should be: %v, got: %v", objecttree.ErrIncorrectCid, err) +} + +func TestFailedAclPayloadSpace_IncorrectSignature(t *testing.T) { + accountKeys, err := accountdata.NewRandom() + require.NoError(t, err) + readKeyBytes := make([]byte, 32) + _, err = rand.Read(readKeyBytes) + require.NoError(t, err) + readKey, err := accountKeys.SignKey.GetPublic().Encrypt(readKeyBytes) + require.NoError(t, err) + masterKey, _, err := crypto.GenerateRandomEd25519KeyPair() + require.NoError(t, err) + rawIdentity, err := accountKeys.SignKey.GetPublic().Raw() + require.NoError(t, err) + identity, err := accountKeys.SignKey.GetPublic().Marshall() + identitySignature, err := masterKey.Sign(rawIdentity) + require.NoError(t, err) + rawMasterKey, err := masterKey.GetPublic().Raw() + require.NoError(t, err) + aclRoot := aclrecordproto.AclRoot{ + Identity: identity, + MasterKey: rawMasterKey, + SpaceId: "SpaceId", + EncryptedReadKey: readKey, + Timestamp: time.Now().Unix(), + IdentitySignature: identitySignature, + } + marshalled, err := aclRoot.Marshal() + require.NoError(t, err) + rawAclRecord := &aclrecordproto.RawAclRecord{ + Payload: marshalled, + Signature: marshalled, + } + marshalledRaw, err := rawAclRecord.Marshal() + require.NoError(t, err) + aclHeadId, err := cidutil.NewCidFromBytes(marshalledRaw) + require.NoError(t, err) + rawWithId := &aclrecordproto.RawAclRecordWithId{ + Payload: marshalledRaw, + Id: aclHeadId, + } + _, err = validateCreateSpaceAclPayload(rawWithId) + assert.NotNil(t, err) + assert.EqualErrorf(t, err, spacestorage.ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", spacestorage.ErrIncorrectSpaceHeader, err) +} + +func TestFailedAclPayloadSpace_IncorrectIdentitySignature(t *testing.T) { + spaceId := "AnySpaceId" + accountKeys, err := accountdata.NewRandom() + require.NoError(t, err) + readKeyBytes := make([]byte, 32) + _, err = rand.Read(readKeyBytes) + if err != nil { + return + } + readKey, err := accountKeys.SignKey.GetPublic().Encrypt(readKeyBytes) + if err != nil { + return + } + masterKey, _, err := crypto.GenerateRandomEd25519KeyPair() + if err != nil { + return + } + masterPubKey := masterKey.GetPublic() + identity, err := accountKeys.SignKey.GetPublic().Marshall() + if err != nil { + return + } + rawMasterKey, err := masterPubKey.Marshall() + if err != nil { + return + } + aclRoot := aclrecordproto.AclRoot{ + Identity: identity, + MasterKey: rawMasterKey, + SpaceId: spaceId, + EncryptedReadKey: readKey, + Timestamp: time.Now().Unix(), + IdentitySignature: identity, + } + marshalled, err := aclRoot.Marshal() + if err != nil { + return + } + signature, err := accountKeys.SignKey.Sign(marshalled) + rawAclRecord := &aclrecordproto.RawAclRecord{ + Payload: marshalled, + Signature: signature, + } + marshalledRaw, err := rawAclRecord.Marshal() + if err != nil { + return + } + aclHeadId, err := cidutil.NewCidFromBytes(marshalledRaw) + if err != nil { + return + } + rawWithId := &aclrecordproto.RawAclRecordWithId{ + Payload: marshalledRaw, + Id: aclHeadId, + } + _, err = validateCreateSpaceAclPayload(rawWithId) + assert.EqualErrorf(t, err, spacestorage.ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", spacestorage.ErrIncorrectSpaceHeader, err) +} + +func TestSuccessSettingsPayloadSpace(t *testing.T) { + accountKeys, err := accountdata.NewRandom() + require.NoError(t, err) + identity, err := accountKeys.SignKey.GetPublic().Marshall() + require.NoError(t, err) + spaceSettingsSeed := make([]byte, 32) + _, err = rand.Read(spaceSettingsSeed) + require.NoError(t, err) + changePayload := make([]byte, 32) + _, err = rand.Read(changePayload) + require.NoError(t, err) + spaceId := "SpaceId" + rootChange := &treechangeproto.RootChange{ + AclHeadId: "AclHeadId", + SpaceId: spaceId, + ChangeType: "ChangeType", + Timestamp: time.Now().Unix(), + Seed: spaceSettingsSeed, + Identity: identity, + ChangePayload: changePayload, + } + marshalledChange, err := rootChange.Marshal() + require.NoError(t, err) + signature, err := accountKeys.SignKey.Sign(marshalledChange) + require.NoError(t, err) + raw := &treechangeproto.RawTreeChange{ + Payload: marshalledChange, + Signature: signature, + } + marshalledRawChange, err := raw.Marshal() + id, err := cidutil.NewCidFromBytes(marshalledRawChange) + require.NoError(t, err) + rawIdChange := &treechangeproto.RawTreeChangeWithId{ + RawChange: marshalledRawChange, + Id: id, + } + _, validationSpaceId, err := validateCreateSpaceSettingsPayload(rawIdChange) + require.Equal(t, validationSpaceId, spaceId) + require.NoError(t, err) +} + +func TestFailSettingsPayloadSpace_InvalidSignature(t *testing.T) { + accountKeys, err := accountdata.NewRandom() + require.NoError(t, err) + identity, err := accountKeys.SignKey.GetPublic().Marshall() + require.NoError(t, err) + spaceSettingsSeed := make([]byte, 32) + _, err = rand.Read(spaceSettingsSeed) + require.NoError(t, err) + changePayload := make([]byte, 32) + _, err = rand.Read(changePayload) + require.NoError(t, err) + rootChange := &treechangeproto.RootChange{ + AclHeadId: "AclHeadId", + SpaceId: "SpaceId", + ChangeType: "ChangeType", + Timestamp: time.Now().Unix(), + Seed: spaceSettingsSeed, + Identity: identity, + ChangePayload: changePayload, + } + marshalledChange, err := rootChange.Marshal() + require.NoError(t, err) + raw := &treechangeproto.RawTreeChange{ + Payload: marshalledChange, + Signature: marshalledChange, + } + marshalledRawChange, err := raw.Marshal() + id, err := cidutil.NewCidFromBytes(marshalledRawChange) + require.NoError(t, err) + rawIdChange := &treechangeproto.RawTreeChangeWithId{ + RawChange: marshalledRawChange, + Id: id, + } + _, _, err = validateCreateSpaceSettingsPayload(rawIdChange) + assert.EqualErrorf(t, err, spacestorage.ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", spacestorage.ErrIncorrectSpaceHeader, err) +} + +func TestFailSettingsPayloadSpace_InvalidCid(t *testing.T) { + accountKeys, err := accountdata.NewRandom() + require.NoError(t, err) + identity, err := accountKeys.SignKey.GetPublic().Marshall() + require.NoError(t, err) + spaceSettingsSeed := make([]byte, 32) + _, err = rand.Read(spaceSettingsSeed) + require.NoError(t, err) + changePayload := make([]byte, 32) + _, err = rand.Read(changePayload) + require.NoError(t, err) + rootChange := &treechangeproto.RootChange{ + AclHeadId: "AclHeadId", + SpaceId: "SpaceId", + ChangeType: "ChangeType", + Timestamp: time.Now().Unix(), + Seed: spaceSettingsSeed, + Identity: identity, + ChangePayload: changePayload, + } + marshalledChange, err := rootChange.Marshal() + require.NoError(t, err) + signature, err := accountKeys.SignKey.Sign(marshalledChange) + require.NoError(t, err) + raw := &treechangeproto.RawTreeChange{ + Payload: marshalledChange, + Signature: signature, + } + marshalledRawChange, err := raw.Marshal() + id := "id" + require.NoError(t, err) + rawIdChange := &treechangeproto.RawTreeChangeWithId{ + RawChange: marshalledRawChange, + Id: id, + } + _, _, err = validateCreateSpaceSettingsPayload(rawIdChange) + assert.EqualErrorf(t, err, spacestorage.ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", spacestorage.ErrIncorrectSpaceHeader, err) +} + +func TestSuccessSameIds(t *testing.T) { + accountKeys, err := accountdata.NewRandom() + require.NoError(t, err) + spaceId, rawHeaderWithId, err := rawHeaderWithId(accountKeys) + require.NoError(t, err) + aclHeadId, rawAclWithId, err := rawAclWithId(accountKeys, spaceId) + require.NoError(t, err) + rawSettingsPayload, err := rawSettingsPayload(accountKeys, spaceId, aclHeadId) + spacePayload := spacestorage.SpaceStorageCreatePayload{ + AclWithId: rawAclWithId, + SpaceHeaderWithId: rawHeaderWithId, + SpaceSettingsWithId: rawSettingsPayload, + } + err = validateSpaceStorageCreatePayload(spacePayload) + require.NoError(t, err) +} + +func TestFailWithAclWrongSpaceId(t *testing.T) { + accountKeys, err := accountdata.NewRandom() + require.NoError(t, err) + spaceId, rawHeaderWithId, err := rawHeaderWithId(accountKeys) + require.NoError(t, err) + aclHeadId, rawAclWithId, err := rawAclWithId(accountKeys, "spaceId") + require.NoError(t, err) + rawSettingsPayload, err := rawSettingsPayload(accountKeys, spaceId, aclHeadId) + spacePayload := spacestorage.SpaceStorageCreatePayload{ + AclWithId: rawAclWithId, + SpaceHeaderWithId: rawHeaderWithId, + SpaceSettingsWithId: rawSettingsPayload, + } + err = validateSpaceStorageCreatePayload(spacePayload) + assert.EqualErrorf(t, err, spacestorage.ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", spacestorage.ErrIncorrectSpaceHeader, err) +} + +func TestFailWithSettingsWrongSpaceId(t *testing.T) { + accountKeys, err := accountdata.NewRandom() + require.NoError(t, err) + spaceId, rawHeaderWithId, err := rawHeaderWithId(accountKeys) + require.NoError(t, err) + aclHeadId, rawAclWithId, err := rawAclWithId(accountKeys, spaceId) + require.NoError(t, err) + rawSettingsPayload, err := rawSettingsPayload(accountKeys, "spaceId", aclHeadId) + spacePayload := spacestorage.SpaceStorageCreatePayload{ + AclWithId: rawAclWithId, + SpaceHeaderWithId: rawHeaderWithId, + SpaceSettingsWithId: rawSettingsPayload, + } + err = validateSpaceStorageCreatePayload(spacePayload) + assert.EqualErrorf(t, err, spacestorage.ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", spacestorage.ErrIncorrectSpaceHeader, err) +} + +func TestFailWithWrongAclHeadIdInSettingsPayload(t *testing.T) { + accountKeys, err := accountdata.NewRandom() + require.NoError(t, err) + spaceId, rawHeaderWithId, err := rawHeaderWithId(accountKeys) + require.NoError(t, err) + _, rawAclWithId, err := rawAclWithId(accountKeys, spaceId) + require.NoError(t, err) + rawSettingsPayload, err := rawSettingsPayload(accountKeys, spaceId, "aclHeadId") + spacePayload := spacestorage.SpaceStorageCreatePayload{ + AclWithId: rawAclWithId, + SpaceHeaderWithId: rawHeaderWithId, + SpaceSettingsWithId: rawSettingsPayload, + } + err = validateSpaceStorageCreatePayload(spacePayload) + assert.EqualErrorf(t, err, spacestorage.ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", spacestorage.ErrIncorrectSpaceHeader, err) +} + +func rawSettingsPayload(accountKeys *accountdata.AccountKeys, spaceId, aclHeadId string) (rawIdChange *treechangeproto.RawTreeChangeWithId, err error) { + identity, err := accountKeys.SignKey.GetPublic().Marshall() + if err != nil { + return + } + spaceSettingsSeed := make([]byte, 32) + _, err = rand.Read(spaceSettingsSeed) + if err != nil { + return + } + changePayload := make([]byte, 32) + _, err = rand.Read(changePayload) + if err != nil { + return + } + rootChange := &treechangeproto.RootChange{ + AclHeadId: aclHeadId, + SpaceId: spaceId, + ChangeType: "ChangeType", + Timestamp: time.Now().Unix(), + Seed: spaceSettingsSeed, + Identity: identity, + ChangePayload: changePayload, + } + marshalledChange, err := rootChange.Marshal() + if err != nil { + return + } + signature, err := accountKeys.SignKey.Sign(marshalledChange) + if err != nil { + return + } + raw := &treechangeproto.RawTreeChange{ + Payload: marshalledChange, + Signature: signature, + } + marshalledRawChange, err := raw.Marshal() + id, err := cidutil.NewCidFromBytes(marshalledRawChange) + if err != nil { + return + } + rawIdChange = &treechangeproto.RawTreeChangeWithId{ + RawChange: marshalledRawChange, + Id: id, + } + + return +} + +func rawAclWithId(accountKeys *accountdata.AccountKeys, spaceId string) (aclHeadId string, rawWithId *aclrecordproto.RawAclRecordWithId, err error) { + readKeyBytes := make([]byte, 32) + _, err = rand.Read(readKeyBytes) + if err != nil { + return + } + readKey, err := accountKeys.SignKey.GetPublic().Encrypt(readKeyBytes) + if err != nil { + return + } + masterKey, _, err := crypto.GenerateRandomEd25519KeyPair() + identity, err := accountKeys.SignKey.GetPublic().Marshall() + if err != nil { + return + } + masterPubKey := masterKey.GetPublic() + rawIdentity, err := accountKeys.SignKey.GetPublic().Raw() + if err != nil { + return + } + identitySignature, err := masterKey.Sign(rawIdentity) + if err != nil { + return + } + rawMasterKey, err := masterPubKey.Marshall() + if err != nil { + return + } + aclRoot := aclrecordproto.AclRoot{ + Identity: identity, + MasterKey: rawMasterKey, + SpaceId: spaceId, + EncryptedReadKey: readKey, + Timestamp: time.Now().Unix(), + IdentitySignature: identitySignature, + } + marshalled, err := aclRoot.Marshal() + if err != nil { + return + } + signature, err := accountKeys.SignKey.Sign(marshalled) + rawAclRecord := &aclrecordproto.RawAclRecord{ + Payload: marshalled, + Signature: signature, + } + marshalledRaw, err := rawAclRecord.Marshal() + if err != nil { + return + } + aclHeadId, err = cidutil.NewCidFromBytes(marshalledRaw) + if err != nil { + return + } + rawWithId = &aclrecordproto.RawAclRecordWithId{ + Payload: marshalledRaw, + Id: aclHeadId, + } + + return +} + +func rawHeaderWithId(accountKeys *accountdata.AccountKeys) (spaceId string, rawWithId *spacesyncproto.RawSpaceHeaderWithId, err error) { + identity, err := accountKeys.SignKey.GetPublic().Marshall() + if err != nil { + return + } + spaceHeaderSeed := make([]byte, 32) + _, err = rand.Read(spaceHeaderSeed) + if err != nil { + return + } + spaceHeaderPayload := make([]byte, 32) + _, err = rand.Read(spaceHeaderPayload) + if err != nil { + return + } + replicationKey := rand.Uint64() + header := &spacesyncproto.SpaceHeader{ + Identity: identity, + Timestamp: time.Now().Unix(), + SpaceType: "SpaceType", + ReplicationKey: replicationKey, + Seed: spaceHeaderSeed, + SpaceHeaderPayload: spaceHeaderPayload, + } + marhalled, err := header.Marshal() + if err != nil { + return + } + signature, err := accountKeys.SignKey.Sign(marhalled) + if err != nil { + return + } + rawHeader := &spacesyncproto.RawSpaceHeader{ + SpaceHeader: marhalled, + Signature: signature, + } + marhalledRawHeader, err := rawHeader.Marshal() + if err != nil { + return + } + id, err := cidutil.NewCidFromBytes(marhalledRawHeader) + if err != nil { + return + } + spaceId = fmt.Sprintf("%s.%s", id, strconv.FormatUint(replicationKey, 36)) + rawWithId = &spacesyncproto.RawSpaceHeaderWithId{ + RawHeader: marhalledRawHeader, + Id: spaceId, + } + + return +} diff --git a/commonspace/spaceservice.go b/commonspace/spaceservice.go index addf1f7f..062e1c12 100644 --- a/commonspace/spaceservice.go +++ b/commonspace/spaceservice.go @@ -80,7 +80,7 @@ func (s *spaceService) CreateSpace(ctx context.Context, payload SpaceCreatePaylo if err != nil { return } - store, err := s.storageProvider.CreateSpaceStorage(storageCreate) + store, err := s.createSpaceStorage(storageCreate) if err != nil { if err == spacestorage.ErrSpaceStorageExists { return storageCreate.SpaceHeaderWithId.Id, nil @@ -105,7 +105,7 @@ func (s *spaceService) DeriveSpace(ctx context.Context, payload SpaceDerivePaylo if err != nil { return } - store, err := s.storageProvider.CreateSpaceStorage(storageCreate) + store, err := s.createSpaceStorage(storageCreate) if err != nil { if err == spacestorage.ErrSpaceStorageExists { return storageCreate.SpaceHeaderWithId.Id, nil @@ -197,7 +197,7 @@ func (s *spaceService) addSpaceStorage(ctx context.Context, spaceDescription Spa Id: spaceDescription.SpaceSettingsId, }, } - st, err = s.storageProvider.CreateSpaceStorage(payload) + st, err = s.createSpaceStorage(payload) if err != nil { err = spacesyncproto.ErrUnexpected if err == spacestorage.ErrSpaceStorageExists { @@ -229,7 +229,7 @@ func (s *spaceService) getSpaceStorageFromRemote(ctx context.Context, id string) return } - st, err = s.storageProvider.CreateSpaceStorage(spacestorage.SpaceStorageCreatePayload{ + st, err = s.createSpaceStorage(spacestorage.SpaceStorageCreatePayload{ AclWithId: &aclrecordproto.RawAclRecordWithId{ Payload: res.Payload.AclPayload, Id: res.Payload.AclPayloadId, @@ -242,3 +242,11 @@ func (s *spaceService) getSpaceStorageFromRemote(ctx context.Context, id string) }) return } + +func (s *spaceService) createSpaceStorage(payload spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error) { + err := validateSpaceStorageCreatePayload(payload) + if err != nil { + return nil, err + } + return s.storageProvider.CreateSpaceStorage(payload) +} diff --git a/commonspace/spacestorage/spacestorage.go b/commonspace/spacestorage/spacestorage.go index 3f21c22e..25991922 100644 --- a/commonspace/spacestorage/spacestorage.go +++ b/commonspace/spacestorage/spacestorage.go @@ -4,19 +4,12 @@ package spacestorage import ( "context" "errors" - "fmt" "github.com/anytypeio/any-sync/app" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/acl/liststorage" - "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/commonspace/spacesyncproto" - "github.com/anytypeio/any-sync/util/cidutil" - "github.com/anytypeio/any-sync/util/crypto" - "github.com/gogo/protobuf/proto" - "strconv" - "strings" ) const CName = "common.commonspace.spacestorage" @@ -67,177 +60,3 @@ type SpaceStorageProvider interface { SpaceExists(id string) bool CreateSpaceStorage(payload SpaceStorageCreatePayload) (SpaceStorage, error) } - -func ValidateSpaceStorageCreatePayload(payload SpaceStorageCreatePayload) (err error) { - err = validateCreateSpaceHeaderPayload(payload.SpaceHeaderWithId) - if err != nil { - return - } - aclSpaceId, err := validateCreateSpaceAclPayload(payload.AclWithId) - if err != nil { - return - } - aclHeadId, settingsSpaceId, err := validateCreateSpaceSettingsPayload(payload.SpaceSettingsWithId) - if err != nil { - return - } - if aclSpaceId != payload.SpaceHeaderWithId.Id || aclSpaceId != settingsSpaceId { - err = ErrIncorrectSpaceHeader - return - } - if aclHeadId != payload.AclWithId.Id { - err = ErrIncorrectSpaceHeader - return - } - return -} - -func validateCreateSpaceHeaderPayload(rawHeaderWithId *spacesyncproto.RawSpaceHeaderWithId) (err error) { - var rawSpaceHeader spacesyncproto.RawSpaceHeader - err = proto.Unmarshal(rawHeaderWithId.RawHeader, &rawSpaceHeader) - if err != nil { - return - } - var header spacesyncproto.SpaceHeader - err = proto.Unmarshal(rawSpaceHeader.SpaceHeader, &header) - if err != nil { - return - } - split := strings.Split(rawHeaderWithId.Id, ".") - if len(split) != 2 { - return ErrIncorrectSpaceHeader - } - if !cidutil.VerifyCid(rawHeaderWithId.RawHeader, split[0]) { - err = objecttree.ErrIncorrectCid - return - } - payloadIdentity, err := crypto.UnmarshalEd25519PublicKeyProto(header.Identity) - if err != nil { - return - } - res, err := payloadIdentity.Verify(rawSpaceHeader.SpaceHeader, rawSpaceHeader.Signature) - if err != nil || !res { - err = ErrIncorrectSpaceHeader - return - } - id, err := cidutil.NewCidFromBytes(rawHeaderWithId.RawHeader) - if err != nil { - return - } - requiredSpaceId := fmt.Sprintf("%s.%s", id, strconv.FormatUint(header.ReplicationKey, 36)) - if requiredSpaceId != rawHeaderWithId.Id { - err = ErrIncorrectSpaceHeader - return - } - - return -} - -func validateCreateSpaceAclPayload(rawWithId *aclrecordproto.RawAclRecordWithId) (spaceId string, err error) { - if !cidutil.VerifyCid(rawWithId.Payload, rawWithId.Id) { - err = objecttree.ErrIncorrectCid - return - } - var rawAcl aclrecordproto.RawAclRecord - err = proto.Unmarshal(rawWithId.Payload, &rawAcl) - if err != nil { - return - } - var aclRoot aclrecordproto.AclRoot - err = proto.Unmarshal(rawAcl.Payload, &aclRoot) - if err != nil { - return - } - payloadIdentity, err := crypto.UnmarshalEd25519PublicKeyProto(aclRoot.Identity) - if err != nil { - return - } - res, err := payloadIdentity.Verify(rawAcl.Payload, rawAcl.Signature) - if err != nil || !res { - err = ErrIncorrectSpaceHeader - return - } - masterKey, err := crypto.UnmarshalEd25519PublicKeyProto(aclRoot.MasterKey) - if err != nil { - return - } - rawIdentity, err := payloadIdentity.Raw() - if err != nil { - return - } - res, err = masterKey.Verify(rawIdentity, aclRoot.IdentitySignature) - if err != nil || !res { - err = ErrIncorrectSpaceHeader - return - } - spaceId = aclRoot.SpaceId - - return -} - -func validateCreateSpaceSettingsPayload(rawWithId *treechangeproto.RawTreeChangeWithId) (aclHeadId string, spaceId string, err error) { - var raw treechangeproto.RawTreeChange - err = proto.Unmarshal(rawWithId.RawChange, &raw) - if err != nil { - return - } - var rootChange treechangeproto.RootChange - err = proto.Unmarshal(raw.Payload, &rootChange) - if err != nil { - return - } - payloadIdentity, err := crypto.UnmarshalEd25519PublicKeyProto(rootChange.Identity) - if err != nil { - return - } - res, err := payloadIdentity.Verify(raw.Payload, raw.Signature) - if err != nil || !res { - err = ErrIncorrectSpaceHeader - return - } - id, err := cidutil.NewCidFromBytes(rawWithId.RawChange) - if id != rawWithId.Id { - err = ErrIncorrectSpaceHeader - return - } - spaceId = rootChange.SpaceId - aclHeadId = rootChange.AclHeadId - - return -} - -// ValidateSpaceHeader Used in coordinator -func ValidateSpaceHeader(spaceId string, header []byte, identity crypto.PubKey) (err error) { - split := strings.Split(spaceId, ".") - if len(split) != 2 { - return ErrIncorrectSpaceHeader - } - if !cidutil.VerifyCid(header, split[0]) { - err = objecttree.ErrIncorrectCid - return - } - raw := &spacesyncproto.RawSpaceHeader{} - err = proto.Unmarshal(header, raw) - if err != nil { - return - } - payload := &spacesyncproto.SpaceHeader{} - err = proto.Unmarshal(raw.SpaceHeader, payload) - if err != nil { - return - } - payloadIdentity, err := crypto.UnmarshalEd25519PublicKeyProto(payload.Identity) - if err != nil { - return - } - if identity != nil && !payloadIdentity.Equals(identity) { - err = ErrIncorrectSpaceHeader - return - } - res, err := identity.Verify(raw.SpaceHeader, raw.Signature) - if err != nil || !res { - err = ErrIncorrectSpaceHeader - return - } - return -} diff --git a/commonspace/spacestorage/spacestorage_test.go b/commonspace/spacestorage/spacestorage_test.go index 8243291e..82a25c1d 100644 --- a/commonspace/spacestorage/spacestorage_test.go +++ b/commonspace/spacestorage/spacestorage_test.go @@ -1,655 +1,2 @@ package spacestorage -import ( - "crypto/rand" - "fmt" - "github.com/anytypeio/any-sync/commonspace/object/accountdata" - "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" - "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" - "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" - "github.com/anytypeio/any-sync/commonspace/spacesyncproto" - "github.com/anytypeio/any-sync/util/cidutil" - "github.com/anytypeio/any-sync/util/crypto" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - rand2 "golang.org/x/exp/rand" - "strconv" - "testing" - "time" -) - -func TestSuccessHeaderPayloadForSpaceCreate(t *testing.T) { - accountKeys, err := accountdata.NewRandom() - require.NoError(t, err) - _, rawHeaderWithId, err := rawHeaderWithId(accountKeys) - require.NoError(t, err) - err = validateCreateSpaceHeaderPayload(rawHeaderWithId) - require.NoError(t, err) -} - -func TestFailedHeaderPayloadForSpaceCreate_InvalidFormatSpaceId(t *testing.T) { - accountKeys, err := accountdata.NewRandom() - require.NoError(t, err) - identity, err := accountKeys.SignKey.GetPublic().Marshall() - require.NoError(t, err) - spaceHeaderSeed := make([]byte, 32) - _, err = rand.Read(spaceHeaderSeed) - require.NoError(t, err) - spaceHeaderPayload := make([]byte, 32) - _, err = rand.Read(spaceHeaderPayload) - require.NoError(t, err) - replicationKey := rand2.Uint64() - header := &spacesyncproto.SpaceHeader{ - Identity: identity, - Timestamp: time.Now().Unix(), - SpaceType: "SpaceType", - ReplicationKey: replicationKey, - Seed: spaceHeaderSeed, - SpaceHeaderPayload: spaceHeaderPayload, - } - marhalled, err := header.Marshal() - require.NoError(t, err) - signature, err := accountKeys.SignKey.Sign(marhalled) - require.NoError(t, err) - rawHeader := &spacesyncproto.RawSpaceHeader{ - SpaceHeader: marhalled, - Signature: signature, - } - marhalledRawHeader, err := rawHeader.Marshal() - require.NoError(t, err) - id, err := cidutil.NewCidFromBytes(marhalled) - require.NoError(t, err) - spaceId := fmt.Sprintf("%s%s", id, strconv.FormatUint(replicationKey, 36)) - rawHeaderWithId := &spacesyncproto.RawSpaceHeaderWithId{ - RawHeader: marhalledRawHeader, - Id: spaceId, - } - err = validateCreateSpaceHeaderPayload(rawHeaderWithId) - assert.EqualErrorf(t, err, ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", ErrIncorrectSpaceHeader, err) -} - -func TestFailedHeaderPayloadForSpaceCreate_CidIsWrong(t *testing.T) { - accountKeys, err := accountdata.NewRandom() - require.NoError(t, err) - identity, err := accountKeys.SignKey.GetPublic().Marshall() - require.NoError(t, err) - spaceHeaderSeed := make([]byte, 32) - _, err = rand.Read(spaceHeaderSeed) - require.NoError(t, err) - spaceHeaderPayload := make([]byte, 32) - _, err = rand.Read(spaceHeaderPayload) - require.NoError(t, err) - replicationKey := rand2.Uint64() - header := &spacesyncproto.SpaceHeader{ - Identity: identity, - Timestamp: time.Now().Unix(), - SpaceType: "SpaceType", - ReplicationKey: replicationKey, - Seed: spaceHeaderSeed, - SpaceHeaderPayload: spaceHeaderPayload, - } - marhalled, err := header.Marshal() - require.NoError(t, err) - signature, err := accountKeys.SignKey.Sign(marhalled) - require.NoError(t, err) - rawHeader := &spacesyncproto.RawSpaceHeader{ - SpaceHeader: marhalled, - Signature: signature, - } - marhalledRawHeader, err := rawHeader.Marshal() - require.NoError(t, err) - id := "faisdfjpiocpoakopkop34" - spaceId := fmt.Sprintf("%s.%s", id, strconv.FormatUint(replicationKey, 36)) - rawHeaderWithId := &spacesyncproto.RawSpaceHeaderWithId{ - RawHeader: marhalledRawHeader, - Id: spaceId, - } - err = validateCreateSpaceHeaderPayload(rawHeaderWithId) - assert.EqualErrorf(t, err, objecttree.ErrIncorrectCid.Error(), "Error should be: %v, got: %v", objecttree.ErrIncorrectCid, err) -} - -func TestFailedHeaderPayloadForSpaceCreate_SignedWithAnotherIdentity(t *testing.T) { - accountKeys, err := accountdata.NewRandom() - require.NoError(t, err) - identity, err := accountKeys.SignKey.GetPublic().Marshall() - require.NoError(t, err) - spaceHeaderSeed := make([]byte, 32) - _, err = rand.Read(spaceHeaderSeed) - require.NoError(t, err) - spaceHeaderPayload := make([]byte, 32) - _, err = rand.Read(spaceHeaderPayload) - require.NoError(t, err) - replicationKey := rand2.Uint64() - header := &spacesyncproto.SpaceHeader{ - Identity: identity, - Timestamp: time.Now().Unix(), - SpaceType: "SpaceType", - ReplicationKey: replicationKey, - Seed: spaceHeaderSeed, - SpaceHeaderPayload: spaceHeaderPayload, - } - marhalled, err := header.Marshal() - require.NoError(t, err) - anotherAccountKeys, err := accountdata.NewRandom() - signature, err := anotherAccountKeys.SignKey.Sign(marhalled) - require.NoError(t, err) - rawHeader := &spacesyncproto.RawSpaceHeader{ - SpaceHeader: marhalled, - Signature: signature, - } - marhalledRawHeader, err := rawHeader.Marshal() - require.NoError(t, err) - id := "faisdfjpiocpoakopkop34" - spaceId := fmt.Sprintf("%s.%s", id, strconv.FormatUint(replicationKey, 36)) - rawHeaderWithId := &spacesyncproto.RawSpaceHeaderWithId{ - RawHeader: marhalledRawHeader, - Id: spaceId, - } - err = validateCreateSpaceHeaderPayload(rawHeaderWithId) - assert.EqualErrorf(t, err, objecttree.ErrIncorrectCid.Error(), "Error should be: %v, got: %v", objecttree.ErrIncorrectCid, err) -} - -func TestSuccessAclPayloadSpace(t *testing.T) { - accountKeys, err := accountdata.NewRandom() - spaceId := "AnySpaceId" - _, rawWithId, err := rawAclWithId(accountKeys, spaceId) - require.NoError(t, err) - validationSpaceId, err := validateCreateSpaceAclPayload(rawWithId) - require.Equal(t, validationSpaceId, spaceId) - require.NoError(t, err) -} - -func TestFailAclPayloadSpace_IncorrectCid(t *testing.T) { - accountKeys, err := accountdata.NewRandom() - require.NoError(t, err) - identity, err := accountKeys.SignKey.GetPublic().Marshall() - require.NoError(t, err) - readKeyBytes := make([]byte, 32) - _, err = rand.Read(readKeyBytes) - require.NoError(t, err) - readKey, err := accountKeys.SignKey.GetPublic().Encrypt(readKeyBytes) - require.NoError(t, err) - masterKey, _, err := crypto.GenerateRandomEd25519KeyPair() - require.NoError(t, err) - rawIdentity, err := accountKeys.SignKey.GetPublic().Raw() - require.NoError(t, err) - identitySignature, err := masterKey.Sign(rawIdentity) - require.NoError(t, err) - rawMasterKey, err := masterKey.GetPublic().Marshall() - require.NoError(t, err) - aclRoot := aclrecordproto.AclRoot{ - Identity: identity, - MasterKey: rawMasterKey, - SpaceId: "SpaceId", - EncryptedReadKey: readKey, - Timestamp: time.Now().Unix(), - IdentitySignature: identitySignature, - } - marshalled, err := aclRoot.Marshal() - require.NoError(t, err) - signature, err := accountKeys.SignKey.Sign(marshalled) - rawAclRecord := &aclrecordproto.RawAclRecord{ - Payload: marshalled, - Signature: signature, - } - marshalledRaw, err := rawAclRecord.Marshal() - require.NoError(t, err) - aclHeadId := "rand" - rawWithId := &aclrecordproto.RawAclRecordWithId{ - Payload: marshalledRaw, - Id: aclHeadId, - } - _, err = validateCreateSpaceAclPayload(rawWithId) - assert.EqualErrorf(t, err, objecttree.ErrIncorrectCid.Error(), "Error should be: %v, got: %v", objecttree.ErrIncorrectCid, err) -} - -func TestFailedAclPayloadSpace_IncorrectSignature(t *testing.T) { - accountKeys, err := accountdata.NewRandom() - require.NoError(t, err) - readKeyBytes := make([]byte, 32) - _, err = rand.Read(readKeyBytes) - require.NoError(t, err) - readKey, err := accountKeys.SignKey.GetPublic().Encrypt(readKeyBytes) - require.NoError(t, err) - masterKey, _, err := crypto.GenerateRandomEd25519KeyPair() - require.NoError(t, err) - rawIdentity, err := accountKeys.SignKey.GetPublic().Raw() - require.NoError(t, err) - identity, err := accountKeys.SignKey.GetPublic().Marshall() - identitySignature, err := masterKey.Sign(rawIdentity) - require.NoError(t, err) - rawMasterKey, err := masterKey.GetPublic().Raw() - require.NoError(t, err) - aclRoot := aclrecordproto.AclRoot{ - Identity: identity, - MasterKey: rawMasterKey, - SpaceId: "SpaceId", - EncryptedReadKey: readKey, - Timestamp: time.Now().Unix(), - IdentitySignature: identitySignature, - } - marshalled, err := aclRoot.Marshal() - require.NoError(t, err) - rawAclRecord := &aclrecordproto.RawAclRecord{ - Payload: marshalled, - Signature: marshalled, - } - marshalledRaw, err := rawAclRecord.Marshal() - require.NoError(t, err) - aclHeadId, err := cidutil.NewCidFromBytes(marshalledRaw) - require.NoError(t, err) - rawWithId := &aclrecordproto.RawAclRecordWithId{ - Payload: marshalledRaw, - Id: aclHeadId, - } - _, err = validateCreateSpaceAclPayload(rawWithId) - assert.NotNil(t, err) - assert.EqualErrorf(t, err, ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", ErrIncorrectSpaceHeader, err) -} - -func TestFailedAclPayloadSpace_IncorrectIdentitySignature(t *testing.T) { - spaceId := "AnySpaceId" - accountKeys, err := accountdata.NewRandom() - require.NoError(t, err) - readKeyBytes := make([]byte, 32) - _, err = rand.Read(readKeyBytes) - if err != nil { - return - } - readKey, err := accountKeys.SignKey.GetPublic().Encrypt(readKeyBytes) - if err != nil { - return - } - masterKey, _, err := crypto.GenerateRandomEd25519KeyPair() - if err != nil { - return - } - masterPubKey := masterKey.GetPublic() - identity, err := accountKeys.SignKey.GetPublic().Marshall() - if err != nil { - return - } - rawMasterKey, err := masterPubKey.Marshall() - if err != nil { - return - } - aclRoot := aclrecordproto.AclRoot{ - Identity: identity, - MasterKey: rawMasterKey, - SpaceId: spaceId, - EncryptedReadKey: readKey, - Timestamp: time.Now().Unix(), - IdentitySignature: identity, - } - marshalled, err := aclRoot.Marshal() - if err != nil { - return - } - signature, err := accountKeys.SignKey.Sign(marshalled) - rawAclRecord := &aclrecordproto.RawAclRecord{ - Payload: marshalled, - Signature: signature, - } - marshalledRaw, err := rawAclRecord.Marshal() - if err != nil { - return - } - aclHeadId, err := cidutil.NewCidFromBytes(marshalledRaw) - if err != nil { - return - } - rawWithId := &aclrecordproto.RawAclRecordWithId{ - Payload: marshalledRaw, - Id: aclHeadId, - } - _, err = validateCreateSpaceAclPayload(rawWithId) - assert.EqualErrorf(t, err, ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", ErrIncorrectSpaceHeader, err) -} - -func TestSuccessSettingsPayloadSpace(t *testing.T) { - accountKeys, err := accountdata.NewRandom() - require.NoError(t, err) - identity, err := accountKeys.SignKey.GetPublic().Marshall() - require.NoError(t, err) - spaceSettingsSeed := make([]byte, 32) - _, err = rand.Read(spaceSettingsSeed) - require.NoError(t, err) - changePayload := make([]byte, 32) - _, err = rand.Read(changePayload) - require.NoError(t, err) - spaceId := "SpaceId" - rootChange := &treechangeproto.RootChange{ - AclHeadId: "AclHeadId", - SpaceId: spaceId, - ChangeType: "ChangeType", - Timestamp: time.Now().Unix(), - Seed: spaceSettingsSeed, - Identity: identity, - ChangePayload: changePayload, - } - marshalledChange, err := rootChange.Marshal() - require.NoError(t, err) - signature, err := accountKeys.SignKey.Sign(marshalledChange) - require.NoError(t, err) - raw := &treechangeproto.RawTreeChange{ - Payload: marshalledChange, - Signature: signature, - } - marshalledRawChange, err := raw.Marshal() - id, err := cidutil.NewCidFromBytes(marshalledRawChange) - require.NoError(t, err) - rawIdChange := &treechangeproto.RawTreeChangeWithId{ - RawChange: marshalledRawChange, - Id: id, - } - _, validationSpaceId, err := validateCreateSpaceSettingsPayload(rawIdChange) - require.Equal(t, validationSpaceId, spaceId) - require.NoError(t, err) -} - -func TestFailSettingsPayloadSpace_InvalidSignature(t *testing.T) { - accountKeys, err := accountdata.NewRandom() - require.NoError(t, err) - identity, err := accountKeys.SignKey.GetPublic().Marshall() - require.NoError(t, err) - spaceSettingsSeed := make([]byte, 32) - _, err = rand.Read(spaceSettingsSeed) - require.NoError(t, err) - changePayload := make([]byte, 32) - _, err = rand.Read(changePayload) - require.NoError(t, err) - rootChange := &treechangeproto.RootChange{ - AclHeadId: "AclHeadId", - SpaceId: "SpaceId", - ChangeType: "ChangeType", - Timestamp: time.Now().Unix(), - Seed: spaceSettingsSeed, - Identity: identity, - ChangePayload: changePayload, - } - marshalledChange, err := rootChange.Marshal() - require.NoError(t, err) - raw := &treechangeproto.RawTreeChange{ - Payload: marshalledChange, - Signature: marshalledChange, - } - marshalledRawChange, err := raw.Marshal() - id, err := cidutil.NewCidFromBytes(marshalledRawChange) - require.NoError(t, err) - rawIdChange := &treechangeproto.RawTreeChangeWithId{ - RawChange: marshalledRawChange, - Id: id, - } - _, _, err = validateCreateSpaceSettingsPayload(rawIdChange) - assert.EqualErrorf(t, err, ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", ErrIncorrectSpaceHeader, err) -} - -func TestFailSettingsPayloadSpace_InvalidCid(t *testing.T) { - accountKeys, err := accountdata.NewRandom() - require.NoError(t, err) - identity, err := accountKeys.SignKey.GetPublic().Marshall() - require.NoError(t, err) - spaceSettingsSeed := make([]byte, 32) - _, err = rand.Read(spaceSettingsSeed) - require.NoError(t, err) - changePayload := make([]byte, 32) - _, err = rand.Read(changePayload) - require.NoError(t, err) - rootChange := &treechangeproto.RootChange{ - AclHeadId: "AclHeadId", - SpaceId: "SpaceId", - ChangeType: "ChangeType", - Timestamp: time.Now().Unix(), - Seed: spaceSettingsSeed, - Identity: identity, - ChangePayload: changePayload, - } - marshalledChange, err := rootChange.Marshal() - require.NoError(t, err) - signature, err := accountKeys.SignKey.Sign(marshalledChange) - require.NoError(t, err) - raw := &treechangeproto.RawTreeChange{ - Payload: marshalledChange, - Signature: signature, - } - marshalledRawChange, err := raw.Marshal() - id := "id" - require.NoError(t, err) - rawIdChange := &treechangeproto.RawTreeChangeWithId{ - RawChange: marshalledRawChange, - Id: id, - } - _, _, err = validateCreateSpaceSettingsPayload(rawIdChange) - assert.EqualErrorf(t, err, ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", ErrIncorrectSpaceHeader, err) -} - -func TestSuccessSameIds(t *testing.T) { - accountKeys, err := accountdata.NewRandom() - require.NoError(t, err) - spaceId, rawHeaderWithId, err := rawHeaderWithId(accountKeys) - require.NoError(t, err) - aclHeadId, rawAclWithId, err := rawAclWithId(accountKeys, spaceId) - require.NoError(t, err) - rawSettingsPayload, err := rawSettingsPayload(accountKeys, spaceId, aclHeadId) - spacePayload := SpaceStorageCreatePayload{ - AclWithId: rawAclWithId, - SpaceHeaderWithId: rawHeaderWithId, - SpaceSettingsWithId: rawSettingsPayload, - } - err = ValidateSpaceStorageCreatePayload(spacePayload) - require.NoError(t, err) -} - -func TestFailWithAclWrongSpaceId(t *testing.T) { - accountKeys, err := accountdata.NewRandom() - require.NoError(t, err) - spaceId, rawHeaderWithId, err := rawHeaderWithId(accountKeys) - require.NoError(t, err) - aclHeadId, rawAclWithId, err := rawAclWithId(accountKeys, "spaceId") - require.NoError(t, err) - rawSettingsPayload, err := rawSettingsPayload(accountKeys, spaceId, aclHeadId) - spacePayload := SpaceStorageCreatePayload{ - AclWithId: rawAclWithId, - SpaceHeaderWithId: rawHeaderWithId, - SpaceSettingsWithId: rawSettingsPayload, - } - err = ValidateSpaceStorageCreatePayload(spacePayload) - assert.EqualErrorf(t, err, ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", ErrIncorrectSpaceHeader, err) -} - -func TestFailWithSettingsWrongSpaceId(t *testing.T) { - accountKeys, err := accountdata.NewRandom() - require.NoError(t, err) - spaceId, rawHeaderWithId, err := rawHeaderWithId(accountKeys) - require.NoError(t, err) - aclHeadId, rawAclWithId, err := rawAclWithId(accountKeys, spaceId) - require.NoError(t, err) - rawSettingsPayload, err := rawSettingsPayload(accountKeys, "spaceId", aclHeadId) - spacePayload := SpaceStorageCreatePayload{ - AclWithId: rawAclWithId, - SpaceHeaderWithId: rawHeaderWithId, - SpaceSettingsWithId: rawSettingsPayload, - } - err = ValidateSpaceStorageCreatePayload(spacePayload) - assert.EqualErrorf(t, err, ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", ErrIncorrectSpaceHeader, err) -} - -func TestFailWithWrongAclHeadIdInSettingsPayload(t *testing.T) { - accountKeys, err := accountdata.NewRandom() - require.NoError(t, err) - spaceId, rawHeaderWithId, err := rawHeaderWithId(accountKeys) - require.NoError(t, err) - _, rawAclWithId, err := rawAclWithId(accountKeys, spaceId) - require.NoError(t, err) - rawSettingsPayload, err := rawSettingsPayload(accountKeys, spaceId, "aclHeadId") - spacePayload := SpaceStorageCreatePayload{ - AclWithId: rawAclWithId, - SpaceHeaderWithId: rawHeaderWithId, - SpaceSettingsWithId: rawSettingsPayload, - } - err = ValidateSpaceStorageCreatePayload(spacePayload) - assert.EqualErrorf(t, err, ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", ErrIncorrectSpaceHeader, err) -} - -func rawSettingsPayload(accountKeys *accountdata.AccountKeys, spaceId, aclHeadId string) (rawIdChange *treechangeproto.RawTreeChangeWithId, err error) { - identity, err := accountKeys.SignKey.GetPublic().Marshall() - if err != nil { - return - } - spaceSettingsSeed := make([]byte, 32) - _, err = rand.Read(spaceSettingsSeed) - if err != nil { - return - } - changePayload := make([]byte, 32) - _, err = rand.Read(changePayload) - if err != nil { - return - } - rootChange := &treechangeproto.RootChange{ - AclHeadId: aclHeadId, - SpaceId: spaceId, - ChangeType: "ChangeType", - Timestamp: time.Now().Unix(), - Seed: spaceSettingsSeed, - Identity: identity, - ChangePayload: changePayload, - } - marshalledChange, err := rootChange.Marshal() - if err != nil { - return - } - signature, err := accountKeys.SignKey.Sign(marshalledChange) - if err != nil { - return - } - raw := &treechangeproto.RawTreeChange{ - Payload: marshalledChange, - Signature: signature, - } - marshalledRawChange, err := raw.Marshal() - id, err := cidutil.NewCidFromBytes(marshalledRawChange) - if err != nil { - return - } - rawIdChange = &treechangeproto.RawTreeChangeWithId{ - RawChange: marshalledRawChange, - Id: id, - } - - return -} - -func rawAclWithId(accountKeys *accountdata.AccountKeys, spaceId string) (aclHeadId string, rawWithId *aclrecordproto.RawAclRecordWithId, err error) { - readKeyBytes := make([]byte, 32) - _, err = rand.Read(readKeyBytes) - if err != nil { - return - } - readKey, err := accountKeys.SignKey.GetPublic().Encrypt(readKeyBytes) - if err != nil { - return - } - masterKey, _, err := crypto.GenerateRandomEd25519KeyPair() - identity, err := accountKeys.SignKey.GetPublic().Marshall() - if err != nil { - return - } - masterPubKey := masterKey.GetPublic() - rawIdentity, err := accountKeys.SignKey.GetPublic().Raw() - if err != nil { - return - } - identitySignature, err := masterKey.Sign(rawIdentity) - if err != nil { - return - } - rawMasterKey, err := masterPubKey.Marshall() - if err != nil { - return - } - aclRoot := aclrecordproto.AclRoot{ - Identity: identity, - MasterKey: rawMasterKey, - SpaceId: spaceId, - EncryptedReadKey: readKey, - Timestamp: time.Now().Unix(), - IdentitySignature: identitySignature, - } - marshalled, err := aclRoot.Marshal() - if err != nil { - return - } - signature, err := accountKeys.SignKey.Sign(marshalled) - rawAclRecord := &aclrecordproto.RawAclRecord{ - Payload: marshalled, - Signature: signature, - } - marshalledRaw, err := rawAclRecord.Marshal() - if err != nil { - return - } - aclHeadId, err = cidutil.NewCidFromBytes(marshalledRaw) - if err != nil { - return - } - rawWithId = &aclrecordproto.RawAclRecordWithId{ - Payload: marshalledRaw, - Id: aclHeadId, - } - - return -} - -func rawHeaderWithId(accountKeys *accountdata.AccountKeys) (spaceId string, rawWithId *spacesyncproto.RawSpaceHeaderWithId, err error) { - identity, err := accountKeys.SignKey.GetPublic().Marshall() - if err != nil { - return - } - spaceHeaderSeed := make([]byte, 32) - _, err = rand.Read(spaceHeaderSeed) - if err != nil { - return - } - spaceHeaderPayload := make([]byte, 32) - _, err = rand.Read(spaceHeaderPayload) - if err != nil { - return - } - replicationKey := rand2.Uint64() - header := &spacesyncproto.SpaceHeader{ - Identity: identity, - Timestamp: time.Now().Unix(), - SpaceType: "SpaceType", - ReplicationKey: replicationKey, - Seed: spaceHeaderSeed, - SpaceHeaderPayload: spaceHeaderPayload, - } - marhalled, err := header.Marshal() - if err != nil { - return - } - signature, err := accountKeys.SignKey.Sign(marhalled) - if err != nil { - return - } - rawHeader := &spacesyncproto.RawSpaceHeader{ - SpaceHeader: marhalled, - Signature: signature, - } - marhalledRawHeader, err := rawHeader.Marshal() - if err != nil { - return - } - id, err := cidutil.NewCidFromBytes(marhalledRawHeader) - if err != nil { - return - } - spaceId = fmt.Sprintf("%s.%s", id, strconv.FormatUint(replicationKey, 36)) - rawWithId = &spacesyncproto.RawSpaceHeaderWithId{ - RawHeader: marhalledRawHeader, - Id: spaceId, - } - - return -} From e684088ca44493fe9ca994306d4481faf9520d00 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 21 Apr 2023 23:21:49 +0200 Subject: [PATCH 31/45] Improve payload validation --- commonspace/payloads.go | 82 ++++++++++-------------------------- commonspace/payloads_test.go | 14 +++--- commonspace/space.go | 4 +- 3 files changed, 32 insertions(+), 68 deletions(-) diff --git a/commonspace/payloads.go b/commonspace/payloads.go index 4d601155..372d3ebd 100644 --- a/commonspace/payloads.go +++ b/commonspace/payloads.go @@ -1,7 +1,6 @@ package commonspace import ( - "fmt" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" @@ -188,7 +187,7 @@ func storagePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload sp } func validateSpaceStorageCreatePayload(payload spacestorage.SpaceStorageCreatePayload) (err error) { - err = validateCreateSpaceHeaderPayload(payload.SpaceHeaderWithId) + err = ValidateSpaceHeader(payload.SpaceHeaderWithId, nil) if err != nil { return } @@ -211,7 +210,16 @@ func validateSpaceStorageCreatePayload(payload spacestorage.SpaceStorageCreatePa return } -func validateCreateSpaceHeaderPayload(rawHeaderWithId *spacesyncproto.RawSpaceHeaderWithId) (err error) { +func ValidateSpaceHeader(rawHeaderWithId *spacesyncproto.RawSpaceHeaderWithId, identity crypto.PubKey) (err error) { + sepIdx := strings.Index(rawHeaderWithId.Id, ".") + if sepIdx == -1 { + err = objecttree.ErrIncorrectCid + return + } + if !cidutil.VerifyCid(rawHeaderWithId.RawHeader, rawHeaderWithId.Id[:sepIdx]) { + err = objecttree.ErrIncorrectCid + return + } var rawSpaceHeader spacesyncproto.RawSpaceHeader err = proto.Unmarshal(rawHeaderWithId.RawHeader, &rawSpaceHeader) if err != nil { @@ -222,14 +230,6 @@ func validateCreateSpaceHeaderPayload(rawHeaderWithId *spacesyncproto.RawSpaceHe if err != nil { return } - split := strings.Split(rawHeaderWithId.Id, ".") - if len(split) != 2 { - return spacestorage.ErrIncorrectSpaceHeader - } - if !cidutil.VerifyCid(rawHeaderWithId.RawHeader, split[0]) { - err = objecttree.ErrIncorrectCid - return - } payloadIdentity, err := crypto.UnmarshalEd25519PublicKeyProto(header.Identity) if err != nil { return @@ -239,16 +239,17 @@ func validateCreateSpaceHeaderPayload(rawHeaderWithId *spacesyncproto.RawSpaceHe err = spacestorage.ErrIncorrectSpaceHeader return } - id, err := cidutil.NewCidFromBytes(rawHeaderWithId.RawHeader) - if err != nil { - return - } - requiredSpaceId := fmt.Sprintf("%s.%s", id, strconv.FormatUint(header.ReplicationKey, 36)) - if requiredSpaceId != rawHeaderWithId.Id { + if rawHeaderWithId.Id[sepIdx+1:] != strconv.FormatUint(header.ReplicationKey, 36) { + err = spacestorage.ErrIncorrectSpaceHeader + return + } + if identity == nil { + return + } + if !payloadIdentity.Equals(identity) { err = spacestorage.ErrIncorrectSpaceHeader return } - return } @@ -295,6 +296,10 @@ func validateCreateSpaceAclPayload(rawWithId *aclrecordproto.RawAclRecordWithId) } func validateCreateSpaceSettingsPayload(rawWithId *treechangeproto.RawTreeChangeWithId) (aclHeadId string, spaceId string, err error) { + if !cidutil.VerifyCid(rawWithId.RawChange, rawWithId.Id) { + err = spacestorage.ErrIncorrectSpaceHeader + return + } var raw treechangeproto.RawTreeChange err = proto.Unmarshal(rawWithId.RawChange, &raw) if err != nil { @@ -314,49 +319,8 @@ func validateCreateSpaceSettingsPayload(rawWithId *treechangeproto.RawTreeChange err = spacestorage.ErrIncorrectSpaceHeader return } - id, err := cidutil.NewCidFromBytes(rawWithId.RawChange) - if id != rawWithId.Id { - err = spacestorage.ErrIncorrectSpaceHeader - return - } spaceId = rootChange.SpaceId aclHeadId = rootChange.AclHeadId return } - -// ValidateSpaceHeader Used in coordinator -func ValidateSpaceHeader(spaceId string, header []byte, identity crypto.PubKey) (err error) { - split := strings.Split(spaceId, ".") - if len(split) != 2 { - return spacestorage.ErrIncorrectSpaceHeader - } - if !cidutil.VerifyCid(header, split[0]) { - err = objecttree.ErrIncorrectCid - return - } - raw := &spacesyncproto.RawSpaceHeader{} - err = proto.Unmarshal(header, raw) - if err != nil { - return - } - payload := &spacesyncproto.SpaceHeader{} - err = proto.Unmarshal(raw.SpaceHeader, payload) - if err != nil { - return - } - payloadIdentity, err := crypto.UnmarshalEd25519PublicKeyProto(payload.Identity) - if err != nil { - return - } - if identity != nil && !payloadIdentity.Equals(identity) { - err = spacestorage.ErrIncorrectSpaceHeader - return - } - res, err := identity.Verify(raw.SpaceHeader, raw.Signature) - if err != nil || !res { - err = spacestorage.ErrIncorrectSpaceHeader - return - } - return -} diff --git a/commonspace/payloads_test.go b/commonspace/payloads_test.go index 13535b34..65bf917d 100644 --- a/commonspace/payloads_test.go +++ b/commonspace/payloads_test.go @@ -23,7 +23,7 @@ func TestSuccessHeaderPayloadForSpaceCreate(t *testing.T) { require.NoError(t, err) _, rawHeaderWithId, err := rawHeaderWithId(accountKeys) require.NoError(t, err) - err = validateCreateSpaceHeaderPayload(rawHeaderWithId) + err = ValidateSpaceHeader(rawHeaderWithId, nil) require.NoError(t, err) } @@ -64,7 +64,7 @@ func TestFailedHeaderPayloadForSpaceCreate_InvalidFormatSpaceId(t *testing.T) { RawHeader: marhalledRawHeader, Id: spaceId, } - err = validateCreateSpaceHeaderPayload(rawHeaderWithId) + err = ValidateSpaceHeader(rawHeaderWithId, nil) assert.EqualErrorf(t, err, spacestorage.ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", spacestorage.ErrIncorrectSpaceHeader, err) } @@ -104,7 +104,7 @@ func TestFailedHeaderPayloadForSpaceCreate_CidIsWrong(t *testing.T) { RawHeader: marhalledRawHeader, Id: spaceId, } - err = validateCreateSpaceHeaderPayload(rawHeaderWithId) + err = ValidateSpaceHeader(rawHeaderWithId, nil) assert.EqualErrorf(t, err, objecttree.ErrIncorrectCid.Error(), "Error should be: %v, got: %v", objecttree.ErrIncorrectCid, err) } @@ -145,7 +145,7 @@ func TestFailedHeaderPayloadForSpaceCreate_SignedWithAnotherIdentity(t *testing. RawHeader: marhalledRawHeader, Id: spaceId, } - err = validateCreateSpaceHeaderPayload(rawHeaderWithId) + err = ValidateSpaceHeader(rawHeaderWithId, nil) assert.EqualErrorf(t, err, objecttree.ErrIncorrectCid.Error(), "Error should be: %v, got: %v", objecttree.ErrIncorrectCid, err) } @@ -637,17 +637,17 @@ func rawHeaderWithId(accountKeys *accountdata.AccountKeys) (spaceId string, rawW SpaceHeader: marhalled, Signature: signature, } - marhalledRawHeader, err := rawHeader.Marshal() + marshalledRawHeader, err := rawHeader.Marshal() if err != nil { return } - id, err := cidutil.NewCidFromBytes(marhalledRawHeader) + id, err := cidutil.NewCidFromBytes(marshalledRawHeader) if err != nil { return } spaceId = fmt.Sprintf("%s.%s", id, strconv.FormatUint(replicationKey, 36)) rawWithId = &spacesyncproto.RawSpaceHeaderWithId{ - RawHeader: marhalledRawHeader, + RawHeader: marshalledRawHeader, Id: spaceId, } diff --git a/commonspace/space.go b/commonspace/space.go index 10090b95..2bcbb4da 100644 --- a/commonspace/space.go +++ b/commonspace/space.go @@ -3,7 +3,6 @@ package commonspace import ( "context" "errors" - "fmt" "github.com/anytypeio/any-sync/accountservice" "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/commonspace/headsync" @@ -30,6 +29,7 @@ import ( "github.com/zeebo/errs" "go.uber.org/zap" "strconv" + "strings" "sync" "sync/atomic" "time" @@ -77,7 +77,7 @@ type SpaceDescription struct { } func NewSpaceId(id string, repKey uint64) string { - return fmt.Sprintf("%s.%s", id, strconv.FormatUint(repKey, 36)) + return strings.Join([]string{id, strconv.FormatUint(repKey, 36)}, ".") } type Space interface { From 63c797d35bf1f7b5a817156ae1289617d045f941 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 21 Apr 2023 23:23:07 +0200 Subject: [PATCH 32/45] Add todos for test helpers --- commonspace/payloads_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/commonspace/payloads_test.go b/commonspace/payloads_test.go index 65bf917d..899b56dd 100644 --- a/commonspace/payloads_test.go +++ b/commonspace/payloads_test.go @@ -541,6 +541,7 @@ func rawSettingsPayload(accountKeys *accountdata.AccountKeys, spaceId, aclHeadId } func rawAclWithId(accountKeys *accountdata.AccountKeys, spaceId string) (aclHeadId string, rawWithId *aclrecordproto.RawAclRecordWithId, err error) { + // TODO: use same storage creation methods as we use in spaces readKeyBytes := make([]byte, 32) _, err = rand.Read(readKeyBytes) if err != nil { @@ -602,6 +603,7 @@ func rawAclWithId(accountKeys *accountdata.AccountKeys, spaceId string) (aclHead } func rawHeaderWithId(accountKeys *accountdata.AccountKeys) (spaceId string, rawWithId *spacesyncproto.RawSpaceHeaderWithId, err error) { + // TODO: use same storage creation methods as we use in spaces identity, err := accountKeys.SignKey.GetPublic().Marshall() if err != nil { return From d85e318553158b0b0e5153535aef4859ddf29cc6 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 21 Apr 2023 23:25:54 +0200 Subject: [PATCH 33/45] Fix test --- commonspace/payloads.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commonspace/payloads.go b/commonspace/payloads.go index 372d3ebd..b54ec22b 100644 --- a/commonspace/payloads.go +++ b/commonspace/payloads.go @@ -213,7 +213,7 @@ func validateSpaceStorageCreatePayload(payload spacestorage.SpaceStorageCreatePa func ValidateSpaceHeader(rawHeaderWithId *spacesyncproto.RawSpaceHeaderWithId, identity crypto.PubKey) (err error) { sepIdx := strings.Index(rawHeaderWithId.Id, ".") if sepIdx == -1 { - err = objecttree.ErrIncorrectCid + err = spacestorage.ErrIncorrectSpaceHeader return } if !cidutil.VerifyCid(rawHeaderWithId.RawHeader, rawHeaderWithId.Id[:sepIdx]) { From 6eb08ca4385516de66bf51fe8932a45ffcf5d774 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 21 Apr 2023 23:29:29 +0200 Subject: [PATCH 34/45] Change identity error --- commonspace/payloads.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/commonspace/payloads.go b/commonspace/payloads.go index b54ec22b..ebf6811b 100644 --- a/commonspace/payloads.go +++ b/commonspace/payloads.go @@ -1,6 +1,7 @@ package commonspace import ( + "errors" "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" @@ -21,6 +22,8 @@ const ( SpaceReserved = "any-sync.space" ) +var ErrIncorrectIdentity = errors.New("incorrect identity") + func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload spacestorage.SpaceStorageCreatePayload, err error) { // marshalling keys identity, err := payload.SigningKey.GetPublic().Marshall() @@ -247,7 +250,7 @@ func ValidateSpaceHeader(rawHeaderWithId *spacesyncproto.RawSpaceHeaderWithId, i return } if !payloadIdentity.Equals(identity) { - err = spacestorage.ErrIncorrectSpaceHeader + err = ErrIncorrectIdentity return } return From 9216896777d197290507908a7769e889e8584323 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 24 Apr 2023 14:12:24 +0200 Subject: [PATCH 35/45] Add errors to proto --- commonspace/spacesyncproto/errors.go | 12 +- .../spacesyncproto/protos/spacesync.proto | 2 + commonspace/spacesyncproto/spacesync.pb.go | 166 +++++++++--------- 3 files changed, 96 insertions(+), 84 deletions(-) diff --git a/commonspace/spacesyncproto/errors.go b/commonspace/spacesyncproto/errors.go index 56ac0446..0760c70b 100644 --- a/commonspace/spacesyncproto/errors.go +++ b/commonspace/spacesyncproto/errors.go @@ -8,9 +8,11 @@ import ( var ( errGroup = rpcerr.ErrGroup(ErrCodes_ErrorOffset) - ErrUnexpected = errGroup.Register(errors.New("unexpected error"), uint64(ErrCodes_Unexpected)) - ErrSpaceMissing = errGroup.Register(errors.New("space is missing"), uint64(ErrCodes_SpaceMissing)) - ErrSpaceExists = errGroup.Register(errors.New("space exists"), uint64(ErrCodes_SpaceExists)) - ErrSpaceNotInCache = errGroup.Register(errors.New("space not in cache"), uint64(ErrCodes_SpaceNotInCache)) - ErrSpaceIsDeleted = errGroup.Register(errors.New("space is deleted"), uint64(ErrCodes_SpaceIsDeleted)) + ErrUnexpected = errGroup.Register(errors.New("unexpected error"), uint64(ErrCodes_Unexpected)) + ErrSpaceMissing = errGroup.Register(errors.New("space is missing"), uint64(ErrCodes_SpaceMissing)) + ErrSpaceExists = errGroup.Register(errors.New("space exists"), uint64(ErrCodes_SpaceExists)) + ErrSpaceNotInCache = errGroup.Register(errors.New("space not in cache"), uint64(ErrCodes_SpaceNotInCache)) + ErrSpaceIsDeleted = errGroup.Register(errors.New("space is deleted"), uint64(ErrCodes_SpaceIsDeleted)) + ErrPeerIsNotResponsible = errGroup.Register(errors.New("peer is not responsible for space"), uint64(ErrCodes_PeerIsNotResponsible)) + ErrReceiptInvalid = errGroup.Register(errors.New("space receipt is not valid"), uint64(ErrCodes_ReceiptIsInvalid)) ) diff --git a/commonspace/spacesyncproto/protos/spacesync.proto b/commonspace/spacesyncproto/protos/spacesync.proto index 3bf03320..d5b461cf 100644 --- a/commonspace/spacesyncproto/protos/spacesync.proto +++ b/commonspace/spacesyncproto/protos/spacesync.proto @@ -9,6 +9,8 @@ enum ErrCodes { SpaceExists = 2; SpaceNotInCache = 3; SpaceIsDeleted = 4; + PeerIsNotResponsible = 5; + ReceiptIsInvalid = 6; ErrorOffset = 100; } diff --git a/commonspace/spacesyncproto/spacesync.pb.go b/commonspace/spacesyncproto/spacesync.pb.go index bb5b0e6e..33caed5b 100644 --- a/commonspace/spacesyncproto/spacesync.pb.go +++ b/commonspace/spacesyncproto/spacesync.pb.go @@ -25,12 +25,14 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type ErrCodes int32 const ( - ErrCodes_Unexpected ErrCodes = 0 - ErrCodes_SpaceMissing ErrCodes = 1 - ErrCodes_SpaceExists ErrCodes = 2 - ErrCodes_SpaceNotInCache ErrCodes = 3 - ErrCodes_SpaceIsDeleted ErrCodes = 4 - ErrCodes_ErrorOffset ErrCodes = 100 + ErrCodes_Unexpected ErrCodes = 0 + ErrCodes_SpaceMissing ErrCodes = 1 + ErrCodes_SpaceExists ErrCodes = 2 + ErrCodes_SpaceNotInCache ErrCodes = 3 + ErrCodes_SpaceIsDeleted ErrCodes = 4 + ErrCodes_PeerIsNotResponsible ErrCodes = 5 + ErrCodes_ReceiptIsInvalid ErrCodes = 6 + ErrCodes_ErrorOffset ErrCodes = 100 ) var ErrCodes_name = map[int32]string{ @@ -39,16 +41,20 @@ var ErrCodes_name = map[int32]string{ 2: "SpaceExists", 3: "SpaceNotInCache", 4: "SpaceIsDeleted", + 5: "PeerIsNotResponsible", + 6: "ReceiptIsInvalid", 100: "ErrorOffset", } var ErrCodes_value = map[string]int32{ - "Unexpected": 0, - "SpaceMissing": 1, - "SpaceExists": 2, - "SpaceNotInCache": 3, - "SpaceIsDeleted": 4, - "ErrorOffset": 100, + "Unexpected": 0, + "SpaceMissing": 1, + "SpaceExists": 2, + "SpaceNotInCache": 3, + "SpaceIsDeleted": 4, + "PeerIsNotResponsible": 5, + "ReceiptIsInvalid": 6, + "ErrorOffset": 100, } func (x ErrCodes) String() string { @@ -1248,73 +1254,75 @@ func init() { } var fileDescriptor_80e49f1f4ac27799 = []byte{ - // 1042 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x56, 0x4f, 0x6f, 0x1b, 0x45, - 0x14, 0xf7, 0x6e, 0xd2, 0x24, 0x7e, 0x71, 0x9c, 0xed, 0x34, 0x6d, 0x8d, 0x1b, 0xb9, 0xd6, 0x08, - 0xa1, 0xa8, 0x87, 0xb4, 0x75, 0x10, 0x52, 0x0b, 0x1c, 0x5a, 0x27, 0xa5, 0x16, 0x2a, 0x89, 0xc6, - 0x54, 0x48, 0x48, 0x3d, 0x4c, 0x76, 0x5f, 0xec, 0x85, 0xf5, 0xee, 0xb2, 0x33, 0x26, 0xf1, 0x91, - 0x13, 0x57, 0xce, 0xf0, 0x35, 0xf8, 0x10, 0x1c, 0xcb, 0x8d, 0x23, 0x4a, 0xbe, 0x08, 0x9a, 0xd9, - 0xd9, 0x3f, 0xb6, 0xd7, 0x95, 0xb8, 0x38, 0x3b, 0xbf, 0xf7, 0xde, 0xef, 0xfd, 0x9b, 0x79, 0x2f, - 0xf0, 0xd4, 0x8d, 0x26, 0x93, 0x28, 0x14, 0x31, 0x77, 0xf1, 0xb1, 0xfe, 0x15, 0xb3, 0xd0, 0x8d, - 0x93, 0x48, 0x46, 0x8f, 0xf5, 0xaf, 0x28, 0xd0, 0x43, 0x0d, 0x90, 0x7a, 0x0e, 0xd0, 0x01, 0xec, - 0xbc, 0x46, 0xee, 0x0d, 0x67, 0xa1, 0xcb, 0x78, 0x38, 0x42, 0x42, 0x60, 0xfd, 0x22, 0x89, 0x26, - 0x2d, 0xab, 0x6b, 0x1d, 0xac, 0x33, 0xfd, 0x4d, 0x9a, 0x60, 0xcb, 0xa8, 0x65, 0x6b, 0xc4, 0x96, - 0x11, 0xd9, 0x83, 0x5b, 0x81, 0x3f, 0xf1, 0x65, 0x6b, 0xad, 0x6b, 0x1d, 0xec, 0xb0, 0xf4, 0x40, - 0xaf, 0xa0, 0x99, 0x53, 0xa1, 0x98, 0x06, 0x52, 0x71, 0x8d, 0xb9, 0x18, 0x6b, 0xae, 0x06, 0xd3, - 0xdf, 0xe4, 0x0b, 0xd8, 0xc2, 0x00, 0x27, 0x18, 0x4a, 0xd1, 0xb2, 0xbb, 0x6b, 0x07, 0xdb, 0xbd, - 0xee, 0x61, 0x11, 0xdf, 0x3c, 0xc1, 0x49, 0xaa, 0xc8, 0x72, 0x0b, 0xe5, 0xd9, 0x8d, 0xa6, 0x61, - 0xee, 0x59, 0x1f, 0xe8, 0xe7, 0x70, 0xb7, 0xd2, 0x50, 0x05, 0xee, 0x7b, 0xda, 0x7d, 0x9d, 0xd9, - 0xbe, 0xa7, 0x03, 0x42, 0xee, 0xe9, 0x54, 0xea, 0x4c, 0x7f, 0xd3, 0x77, 0xb0, 0x5b, 0x18, 0xff, - 0x34, 0x45, 0x21, 0x49, 0x0b, 0x36, 0x75, 0x48, 0x83, 0xcc, 0x36, 0x3b, 0x92, 0x27, 0xb0, 0x91, - 0xa8, 0x32, 0x65, 0xb1, 0xb7, 0xaa, 0x62, 0x57, 0x0a, 0xcc, 0xe8, 0xd1, 0xaf, 0xc0, 0x29, 0xc5, - 0x16, 0x47, 0xa1, 0x40, 0x72, 0x04, 0x9b, 0x89, 0x8e, 0x53, 0xb4, 0x2c, 0x4d, 0xf3, 0xd1, 0xca, - 0x12, 0xb0, 0x4c, 0x93, 0xfe, 0x61, 0xc1, 0xed, 0xd3, 0xf3, 0x1f, 0xd0, 0x95, 0x4a, 0xfa, 0x06, - 0x85, 0xe0, 0x23, 0xfc, 0x40, 0xa8, 0xfb, 0x50, 0x4f, 0xd2, 0x7c, 0x06, 0x59, 0xc2, 0x05, 0xa0, - 0xec, 0x12, 0x8c, 0x83, 0xd9, 0xc0, 0xd3, 0xa5, 0xac, 0xb3, 0xec, 0xa8, 0x24, 0x31, 0x9f, 0x05, - 0x11, 0xf7, 0x5a, 0xeb, 0xba, 0x6f, 0xd9, 0x91, 0xb4, 0x61, 0x2b, 0xd2, 0x01, 0x0c, 0xbc, 0xd6, - 0x2d, 0x6d, 0x94, 0x9f, 0x29, 0x82, 0x33, 0x54, 0x8e, 0xcf, 0xa6, 0x62, 0x9c, 0x95, 0xf1, 0x69, - 0xc1, 0xa4, 0x62, 0xdb, 0xee, 0xdd, 0x2f, 0xa5, 0x99, 0x6a, 0xa7, 0xe2, 0xc2, 0x45, 0x07, 0xa0, - 0x9f, 0xa0, 0x87, 0xa1, 0xf4, 0x79, 0xa0, 0xa3, 0x6e, 0xb0, 0x12, 0x42, 0xef, 0xc0, 0xed, 0x92, - 0x9b, 0xb4, 0x9c, 0x94, 0xe6, 0xbe, 0x83, 0x20, 0xf3, 0xbd, 0xd0, 0x79, 0xfa, 0x2a, 0x37, 0x54, - 0x3a, 0xa6, 0x0f, 0xff, 0x3f, 0x40, 0xfa, 0x8b, 0x0d, 0x8d, 0xb2, 0x84, 0xbc, 0x80, 0x6d, 0x6d, - 0xa3, 0xda, 0x86, 0x89, 0xe1, 0x79, 0x58, 0xe2, 0x61, 0xfc, 0x72, 0x58, 0x28, 0x7c, 0xe7, 0xcb, - 0xf1, 0xc0, 0x63, 0x65, 0x1b, 0x95, 0x34, 0x77, 0x03, 0x43, 0x98, 0x25, 0x5d, 0x20, 0x84, 0x42, - 0xa3, 0x38, 0xe5, 0x0d, 0x9b, 0xc3, 0x48, 0x0f, 0xf6, 0x34, 0xe5, 0x10, 0xa5, 0xf4, 0xc3, 0x91, - 0x38, 0x9b, 0x6b, 0x61, 0xa5, 0x8c, 0x7c, 0x06, 0xf7, 0xaa, 0xf0, 0xbc, 0xbb, 0x2b, 0xa4, 0xf4, - 0x6f, 0x0b, 0xb6, 0x4b, 0x29, 0xa9, 0x7b, 0xe1, 0xeb, 0x06, 0xc9, 0x99, 0x79, 0xea, 0xf9, 0x59, - 0xdd, 0x42, 0xe9, 0x4f, 0x50, 0x48, 0x3e, 0x89, 0x75, 0x6a, 0x6b, 0xac, 0x00, 0x94, 0x54, 0xfb, - 0xf8, 0x76, 0x16, 0xa3, 0x49, 0xab, 0x00, 0xc8, 0x27, 0xd0, 0x54, 0x97, 0xd2, 0x77, 0xb9, 0xf4, - 0xa3, 0xf0, 0x6b, 0x9c, 0xe9, 0x6c, 0xd6, 0xd9, 0x02, 0xaa, 0x5e, 0xb5, 0x40, 0x4c, 0xa3, 0x6e, - 0x30, 0xfd, 0x4d, 0x0e, 0x81, 0x94, 0x4a, 0x9c, 0x55, 0x63, 0x43, 0x6b, 0x54, 0x48, 0xe8, 0x19, - 0x34, 0xe7, 0x1b, 0x45, 0xba, 0xcb, 0x8d, 0x6d, 0xcc, 0xf7, 0x4d, 0x45, 0xef, 0x8f, 0x42, 0x2e, - 0xa7, 0x09, 0x9a, 0xb6, 0x15, 0x00, 0x3d, 0x86, 0xbd, 0xaa, 0xd6, 0xeb, 0x77, 0xc9, 0x2f, 0xe7, - 0x58, 0x0b, 0xc0, 0xdc, 0x5b, 0x3b, 0xbf, 0xb7, 0xbf, 0x5b, 0xb0, 0x37, 0x2c, 0xb7, 0xa1, 0x1f, - 0x85, 0x52, 0x8d, 0xb6, 0x2f, 0xa1, 0x91, 0x3e, 0xbe, 0x63, 0x0c, 0x50, 0x62, 0xc5, 0x05, 0x3e, - 0x2d, 0x89, 0x5f, 0xd7, 0xd8, 0x9c, 0x3a, 0x79, 0x6e, 0xb2, 0x33, 0xd6, 0xb6, 0xb6, 0xbe, 0xb7, - 0x78, 0xfd, 0x73, 0xe3, 0xb2, 0xf2, 0xcb, 0x4d, 0xb8, 0xf5, 0x33, 0x0f, 0xa6, 0x48, 0x3b, 0xd0, - 0x28, 0x3b, 0x59, 0x7a, 0x74, 0x47, 0xe6, 0x9e, 0x18, 0xf1, 0xc7, 0xb0, 0xe3, 0xe9, 0xaf, 0xe4, - 0x0c, 0x31, 0xc9, 0x27, 0xd6, 0x3c, 0x48, 0xdf, 0xc1, 0xdd, 0xb9, 0x84, 0x87, 0x21, 0x8f, 0xc5, - 0x38, 0x92, 0xea, 0x99, 0xa4, 0x9a, 0xde, 0xc0, 0x4b, 0x07, 0x67, 0x9d, 0x95, 0x90, 0x65, 0x7a, - 0xbb, 0x8a, 0xfe, 0x57, 0x0b, 0x1a, 0x19, 0xf5, 0x31, 0x97, 0x9c, 0x3c, 0x83, 0x4d, 0x37, 0xad, - 0xa9, 0x19, 0xc6, 0x0f, 0x17, 0xab, 0xb0, 0x50, 0x7a, 0x96, 0xe9, 0xab, 0x5d, 0x26, 0x4c, 0x74, - 0xa6, 0x82, 0xdd, 0x55, 0xb6, 0x59, 0x16, 0x2c, 0xb7, 0xa0, 0x3f, 0x9a, 0x91, 0x34, 0x9c, 0x9e, - 0x0b, 0x37, 0xf1, 0x63, 0x75, 0x9d, 0xd5, 0x5b, 0x32, 0x03, 0x3c, 0x4b, 0x31, 0x3f, 0x93, 0xe7, - 0xb0, 0xc1, 0x5d, 0xa5, 0xa5, 0x9d, 0x35, 0x7b, 0x74, 0xc9, 0x59, 0x89, 0xe9, 0x85, 0xd6, 0x64, - 0xc6, 0xe2, 0xd1, 0x25, 0x6c, 0x9d, 0x24, 0x49, 0x3f, 0xf2, 0x50, 0x90, 0x26, 0xc0, 0xdb, 0x10, - 0xaf, 0x62, 0x74, 0x25, 0x7a, 0x4e, 0x8d, 0x38, 0x66, 0xa4, 0xbd, 0xf1, 0x85, 0xf0, 0xc3, 0x91, - 0x63, 0x91, 0x5d, 0xd3, 0xb8, 0x93, 0x2b, 0x5f, 0x48, 0xe1, 0xd8, 0xe4, 0x0e, 0xec, 0x6a, 0xe0, - 0x9b, 0x48, 0x0e, 0xc2, 0x3e, 0x77, 0xc7, 0xe8, 0xac, 0x11, 0x02, 0x4d, 0x0d, 0x0e, 0x44, 0xda, - 0x60, 0xcf, 0x59, 0x57, 0x96, 0x27, 0x49, 0x12, 0x25, 0xa7, 0x17, 0x17, 0x02, 0xa5, 0xe3, 0x3d, - 0x7a, 0x06, 0xf7, 0x57, 0xc4, 0x46, 0x76, 0xa0, 0x6e, 0xd0, 0x73, 0x74, 0x6a, 0xca, 0xf4, 0x6d, - 0x28, 0x72, 0xc0, 0xea, 0xfd, 0x69, 0x43, 0x3d, 0xb5, 0x9d, 0x85, 0x2e, 0xe9, 0xc3, 0x56, 0xb6, - 0x1a, 0x49, 0xbb, 0x72, 0x5f, 0xea, 0xc9, 0xdf, 0x7e, 0x50, 0xbd, 0x4b, 0xd3, 0x89, 0xff, 0xca, - 0x30, 0xaa, 0xfd, 0x41, 0x1e, 0x2c, 0x4d, 0xfb, 0x62, 0x79, 0xb5, 0xf7, 0xab, 0x85, 0x4b, 0x3c, - 0x41, 0x50, 0xc5, 0x93, 0x2f, 0xa2, 0x2a, 0x9e, 0xd2, 0x06, 0x62, 0xe0, 0x14, 0x3b, 0x7d, 0x28, - 0x13, 0xe4, 0x13, 0xb2, 0xbf, 0xf4, 0x86, 0x4b, 0x0b, 0xbf, 0xfd, 0x41, 0xe9, 0x81, 0xf5, 0xc4, - 0x7a, 0xf9, 0xe9, 0x5f, 0xd7, 0x1d, 0xeb, 0xfd, 0x75, 0xc7, 0xfa, 0xf7, 0xba, 0x63, 0xfd, 0x76, - 0xd3, 0xa9, 0xbd, 0xbf, 0xe9, 0xd4, 0xfe, 0xb9, 0xe9, 0xd4, 0xbe, 0x6f, 0xaf, 0xfe, 0x57, 0xf1, - 0x7c, 0x43, 0xff, 0x39, 0xfa, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x13, 0xff, 0xe7, 0x17, 0x4f, 0x0a, - 0x00, 0x00, + // 1077 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x56, 0xcd, 0x6e, 0xdb, 0x46, + 0x10, 0x16, 0xe9, 0x5f, 0x8d, 0x65, 0x99, 0xd9, 0x28, 0x89, 0xaa, 0x18, 0x8a, 0xb0, 0x28, 0x0a, + 0x23, 0x07, 0x27, 0xb1, 0x8b, 0x02, 0x49, 0xdb, 0x43, 0x62, 0x3b, 0x0d, 0x51, 0x24, 0x36, 0x56, + 0x0d, 0x0a, 0x14, 0xc8, 0x61, 0x4d, 0x8e, 0x2d, 0xb6, 0x14, 0xc9, 0x72, 0x57, 0x89, 0x75, 0xec, + 0xa9, 0xd7, 0x9e, 0xdb, 0x07, 0xe8, 0x0b, 0xf4, 0x21, 0x7a, 0x4c, 0x6f, 0x3d, 0x16, 0xf6, 0x8b, + 0x14, 0xbb, 0x5c, 0xfe, 0xc8, 0xa2, 0x02, 0xe4, 0x22, 0xed, 0x7e, 0x33, 0xf3, 0xcd, 0xdf, 0xee, + 0x0e, 0xe1, 0x91, 0x17, 0x8f, 0xc7, 0x71, 0x24, 0x12, 0xee, 0xe1, 0x03, 0xfd, 0x2b, 0xa6, 0x91, + 0x97, 0xa4, 0xb1, 0x8c, 0x1f, 0xe8, 0x5f, 0x51, 0xa2, 0xbb, 0x1a, 0x20, 0xcd, 0x02, 0xa0, 0x2e, + 0x6c, 0xbe, 0x40, 0xee, 0x0f, 0xa7, 0x91, 0xc7, 0x78, 0x74, 0x8e, 0x84, 0xc0, 0xf2, 0x59, 0x1a, + 0x8f, 0xbb, 0xd6, 0xc0, 0xda, 0x59, 0x66, 0x7a, 0x4d, 0xda, 0x60, 0xcb, 0xb8, 0x6b, 0x6b, 0xc4, + 0x96, 0x31, 0xe9, 0xc0, 0x4a, 0x18, 0x8c, 0x03, 0xd9, 0x5d, 0x1a, 0x58, 0x3b, 0x9b, 0x2c, 0xdb, + 0xd0, 0x0b, 0x68, 0x17, 0x54, 0x28, 0x26, 0xa1, 0x54, 0x5c, 0x23, 0x2e, 0x46, 0x9a, 0xab, 0xc5, + 0xf4, 0x9a, 0x7c, 0x05, 0xeb, 0x18, 0xe2, 0x18, 0x23, 0x29, 0xba, 0xf6, 0x60, 0x69, 0x67, 0x63, + 0x6f, 0xb0, 0x5b, 0xc6, 0x37, 0x4b, 0x70, 0x94, 0x29, 0xb2, 0xc2, 0x42, 0x79, 0xf6, 0xe2, 0x49, + 0x54, 0x78, 0xd6, 0x1b, 0xfa, 0x25, 0xdc, 0xaa, 0x35, 0x54, 0x81, 0x07, 0xbe, 0x76, 0xdf, 0x64, + 0x76, 0xe0, 0xeb, 0x80, 0x90, 0xfb, 0x3a, 0x95, 0x26, 0xd3, 0x6b, 0xfa, 0x06, 0xb6, 0x4a, 0xe3, + 0x9f, 0x27, 0x28, 0x24, 0xe9, 0xc2, 0x9a, 0x0e, 0xc9, 0xcd, 0x6d, 0xf3, 0x2d, 0x79, 0x08, 0xab, + 0xa9, 0x2a, 0x53, 0x1e, 0x7b, 0xb7, 0x2e, 0x76, 0xa5, 0xc0, 0x8c, 0x1e, 0xfd, 0x06, 0x9c, 0x4a, + 0x6c, 0x49, 0x1c, 0x09, 0x24, 0xfb, 0xb0, 0x96, 0xea, 0x38, 0x45, 0xd7, 0xd2, 0x34, 0x9f, 0x2c, + 0x2c, 0x01, 0xcb, 0x35, 0xe9, 0x1f, 0x16, 0xdc, 0x38, 0x3e, 0xfd, 0x11, 0x3d, 0xa9, 0xa4, 0x2f, + 0x51, 0x08, 0x7e, 0x8e, 0x1f, 0x08, 0x75, 0x1b, 0x9a, 0x69, 0x96, 0x8f, 0x9b, 0x27, 0x5c, 0x02, + 0xca, 0x2e, 0xc5, 0x24, 0x9c, 0xba, 0xbe, 0x2e, 0x65, 0x93, 0xe5, 0x5b, 0x25, 0x49, 0xf8, 0x34, + 0x8c, 0xb9, 0xdf, 0x5d, 0xd6, 0x7d, 0xcb, 0xb7, 0xa4, 0x07, 0xeb, 0xb1, 0x0e, 0xc0, 0xf5, 0xbb, + 0x2b, 0xda, 0xa8, 0xd8, 0x53, 0x04, 0x67, 0xa8, 0x1c, 0x9f, 0x4c, 0xc4, 0x28, 0x2f, 0xe3, 0xa3, + 0x92, 0x49, 0xc5, 0xb6, 0xb1, 0x77, 0xa7, 0x92, 0x66, 0xa6, 0x9d, 0x89, 0x4b, 0x17, 0x7d, 0x80, + 0x83, 0x14, 0x7d, 0x8c, 0x64, 0xc0, 0x43, 0x1d, 0x75, 0x8b, 0x55, 0x10, 0x7a, 0x13, 0x6e, 0x54, + 0xdc, 0x64, 0xe5, 0xa4, 0xb4, 0xf0, 0x1d, 0x86, 0xb9, 0xef, 0x6b, 0x9d, 0xa7, 0xcf, 0x0b, 0x43, + 0xa5, 0x63, 0xfa, 0xf0, 0xf1, 0x01, 0xd2, 0x5f, 0x6c, 0x68, 0x55, 0x25, 0xe4, 0x29, 0x6c, 0x68, + 0x1b, 0xd5, 0x36, 0x4c, 0x0d, 0xcf, 0xbd, 0x0a, 0x0f, 0xe3, 0xef, 0x86, 0xa5, 0xc2, 0xf7, 0x81, + 0x1c, 0xb9, 0x3e, 0xab, 0xda, 0xa8, 0xa4, 0xb9, 0x17, 0x1a, 0xc2, 0x3c, 0xe9, 0x12, 0x21, 0x14, + 0x5a, 0xe5, 0xae, 0x68, 0xd8, 0x0c, 0x46, 0xf6, 0xa0, 0xa3, 0x29, 0x87, 0x28, 0x65, 0x10, 0x9d, + 0x8b, 0x93, 0x99, 0x16, 0xd6, 0xca, 0xc8, 0x17, 0x70, 0xbb, 0x0e, 0x2f, 0xba, 0xbb, 0x40, 0x4a, + 0xff, 0xb1, 0x60, 0xa3, 0x92, 0x92, 0x3a, 0x17, 0x81, 0x6e, 0x90, 0x9c, 0x9a, 0xab, 0x5e, 0xec, + 0xd5, 0x29, 0x94, 0xc1, 0x18, 0x85, 0xe4, 0xe3, 0x44, 0xa7, 0xb6, 0xc4, 0x4a, 0x40, 0x49, 0xb5, + 0x8f, 0xef, 0xa6, 0x09, 0x9a, 0xb4, 0x4a, 0x80, 0x7c, 0x06, 0x6d, 0x75, 0x28, 0x03, 0x8f, 0xcb, + 0x20, 0x8e, 0xbe, 0xc5, 0xa9, 0xce, 0x66, 0x99, 0x5d, 0x43, 0xd5, 0xad, 0x16, 0x88, 0x59, 0xd4, + 0x2d, 0xa6, 0xd7, 0x64, 0x17, 0x48, 0xa5, 0xc4, 0x79, 0x35, 0x56, 0xb5, 0x46, 0x8d, 0x84, 0x9e, + 0x40, 0x7b, 0xb6, 0x51, 0x64, 0x30, 0xdf, 0xd8, 0xd6, 0x6c, 0xdf, 0x54, 0xf4, 0xc1, 0x79, 0xc4, + 0xe5, 0x24, 0x45, 0xd3, 0xb6, 0x12, 0xa0, 0x87, 0xd0, 0xa9, 0x6b, 0xbd, 0xbe, 0x97, 0xfc, 0xdd, + 0x0c, 0x6b, 0x09, 0x98, 0x73, 0x6b, 0x17, 0xe7, 0xf6, 0x77, 0x0b, 0x3a, 0xc3, 0x6a, 0x1b, 0x0e, + 0xe2, 0x48, 0xaa, 0xa7, 0xed, 0x6b, 0x68, 0x65, 0x97, 0xef, 0x10, 0x43, 0x94, 0x58, 0x73, 0x80, + 0x8f, 0x2b, 0xe2, 0x17, 0x0d, 0x36, 0xa3, 0x4e, 0x9e, 0x98, 0xec, 0x8c, 0xb5, 0xad, 0xad, 0x6f, + 0x5f, 0x3f, 0xfe, 0x85, 0x71, 0x55, 0xf9, 0xd9, 0x1a, 0xac, 0xbc, 0xe5, 0xe1, 0x04, 0x69, 0x1f, + 0x5a, 0x55, 0x27, 0x73, 0x97, 0x6e, 0xdf, 0x9c, 0x13, 0x23, 0xfe, 0x14, 0x36, 0x7d, 0xbd, 0x4a, + 0x4f, 0x10, 0xd3, 0xe2, 0xc5, 0x9a, 0x05, 0xe9, 0x1b, 0xb8, 0x35, 0x93, 0xf0, 0x30, 0xe2, 0x89, + 0x18, 0xc5, 0x52, 0x5d, 0x93, 0x4c, 0xd3, 0x77, 0xfd, 0xec, 0xe1, 0x6c, 0xb2, 0x0a, 0x32, 0x4f, + 0x6f, 0xd7, 0xd1, 0xff, 0x6a, 0x41, 0x2b, 0xa7, 0x3e, 0xe4, 0x92, 0x93, 0xc7, 0xb0, 0xe6, 0x65, + 0x35, 0x35, 0x8f, 0xf1, 0xbd, 0xeb, 0x55, 0xb8, 0x56, 0x7a, 0x96, 0xeb, 0xab, 0x59, 0x26, 0x4c, + 0x74, 0xa6, 0x82, 0x83, 0x45, 0xb6, 0x79, 0x16, 0xac, 0xb0, 0xa0, 0x3f, 0x99, 0x27, 0x69, 0x38, + 0x39, 0x15, 0x5e, 0x1a, 0x24, 0xea, 0x38, 0xab, 0xbb, 0x64, 0x1e, 0xf0, 0x3c, 0xc5, 0x62, 0x4f, + 0x9e, 0xc0, 0x2a, 0xf7, 0x94, 0x96, 0x76, 0xd6, 0xde, 0xa3, 0x73, 0xce, 0x2a, 0x4c, 0x4f, 0xb5, + 0x26, 0x33, 0x16, 0xf7, 0xff, 0xb4, 0x60, 0xfd, 0x28, 0x4d, 0x0f, 0x62, 0x1f, 0x05, 0x69, 0x03, + 0xbc, 0x8e, 0xf0, 0x22, 0x41, 0x4f, 0xa2, 0xef, 0x34, 0x88, 0x63, 0xde, 0xb4, 0x97, 0x81, 0x10, + 0x41, 0x74, 0xee, 0x58, 0x64, 0xcb, 0x74, 0xee, 0xe8, 0x22, 0x10, 0x52, 0x38, 0x36, 0xb9, 0x09, + 0x5b, 0x1a, 0x78, 0x15, 0x4b, 0x37, 0x3a, 0xe0, 0xde, 0x08, 0x9d, 0x25, 0x42, 0xa0, 0xad, 0x41, + 0x57, 0x64, 0x1d, 0xf6, 0x9d, 0x65, 0xd2, 0x85, 0x8e, 0xae, 0xb4, 0x78, 0x15, 0x4b, 0xf3, 0xd0, + 0x06, 0xa7, 0x21, 0x3a, 0x2b, 0xa4, 0x03, 0x0e, 0x43, 0x0f, 0x83, 0x44, 0xba, 0xc2, 0x8d, 0xde, + 0xf2, 0x30, 0xf0, 0x9d, 0x55, 0xe5, 0xe9, 0x28, 0x4d, 0xe3, 0xf4, 0xf8, 0xec, 0x4c, 0xa0, 0x74, + 0xfc, 0xfb, 0x8f, 0xe1, 0xce, 0x82, 0x64, 0xc8, 0x26, 0x34, 0x0d, 0x7a, 0x8a, 0x4e, 0x43, 0x99, + 0xbe, 0x8e, 0x44, 0x01, 0x58, 0x7b, 0x7f, 0xd9, 0xd0, 0xcc, 0x6c, 0xa7, 0x91, 0x47, 0x0e, 0x60, + 0x3d, 0x9f, 0xa5, 0xa4, 0x57, 0x3b, 0x60, 0xf5, 0xa8, 0xe8, 0xdd, 0xad, 0x1f, 0xbe, 0xd9, 0x88, + 0x78, 0x6e, 0x18, 0xd5, 0xc0, 0x21, 0x77, 0xe7, 0xc6, 0x43, 0x39, 0xed, 0x7a, 0xdb, 0xf5, 0xc2, + 0x39, 0x9e, 0x30, 0xac, 0xe3, 0x29, 0x26, 0x57, 0x1d, 0x4f, 0x65, 0x64, 0x31, 0x70, 0xca, 0x8f, + 0x80, 0xa1, 0x4c, 0x91, 0x8f, 0xc9, 0xf6, 0xdc, 0xa5, 0xaf, 0x7c, 0x21, 0xf4, 0x3e, 0x28, 0xdd, + 0xb1, 0x1e, 0x5a, 0xcf, 0x3e, 0xff, 0xfb, 0xb2, 0x6f, 0xbd, 0xbf, 0xec, 0x5b, 0xff, 0x5d, 0xf6, + 0xad, 0xdf, 0xae, 0xfa, 0x8d, 0xf7, 0x57, 0xfd, 0xc6, 0xbf, 0x57, 0xfd, 0xc6, 0x0f, 0xbd, 0xc5, + 0xdf, 0x96, 0xa7, 0xab, 0xfa, 0x6f, 0xff, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd3, 0x01, 0xff, + 0xb5, 0x80, 0x0a, 0x00, 0x00, } func (m *HeadSyncRange) Marshal() (dAtA []byte, err error) { From 5082a10a18b76b8772e88bceed6b6f9e48965b30 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Mon, 24 Apr 2023 15:04:11 +0200 Subject: [PATCH 36/45] nodeconf update interval config --- nodeconf/config.go | 4 ++++ nodeconf/service.go | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/nodeconf/config.go b/nodeconf/config.go index 911ee745..1d117d88 100644 --- a/nodeconf/config.go +++ b/nodeconf/config.go @@ -9,6 +9,10 @@ type ConfigGetter interface { GetNodeConf() Configuration } +type ConfigUpdateGetter interface { + GetNodeConfUpdateInterval() int +} + var ( ErrConfigurationNotFound = errors.New("node nodeConf not found") ) diff --git a/nodeconf/service.go b/nodeconf/service.go index 3f12e462..ef2dd9ef 100644 --- a/nodeconf/service.go +++ b/nodeconf/service.go @@ -49,7 +49,12 @@ func (s *service) Init(a *app.App) (err error) { lastStored = s.config err = nil } - s.sync = periodicsync.NewPeriodicSync(600, 0, func(ctx context.Context) (err error) { + var updatePeriodSec = 600 + if confUpd, ok := a.MustComponent("config").(ConfigUpdateGetter); ok && confUpd.GetNodeConfUpdateInterval() > 0 { + updatePeriodSec = confUpd.GetNodeConfUpdateInterval() + } + + s.sync = periodicsync.NewPeriodicSync(updatePeriodSec, 0, func(ctx context.Context) (err error) { err = s.updateConfiguration(ctx) if err != nil { if err == ErrConfigurationNotChanged { From b0f3fec939c74e247e017d03372fa9604c20efb6 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 24 Apr 2023 20:33:35 +0200 Subject: [PATCH 37/45] Add receipt check --- commonspace/checks.go | 44 ++++++++++++ coordinator/coordinatorclient/receipt.go | 85 ++++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 commonspace/checks.go create mode 100644 coordinator/coordinatorclient/receipt.go diff --git a/commonspace/checks.go b/commonspace/checks.go new file mode 100644 index 00000000..5469e2f2 --- /dev/null +++ b/commonspace/checks.go @@ -0,0 +1,44 @@ +package commonspace + +import ( + "context" + "github.com/anytypeio/any-sync/commonspace/spacesyncproto" + "github.com/anytypeio/any-sync/net/peer" + "github.com/anytypeio/any-sync/nodeconf" + "github.com/anytypeio/any-sync/util/crypto" + "golang.org/x/exp/slices" +) + +func CheckResponsible(ctx context.Context, confService nodeconf.Service, spaceId string) (err error) { + peerId, err := peer.CtxPeerId(ctx) + if err != nil { + return + } + if isClient(confService, peerId) && !confService.IsResponsible(spaceId) { + return spacesyncproto.ErrPeerIsNotResponsible + } + return +} + +func isClient(confService nodeconf.Service, peerId string) bool { + return len(confService.NodeTypes(peerId)) == 0 +} + +func checkCoordinator(confService nodeconf.Service, identity []byte, payload, signature []byte) (err error) { + controlKey, err := crypto.UnmarshalEd25519PublicKey(identity) + if err != nil { + return + } + nodeTypes := confService.NodeTypes(controlKey.PeerId()) + if len(nodeTypes) == 0 || !slices.Contains(nodeTypes, nodeconf.NodeTypeCoordinator) { + return errNoSuchCoordinatorNode + } + res, err := controlKey.Verify(payload, signature) + if err != nil { + return + } + if !res { + return errReceiptSignatureIncorrect + } + return +} diff --git a/coordinator/coordinatorclient/receipt.go b/coordinator/coordinatorclient/receipt.go new file mode 100644 index 00000000..4a7f3162 --- /dev/null +++ b/coordinator/coordinatorclient/receipt.go @@ -0,0 +1,85 @@ +package coordinatorclient + +import ( + "bytes" + "context" + "errors" + "github.com/anytypeio/any-sync/commonspace/spacesyncproto" + "github.com/anytypeio/any-sync/coordinator/coordinatorproto" + "github.com/anytypeio/any-sync/net/peer" + "github.com/anytypeio/any-sync/nodeconf" + "github.com/anytypeio/any-sync/util/crypto" + "github.com/gogo/protobuf/proto" + "golang.org/x/exp/slices" + "time" +) + +var ( + errReceiptSignatureIncorrect = errors.New("receipt signature is incorrect") + errNoSuchCoordinatorNode = errors.New("no such control node") + errReceiptSpaceIdIncorrect = errors.New("receipt space id is incorrect") + errReceiptPeerIdIncorrect = errors.New("receipt peer id is incorrect") + errReceiptAccountIncorrect = errors.New("receipt account id is incorrect") + errReceiptExpired = errors.New("receipt is expired") +) + +func CheckReceipt(ctx context.Context, confService nodeconf.Service, request *spacesyncproto.SpacePushRequest) (err error) { + peerId, err := peer.CtxPeerId(ctx) + if err != nil { + return + } + accountIdentity, err := peer.CtxIdentity(ctx) + if err != nil { + return + } + credential := &coordinatorproto.SpaceReceiptWithSignature{} + err = proto.Unmarshal(request.GetCredential(), credential) + if err != nil { + return + } + payload := &coordinatorproto.SpaceReceipt{} + err = proto.Unmarshal(credential.GetSpaceReceiptPayload(), payload) + if err != nil { + return + } + if payload.GetSpaceId() != request.GetPayload().GetSpaceHeader().GetId() { + return errReceiptSpaceIdIncorrect + } + if payload.GetPeerId() != peerId { + return errReceiptPeerIdIncorrect + } + if !bytes.Equal(payload.GetAccountIdentity(), accountIdentity) { + return errReceiptAccountIncorrect + } + err = checkCoordinator( + confService, + payload.GetControlNodeIdentity(), + credential.GetSpaceReceiptPayload(), + credential.GetSignature()) + if err != nil { + return + } + if payload.GetValidUntil() < uint64(time.Now().Unix()) { + return errReceiptExpired + } + return +} + +func checkCoordinator(confService nodeconf.Service, identity []byte, payload, signature []byte) (err error) { + cooordinatorKey, err := crypto.UnmarshalEd25519PublicKey(identity) + if err != nil { + return + } + nodeTypes := confService.NodeTypes(cooordinatorKey.PeerId()) + if len(nodeTypes) == 0 || !slices.Contains(nodeTypes, nodeconf.NodeTypeCoordinator) { + return errNoSuchCoordinatorNode + } + res, err := cooordinatorKey.Verify(payload, signature) + if err != nil { + return + } + if !res { + return errReceiptSignatureIncorrect + } + return +} From 4b330e8b2da3db16417b45cdbbd242d3bdcd02fc Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 25 Apr 2023 09:54:12 +0200 Subject: [PATCH 38/45] Move receipt logic to any-sync, write tests --- coordinator/coordinatorclient/receipt.go | 85 ------------ coordinator/coordinatorproto/receipt.go | 100 ++++++++++++++ coordinator/coordinatorproto/receipt_test.go | 130 +++++++++++++++++++ 3 files changed, 230 insertions(+), 85 deletions(-) delete mode 100644 coordinator/coordinatorclient/receipt.go create mode 100644 coordinator/coordinatorproto/receipt.go create mode 100644 coordinator/coordinatorproto/receipt_test.go diff --git a/coordinator/coordinatorclient/receipt.go b/coordinator/coordinatorclient/receipt.go deleted file mode 100644 index 4a7f3162..00000000 --- a/coordinator/coordinatorclient/receipt.go +++ /dev/null @@ -1,85 +0,0 @@ -package coordinatorclient - -import ( - "bytes" - "context" - "errors" - "github.com/anytypeio/any-sync/commonspace/spacesyncproto" - "github.com/anytypeio/any-sync/coordinator/coordinatorproto" - "github.com/anytypeio/any-sync/net/peer" - "github.com/anytypeio/any-sync/nodeconf" - "github.com/anytypeio/any-sync/util/crypto" - "github.com/gogo/protobuf/proto" - "golang.org/x/exp/slices" - "time" -) - -var ( - errReceiptSignatureIncorrect = errors.New("receipt signature is incorrect") - errNoSuchCoordinatorNode = errors.New("no such control node") - errReceiptSpaceIdIncorrect = errors.New("receipt space id is incorrect") - errReceiptPeerIdIncorrect = errors.New("receipt peer id is incorrect") - errReceiptAccountIncorrect = errors.New("receipt account id is incorrect") - errReceiptExpired = errors.New("receipt is expired") -) - -func CheckReceipt(ctx context.Context, confService nodeconf.Service, request *spacesyncproto.SpacePushRequest) (err error) { - peerId, err := peer.CtxPeerId(ctx) - if err != nil { - return - } - accountIdentity, err := peer.CtxIdentity(ctx) - if err != nil { - return - } - credential := &coordinatorproto.SpaceReceiptWithSignature{} - err = proto.Unmarshal(request.GetCredential(), credential) - if err != nil { - return - } - payload := &coordinatorproto.SpaceReceipt{} - err = proto.Unmarshal(credential.GetSpaceReceiptPayload(), payload) - if err != nil { - return - } - if payload.GetSpaceId() != request.GetPayload().GetSpaceHeader().GetId() { - return errReceiptSpaceIdIncorrect - } - if payload.GetPeerId() != peerId { - return errReceiptPeerIdIncorrect - } - if !bytes.Equal(payload.GetAccountIdentity(), accountIdentity) { - return errReceiptAccountIncorrect - } - err = checkCoordinator( - confService, - payload.GetControlNodeIdentity(), - credential.GetSpaceReceiptPayload(), - credential.GetSignature()) - if err != nil { - return - } - if payload.GetValidUntil() < uint64(time.Now().Unix()) { - return errReceiptExpired - } - return -} - -func checkCoordinator(confService nodeconf.Service, identity []byte, payload, signature []byte) (err error) { - cooordinatorKey, err := crypto.UnmarshalEd25519PublicKey(identity) - if err != nil { - return - } - nodeTypes := confService.NodeTypes(cooordinatorKey.PeerId()) - if len(nodeTypes) == 0 || !slices.Contains(nodeTypes, nodeconf.NodeTypeCoordinator) { - return errNoSuchCoordinatorNode - } - res, err := cooordinatorKey.Verify(payload, signature) - if err != nil { - return - } - if !res { - return errReceiptSignatureIncorrect - } - return -} diff --git a/coordinator/coordinatorproto/receipt.go b/coordinator/coordinatorproto/receipt.go new file mode 100644 index 00000000..630eeffb --- /dev/null +++ b/coordinator/coordinatorproto/receipt.go @@ -0,0 +1,100 @@ +package coordinatorproto + +import ( + "bytes" + "errors" + "github.com/anytypeio/any-sync/util/crypto" + "github.com/gogo/protobuf/proto" + "golang.org/x/exp/slices" + "time" +) + +var ( + errReceiptSignatureIncorrect = errors.New("receipt signature is incorrect") + errNoSuchCoordinatorNode = errors.New("no such control node") + errReceiptSpaceIdIncorrect = errors.New("receipt space id is incorrect") + errReceiptPeerIdIncorrect = errors.New("receipt peer id is incorrect") + errReceiptAccountIncorrect = errors.New("receipt account is incorrect") + errReceiptExpired = errors.New("receipt is expired") +) + +func PrepareSpaceReceipt(spaceId, peerId string, validPeriod time.Duration, accountPubKey crypto.PubKey, nodeKey crypto.PrivKey) (signedReceipt *SpaceReceiptWithSignature, err error) { + marshalledAccount, err := accountPubKey.Marshall() + if err != nil { + return + } + marshalledNode, err := nodeKey.GetPublic().Marshall() + if err != nil { + return + } + receipt := &SpaceReceipt{ + SpaceId: spaceId, + PeerId: peerId, + AccountIdentity: marshalledAccount, + ControlNodeIdentity: marshalledNode, + ValidUntil: uint64(time.Now().Add(validPeriod).Unix()), + } + receiptData, err := receipt.Marshal() + if err != nil { + return + } + sign, err := nodeKey.Sign(receiptData) + if err != nil { + return + } + return &SpaceReceiptWithSignature{ + SpaceReceiptPayload: receiptData, + Signature: sign, + }, nil +} + +func CheckReceipt(peerId, spaceId string, accountIdentity []byte, coordinators []string, receipt *SpaceReceiptWithSignature) (err error) { + payload := &SpaceReceipt{} + err = proto.Unmarshal(receipt.GetSpaceReceiptPayload(), payload) + if err != nil { + return + } + if payload.SpaceId != spaceId { + return errReceiptSpaceIdIncorrect + } + if payload.PeerId != peerId { + return errReceiptPeerIdIncorrect + } + protoIdentity, err := crypto.UnmarshalEd25519PublicKeyProto(payload.AccountIdentity) + if err != nil { + return + } + if !bytes.Equal(protoIdentity.Storage(), accountIdentity) { + return errReceiptAccountIncorrect + } + err = checkCoordinator( + coordinators, + payload.ControlNodeIdentity, + receipt.GetSpaceReceiptPayload(), + receipt.GetSignature()) + if err != nil { + return + } + if payload.GetValidUntil() <= uint64(time.Now().Unix()) { + return errReceiptExpired + } + return +} + +func checkCoordinator(coordinators []string, identity []byte, payload, signature []byte) (err error) { + coordinatorKey, err := crypto.UnmarshalEd25519PublicKeyProto(identity) + if err != nil { + return + } + if !slices.Contains(coordinators, coordinatorKey.PeerId()) { + return errNoSuchCoordinatorNode + } + res, err := coordinatorKey.Verify(payload, signature) + if err != nil { + return + } + if !res { + return errReceiptSignatureIncorrect + } + return +} diff --git a/coordinator/coordinatorproto/receipt_test.go b/coordinator/coordinatorproto/receipt_test.go new file mode 100644 index 00000000..38ca6d75 --- /dev/null +++ b/coordinator/coordinatorproto/receipt_test.go @@ -0,0 +1,130 @@ +package coordinatorproto + +import ( + "context" + "crypto/rand" + "github.com/anytypeio/any-sync/util/crypto" + "github.com/gogo/protobuf/proto" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +type fixture struct { + coordinatorKey crypto.PrivKey + accountKey crypto.PubKey + accountIdentity []byte + ctx context.Context + originalReceipt *SpaceReceipt + signedReceipt *SpaceReceiptWithSignature + spaceId string + peerId string +} + +func newFixture(t *testing.T) *fixture { + coordinatorKey, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + signKey, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + signKeyRaw, err := signKey.GetPublic().Raw() + require.NoError(t, err) + return &fixture{ + spaceId: "spaceId", + peerId: "peerId", + accountIdentity: signKeyRaw, + coordinatorKey: coordinatorKey, + accountKey: signKey.GetPublic(), + } +} + +func (fx *fixture) prepareReceipt(t *testing.T, validPeriod time.Duration) { + var err error + fx.signedReceipt, err = PrepareSpaceReceipt(fx.spaceId, fx.peerId, validPeriod, fx.accountKey, fx.coordinatorKey) + require.NoError(t, err) + fx.originalReceipt = &SpaceReceipt{} + err = proto.Unmarshal(fx.signedReceipt.SpaceReceiptPayload, fx.originalReceipt) + require.NoError(t, err) + return +} + +func (fx *fixture) updateReceipt(t *testing.T, update func(t *testing.T, receipt *SpaceReceipt)) { + update(t, fx.originalReceipt) + marshalled, err := proto.Marshal(fx.originalReceipt) + require.NoError(t, err) + signature, err := fx.coordinatorKey.Sign(marshalled) + require.NoError(t, err) + fx.signedReceipt = &SpaceReceiptWithSignature{ + SpaceReceiptPayload: marshalled, + Signature: signature, + } +} + +func TestReceiptValid(t *testing.T) { + fx := newFixture(t) + fx.prepareReceipt(t, time.Second) + err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + require.NoError(t, err) +} + +func TestReceiptIncorrectSpaceId(t *testing.T) { + fx := newFixture(t) + fx.prepareReceipt(t, time.Second) + err := CheckReceipt(fx.peerId, "otherId", fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + require.Error(t, errReceiptSpaceIdIncorrect, err) +} + +func TestReceiptIncorrectPeerId(t *testing.T) { + fx := newFixture(t) + fx.prepareReceipt(t, time.Second) + err := CheckReceipt("otherId", fx.spaceId, fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + require.Error(t, errReceiptPeerIdIncorrect, err) +} + +func TestReceiptIncorrectAccountIdentity(t *testing.T) { + fx := newFixture(t) + fx.prepareReceipt(t, time.Second) + err := CheckReceipt(fx.peerId, fx.spaceId, []byte("some identity"), []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + require.Error(t, errReceiptAccountIncorrect, err) +} + +func TestReceiptIncorrectCoordinatorKey(t *testing.T) { + fx := newFixture(t) + fx.prepareReceipt(t, time.Second) + + t.Run("random key payload", func(t *testing.T) { + fx.updateReceipt(t, func(t *testing.T, receipt *SpaceReceipt) { + receipt.AccountIdentity = []byte("some random stuff") + }) + err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + require.Error(t, err) + }) + t.Run("random incorrect key", func(t *testing.T) { + fx.updateReceipt(t, func(t *testing.T, receipt *SpaceReceipt) { + randomKey, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + keyBytes, err := randomKey.GetPublic().Marshall() + require.NoError(t, err) + receipt.AccountIdentity = keyBytes + }) + err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + require.Error(t, errNoSuchCoordinatorNode, err) + }) +} + +func TestReceiptIncorrectSignature(t *testing.T) { + fx := newFixture(t) + fx.prepareReceipt(t, time.Second) + fx.signedReceipt.Signature = []byte("random sig") + err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + require.Error(t, errReceiptSignatureIncorrect, err) +} + +func TestReceiptExpired(t *testing.T) { + fx := newFixture(t) + fx.prepareReceipt(t, time.Second) + fx.updateReceipt(t, func(t *testing.T, receipt *SpaceReceipt) { + receipt.ValidUntil = uint64(time.Now().Add(-time.Second).Unix()) + }) + err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + require.Error(t, errReceiptExpired, err) +} From bf44f1b24f183ee126997cd59531284256294a8a Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 25 Apr 2023 10:04:42 +0200 Subject: [PATCH 39/45] Use same format for identity --- coordinator/coordinatorproto/receipt.go | 8 ++++++-- coordinator/coordinatorproto/receipt_test.go | 8 ++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/coordinator/coordinatorproto/receipt.go b/coordinator/coordinatorproto/receipt.go index 630eeffb..1c87bb12 100644 --- a/coordinator/coordinatorproto/receipt.go +++ b/coordinator/coordinatorproto/receipt.go @@ -60,11 +60,15 @@ func CheckReceipt(peerId, spaceId string, accountIdentity []byte, coordinators [ if payload.PeerId != peerId { return errReceiptPeerIdIncorrect } - protoIdentity, err := crypto.UnmarshalEd25519PublicKeyProto(payload.AccountIdentity) + protoRaw, err := crypto.UnmarshalEd25519PublicKeyProto(payload.AccountIdentity) if err != nil { return } - if !bytes.Equal(protoIdentity.Storage(), accountIdentity) { + accountRaw, err := crypto.UnmarshalEd25519PublicKeyProto(accountIdentity) + if err != nil { + return + } + if !bytes.Equal(protoRaw.Storage(), accountRaw.Storage()) { return errReceiptAccountIncorrect } err = checkCoordinator( diff --git a/coordinator/coordinatorproto/receipt_test.go b/coordinator/coordinatorproto/receipt_test.go index 38ca6d75..1a85a7d1 100644 --- a/coordinator/coordinatorproto/receipt_test.go +++ b/coordinator/coordinatorproto/receipt_test.go @@ -24,16 +24,16 @@ type fixture struct { func newFixture(t *testing.T) *fixture { coordinatorKey, _, err := crypto.GenerateEd25519Key(rand.Reader) require.NoError(t, err) - signKey, _, err := crypto.GenerateEd25519Key(rand.Reader) + accountKey, _, err := crypto.GenerateEd25519Key(rand.Reader) require.NoError(t, err) - signKeyRaw, err := signKey.GetPublic().Raw() + accountKeyProto, err := accountKey.GetPublic().Marshall() require.NoError(t, err) return &fixture{ spaceId: "spaceId", peerId: "peerId", - accountIdentity: signKeyRaw, + accountIdentity: accountKeyProto, coordinatorKey: coordinatorKey, - accountKey: signKey.GetPublic(), + accountKey: accountKey.GetPublic(), } } From 854913a22b54adc3532f4625464617ef167fe36a Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 25 Apr 2023 10:38:22 +0200 Subject: [PATCH 40/45] Remove commonspace checks --- commonspace/checks.go | 44 ------------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 commonspace/checks.go diff --git a/commonspace/checks.go b/commonspace/checks.go deleted file mode 100644 index 5469e2f2..00000000 --- a/commonspace/checks.go +++ /dev/null @@ -1,44 +0,0 @@ -package commonspace - -import ( - "context" - "github.com/anytypeio/any-sync/commonspace/spacesyncproto" - "github.com/anytypeio/any-sync/net/peer" - "github.com/anytypeio/any-sync/nodeconf" - "github.com/anytypeio/any-sync/util/crypto" - "golang.org/x/exp/slices" -) - -func CheckResponsible(ctx context.Context, confService nodeconf.Service, spaceId string) (err error) { - peerId, err := peer.CtxPeerId(ctx) - if err != nil { - return - } - if isClient(confService, peerId) && !confService.IsResponsible(spaceId) { - return spacesyncproto.ErrPeerIsNotResponsible - } - return -} - -func isClient(confService nodeconf.Service, peerId string) bool { - return len(confService.NodeTypes(peerId)) == 0 -} - -func checkCoordinator(confService nodeconf.Service, identity []byte, payload, signature []byte) (err error) { - controlKey, err := crypto.UnmarshalEd25519PublicKey(identity) - if err != nil { - return - } - nodeTypes := confService.NodeTypes(controlKey.PeerId()) - if len(nodeTypes) == 0 || !slices.Contains(nodeTypes, nodeconf.NodeTypeCoordinator) { - return errNoSuchCoordinatorNode - } - res, err := controlKey.Verify(payload, signature) - if err != nil { - return - } - if !res { - return errReceiptSignatureIncorrect - } - return -} From cadd63d45804e532aa5b4ab3662dd44219fff760 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 25 Apr 2023 14:00:38 +0200 Subject: [PATCH 41/45] Nodeconf error and periodicsync partial rename --- nodeconf/service.go | 2 +- util/periodicsync/periodicsync.go | 55 +++++++++++++------------- util/periodicsync/periodicsync_test.go | 4 +- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/nodeconf/service.go b/nodeconf/service.go index ef2dd9ef..d19d0874 100644 --- a/nodeconf/service.go +++ b/nodeconf/service.go @@ -57,7 +57,7 @@ func (s *service) Init(a *app.App) (err error) { s.sync = periodicsync.NewPeriodicSync(updatePeriodSec, 0, func(ctx context.Context) (err error) { err = s.updateConfiguration(ctx) if err != nil { - if err == ErrConfigurationNotChanged { + if err == ErrConfigurationNotChanged || err == ErrConfigurationNotFound { err = nil } } diff --git a/util/periodicsync/periodicsync.go b/util/periodicsync/periodicsync.go index 3ce74cdd..abe998c0 100644 --- a/util/periodicsync/periodicsync.go +++ b/util/periodicsync/periodicsync.go @@ -15,64 +15,65 @@ type PeriodicSync interface { type SyncerFunc func(ctx context.Context) error -func NewPeriodicSync(periodSeconds int, timeout time.Duration, syncer SyncerFunc, l logger.CtxLogger) PeriodicSync { +func NewPeriodicSync(periodSeconds int, timeout time.Duration, caller SyncerFunc, l logger.CtxLogger) PeriodicSync { + // TODO: rename to PeriodicCall ctx, cancel := context.WithCancel(context.Background()) - ctx = logger.CtxWithFields(ctx, zap.String("rootOp", "periodicSync")) - return &periodicSync{ - syncer: syncer, + ctx = logger.CtxWithFields(ctx, zap.String("rootOp", "periodicCall")) + return &periodicCall{ + caller: caller, log: l, - syncCtx: ctx, - syncCancel: cancel, - syncLoopDone: make(chan struct{}), + loopCtx: ctx, + loopCancel: cancel, + loopDone: make(chan struct{}), periodSeconds: periodSeconds, timeout: timeout, } } -type periodicSync struct { +type periodicCall struct { log logger.CtxLogger - syncer SyncerFunc - syncCtx context.Context - syncCancel context.CancelFunc - syncLoopDone chan struct{} + caller SyncerFunc + loopCtx context.Context + loopCancel context.CancelFunc + loopDone chan struct{} periodSeconds int timeout time.Duration } -func (p *periodicSync) Run() { - go p.syncLoop(p.periodSeconds) +func (p *periodicCall) Run() { + go p.loop(p.periodSeconds) } -func (p *periodicSync) syncLoop(periodSeconds int) { +func (p *periodicCall) loop(periodSeconds int) { period := time.Duration(periodSeconds) * time.Second - defer close(p.syncLoopDone) - doSync := func() { - ctx := p.syncCtx + defer close(p.loopDone) + doCall := func() { + ctx := p.loopCtx if p.timeout != 0 { var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(p.syncCtx, p.timeout) + ctx, cancel = context.WithTimeout(p.loopCtx, p.timeout) defer cancel() } - if err := p.syncer(ctx); err != nil { - p.log.Warn("periodic sync error", zap.Error(err)) + if err := p.caller(ctx); err != nil { + p.log.Warn("periodic call error", zap.Error(err)) } } - doSync() + doCall() if period > 0 { ticker := time.NewTicker(period) defer ticker.Stop() for { select { - case <-p.syncCtx.Done(): + case <-p.loopCtx.Done(): return case <-ticker.C: - doSync() + doCall() } } } } -func (p *periodicSync) Close() { - p.syncCancel() - <-p.syncLoopDone +func (p *periodicCall) Close() { + p.loopCancel() + <-p.loopDone } diff --git a/util/periodicsync/periodicsync_test.go b/util/periodicsync/periodicsync_test.go index 6ed933a4..fa1c091d 100644 --- a/util/periodicsync/periodicsync_test.go +++ b/util/periodicsync/periodicsync_test.go @@ -16,7 +16,7 @@ func TestPeriodicSync_Run(t *testing.T) { l := logger.NewNamed("sync") - t.Run("diff syncer 1 time", func(t *testing.T) { + t.Run("loop call 1 time", func(t *testing.T) { secs := 0 times := 0 diffSyncer := func(ctx context.Context) (err error) { @@ -30,7 +30,7 @@ func TestPeriodicSync_Run(t *testing.T) { require.Equal(t, 1, times) }) - t.Run("diff syncer 2 times", func(t *testing.T) { + t.Run("loop call 2 times", func(t *testing.T) { secs := 1 times := 0 From 6ec030942f9eaef5c125fff94d2cd86e544a66ed Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 25 Apr 2023 14:25:08 +0200 Subject: [PATCH 42/45] More info in receipt errors and change tests --- coordinator/coordinatorproto/receipt.go | 6 ++++-- coordinator/coordinatorproto/receipt_test.go | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/coordinator/coordinatorproto/receipt.go b/coordinator/coordinatorproto/receipt.go index 1c87bb12..67287d9d 100644 --- a/coordinator/coordinatorproto/receipt.go +++ b/coordinator/coordinatorproto/receipt.go @@ -3,6 +3,7 @@ package coordinatorproto import ( "bytes" "errors" + "fmt" "github.com/anytypeio/any-sync/util/crypto" "github.com/gogo/protobuf/proto" "golang.org/x/exp/slices" @@ -11,7 +12,7 @@ import ( var ( errReceiptSignatureIncorrect = errors.New("receipt signature is incorrect") - errNoSuchCoordinatorNode = errors.New("no such control node") + errNoSuchCoordinatorNode = errors.New("no such coordinator node") errReceiptSpaceIdIncorrect = errors.New("receipt space id is incorrect") errReceiptPeerIdIncorrect = errors.New("receipt peer id is incorrect") errReceiptAccountIncorrect = errors.New("receipt account is incorrect") @@ -90,8 +91,9 @@ func checkCoordinator(coordinators []string, identity []byte, payload, signature if err != nil { return } + receiptCoordinator := coordinatorKey.PeerId() if !slices.Contains(coordinators, coordinatorKey.PeerId()) { - return errNoSuchCoordinatorNode + return fmt.Errorf("got coordinator %s: %w", receiptCoordinator, errNoSuchCoordinatorNode) } res, err := coordinatorKey.Verify(payload, signature) if err != nil { diff --git a/coordinator/coordinatorproto/receipt_test.go b/coordinator/coordinatorproto/receipt_test.go index 1a85a7d1..fc2bc200 100644 --- a/coordinator/coordinatorproto/receipt_test.go +++ b/coordinator/coordinatorproto/receipt_test.go @@ -3,6 +3,7 @@ package coordinatorproto import ( "context" "crypto/rand" + "errors" "github.com/anytypeio/any-sync/util/crypto" "github.com/gogo/protobuf/proto" "github.com/stretchr/testify/require" @@ -93,7 +94,7 @@ func TestReceiptIncorrectCoordinatorKey(t *testing.T) { t.Run("random key payload", func(t *testing.T) { fx.updateReceipt(t, func(t *testing.T, receipt *SpaceReceipt) { - receipt.AccountIdentity = []byte("some random stuff") + receipt.ControlNodeIdentity = []byte("some random stuff") }) err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) require.Error(t, err) @@ -104,10 +105,10 @@ func TestReceiptIncorrectCoordinatorKey(t *testing.T) { require.NoError(t, err) keyBytes, err := randomKey.GetPublic().Marshall() require.NoError(t, err) - receipt.AccountIdentity = keyBytes + receipt.ControlNodeIdentity = keyBytes }) err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) - require.Error(t, errNoSuchCoordinatorNode, err) + require.True(t, errors.Is(err, errNoSuchCoordinatorNode)) }) } From 1d29aba45ee1151b61da8fcbd2c9629c1110ab20 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 25 Apr 2023 20:49:31 +0200 Subject: [PATCH 43/45] Update space receipt to include network id --- .../coordinatorproto/coordinator.pb.go | 162 +++++++++--------- .../coordinatorproto/protos/coordinator.proto | 12 +- coordinator/coordinatorproto/receipt.go | 46 +++-- coordinator/coordinatorproto/receipt_test.go | 39 ++--- 4 files changed, 126 insertions(+), 133 deletions(-) diff --git a/coordinator/coordinatorproto/coordinator.pb.go b/coordinator/coordinatorproto/coordinator.pb.go index a08e0891..cef0151b 100644 --- a/coordinator/coordinatorproto/coordinator.pb.go +++ b/coordinator/coordinatorproto/coordinator.pb.go @@ -345,15 +345,15 @@ func (m *SpaceReceiptWithSignature) GetSignature() []byte { // SpaceReceipt contains permission to SpacePush operation type SpaceReceipt struct { - // spaceId + // SpaceId is the identifier of space SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` - // peerId of receipt requester + // PeerId of receipt requester PeerId string `protobuf:"bytes,2,opt,name=peerId,proto3" json:"peerId,omitempty"` - // identity of space owner + // AccountIdentity is an identity of a space owner AccountIdentity []byte `protobuf:"bytes,3,opt,name=accountIdentity,proto3" json:"accountIdentity,omitempty"` - // identity of control node - ControlNodeIdentity []byte `protobuf:"bytes,4,opt,name=controlNodeIdentity,proto3" json:"controlNodeIdentity,omitempty"` - // unix-timestamp with a deadline time of receipt validity + // NetworkId is the id of a network where the receipt is issued + NetworkId string `protobuf:"bytes,4,opt,name=networkId,proto3" json:"networkId,omitempty"` + // ValidUntil is a unix-timestamp with a deadline time of receipt validity ValidUntil uint64 `protobuf:"varint,5,opt,name=validUntil,proto3" json:"validUntil,omitempty"` } @@ -411,11 +411,11 @@ func (m *SpaceReceipt) GetAccountIdentity() []byte { return nil } -func (m *SpaceReceipt) GetControlNodeIdentity() []byte { +func (m *SpaceReceipt) GetNetworkId() string { if m != nil { - return m.ControlNodeIdentity + return m.NetworkId } - return nil + return "" } func (m *SpaceReceipt) GetValidUntil() uint64 { @@ -929,65 +929,65 @@ func init() { } var fileDescriptor_d94f6f99586adae2 = []byte{ - // 918 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xcd, 0x6e, 0x23, 0x45, - 0x10, 0xf6, 0xd8, 0x4e, 0x82, 0xcb, 0x91, 0x77, 0xd2, 0x24, 0x61, 0x30, 0x66, 0xb0, 0x06, 0x58, - 0x4c, 0x40, 0xd9, 0x95, 0x17, 0x10, 0x88, 0x0b, 0x60, 0x16, 0x29, 0x08, 0x85, 0x68, 0x12, 0x83, - 0x80, 0x03, 0x9a, 0x9d, 0xa9, 0x24, 0xad, 0x38, 0xdd, 0x43, 0x77, 0x9b, 0x4d, 0x0e, 0x48, 0x3c, - 0x02, 0x27, 0x0e, 0x3c, 0x03, 0x6f, 0x00, 0x0f, 0xc0, 0x31, 0x47, 0x8e, 0x28, 0x91, 0x78, 0x0e, - 0xd4, 0xf3, 0xe7, 0x1e, 0x7b, 0x9c, 0x20, 0xed, 0xc5, 0x76, 0x7f, 0xf5, 0xfb, 0x55, 0x55, 0x57, - 0x1b, 0xde, 0x0d, 0x39, 0x17, 0x11, 0x65, 0x81, 0xe2, 0xe2, 0x81, 0xf1, 0x3b, 0x16, 0x5c, 0xf1, - 0x07, 0xc9, 0xa7, 0x34, 0xf1, 0xdd, 0x04, 0x22, 0x6d, 0x03, 0xf2, 0x7e, 0xb3, 0xc0, 0x3e, 0x8c, - 0x83, 0x10, 0x0f, 0xe9, 0x09, 0xf3, 0xf1, 0x87, 0x29, 0x4a, 0x45, 0x1c, 0x58, 0x93, 0x1a, 0xdb, - 0x8b, 0x1c, 0xab, 0x6f, 0x0d, 0x5a, 0x7e, 0x7e, 0x24, 0xdb, 0xb0, 0x7a, 0x8a, 0x41, 0x84, 0xc2, - 0xa9, 0xf7, 0xad, 0xc1, 0xba, 0x9f, 0x9d, 0x48, 0x1f, 0xda, 0x7c, 0x12, 0xed, 0x45, 0xc8, 0x14, - 0x55, 0x97, 0x4e, 0x23, 0x11, 0x9a, 0x10, 0x19, 0xc2, 0x26, 0xc3, 0xa7, 0xf9, 0x51, 0x47, 0x0b, - 0xd4, 0x54, 0xa0, 0xd3, 0x4c, 0x54, 0x2b, 0x65, 0x9e, 0x02, 0x92, 0xe6, 0xa6, 0x02, 0x35, 0x95, - 0x07, 0xc1, 0xe5, 0x84, 0x07, 0x11, 0x79, 0x08, 0xab, 0x32, 0x01, 0x92, 0xe4, 0x3a, 0x43, 0x67, - 0xd7, 0xe4, 0x68, 0x18, 0xf8, 0x99, 0x1e, 0x79, 0x1b, 0x36, 0x22, 0x9c, 0xa0, 0xa2, 0x9c, 0x1d, - 0xd1, 0x73, 0x94, 0x2a, 0x38, 0x8f, 0x13, 0x02, 0x0d, 0x7f, 0x51, 0xe0, 0x8d, 0x61, 0xc3, 0xa8, - 0x88, 0x8c, 0x39, 0x93, 0x48, 0x3e, 0x82, 0x35, 0x81, 0x21, 0xd2, 0x58, 0x25, 0x51, 0xdb, 0xc3, - 0xfb, 0x8b, 0x51, 0xfd, 0x54, 0xe1, 0x6b, 0xaa, 0x4e, 0x0b, 0x0e, 0x7e, 0x6e, 0xe6, 0x9d, 0xc1, - 0x8b, 0x4b, 0xb5, 0xc8, 0x43, 0x78, 0x5e, 0x1a, 0xc2, 0x8c, 0x6a, 0x12, 0x6a, 0xdd, 0xaf, 0x12, - 0x91, 0x1e, 0xb4, 0x64, 0x51, 0xc4, 0xb4, 0x19, 0x33, 0xc0, 0xfb, 0xd3, 0x82, 0x75, 0x33, 0xda, - 0xed, 0x2d, 0x8d, 0x11, 0xc5, 0x5e, 0x94, 0x78, 0x69, 0xf9, 0xd9, 0x89, 0x0c, 0xe0, 0x5e, 0x10, - 0x86, 0x7c, 0xca, 0xd4, 0x5c, 0x5b, 0xe7, 0x61, 0x9d, 0x7c, 0xc8, 0x99, 0x12, 0x7c, 0xb2, 0xcf, - 0x23, 0x2c, 0xb4, 0xd3, 0xce, 0x56, 0x89, 0x88, 0x0b, 0xf0, 0x63, 0x30, 0xa1, 0xd1, 0x98, 0x29, - 0x3a, 0x71, 0x56, 0xfa, 0xd6, 0xa0, 0xe9, 0x1b, 0x88, 0xf7, 0x1d, 0x6c, 0x7d, 0x46, 0x27, 0xf8, - 0x05, 0x3d, 0xa7, 0x6a, 0x74, 0x8a, 0xe1, 0x59, 0x3e, 0x99, 0x15, 0x49, 0x59, 0xd5, 0x49, 0x19, - 0x84, 0xeb, 0x25, 0xc2, 0xde, 0x2e, 0x6c, 0xcf, 0x3b, 0xcf, 0x9a, 0xbc, 0x09, 0x2b, 0x13, 0x8d, - 0x26, 0x3e, 0x9b, 0x7e, 0x7a, 0xf0, 0x1e, 0xc1, 0x0b, 0xc6, 0x50, 0x95, 0xd2, 0x59, 0x5a, 0x55, - 0x6f, 0x0c, 0xce, 0xa2, 0x51, 0x16, 0xe6, 0x03, 0x58, 0x8b, 0x8d, 0x06, 0xb7, 0x87, 0xaf, 0x2c, - 0x9b, 0xe0, 0xac, 0xd9, 0x7e, 0xae, 0xef, 0xfd, 0x6a, 0xcd, 0xf9, 0x0d, 0xd8, 0x09, 0xde, 0x7d, - 0x6d, 0x77, 0xc0, 0xce, 0xe7, 0x3c, 0x35, 0x29, 0xaa, 0xb2, 0x80, 0x93, 0x77, 0x60, 0xab, 0x8c, - 0xe5, 0xc3, 0x98, 0x76, 0xbf, 0x5a, 0xe8, 0x7d, 0x95, 0x4d, 0x77, 0x39, 0xaf, 0x67, 0x27, 0xfc, - 0x21, 0xbc, 0xb4, 0x8f, 0xea, 0x29, 0x17, 0x67, 0x23, 0xce, 0x8e, 0xe9, 0xc9, 0x54, 0x04, 0x3a, - 0x78, 0x4e, 0xb9, 0x07, 0xad, 0x70, 0x2a, 0x04, 0xea, 0xc6, 0x67, 0xa4, 0x67, 0x80, 0xf7, 0x87, - 0x05, 0xbd, 0x6a, 0xeb, 0x2c, 0xb1, 0x01, 0xdc, 0x0b, 0x4d, 0x41, 0xe1, 0x64, 0x1e, 0xd6, 0x81, - 0x58, 0xea, 0xa9, 0x28, 0xdd, 0x0c, 0x20, 0x6f, 0xc0, 0x0a, 0xe3, 0x11, 0x4a, 0xa7, 0xd1, 0x6f, - 0x0c, 0xda, 0xc3, 0x8d, 0x12, 0x3d, 0x3d, 0xf9, 0x7e, 0x2a, 0xd7, 0x8d, 0x08, 0x05, 0x06, 0xf9, - 0xc2, 0x19, 0x33, 0x7a, 0x91, 0xdc, 0x93, 0xa6, 0xbf, 0x80, 0x7b, 0x14, 0x9a, 0xda, 0xd4, 0xb8, - 0xa0, 0x56, 0xe9, 0x82, 0xf6, 0xa0, 0x15, 0x44, 0x91, 0x40, 0x29, 0x51, 0x3a, 0xf5, 0x7e, 0x43, - 0xa7, 0x54, 0x00, 0xe4, 0x2d, 0x58, 0x51, 0x97, 0x71, 0x96, 0x52, 0x67, 0xb8, 0xb5, 0x90, 0xd2, - 0xd1, 0x65, 0x8c, 0x7e, 0xaa, 0xb3, 0xf3, 0xb3, 0x05, 0xf0, 0x58, 0x08, 0x2e, 0x46, 0x49, 0x96, - 0x1d, 0x80, 0x31, 0xc3, 0x8b, 0x18, 0x43, 0x85, 0x91, 0x5d, 0x23, 0x76, 0xb6, 0x4c, 0x3e, 0xd5, - 0xad, 0xc7, 0xc8, 0xb6, 0x88, 0x03, 0x9b, 0x33, 0x84, 0x72, 0x76, 0x80, 0x2c, 0xa2, 0xec, 0xc4, - 0xae, 0x17, 0xba, 0x23, 0x4d, 0x07, 0x23, 0xbb, 0x41, 0x08, 0x74, 0x12, 0x64, 0x9f, 0xab, 0xc7, - 0x17, 0x54, 0x2a, 0x69, 0x37, 0x89, 0x0d, 0xed, 0x24, 0xde, 0x97, 0xc7, 0xc7, 0x12, 0x95, 0xfd, - 0x7b, 0x7d, 0xe7, 0x27, 0x68, 0x1b, 0x73, 0x40, 0xb6, 0x4b, 0xab, 0x3f, 0x77, 0x56, 0x23, 0x2e, - 0x74, 0xcd, 0x71, 0x49, 0xc3, 0xe6, 0x59, 0xd8, 0xd6, 0x9c, 0x3c, 0x17, 0x1c, 0xaa, 0x40, 0x68, - 0xfb, 0xfa, 0x9c, 0xdf, 0x9c, 0x50, 0x63, 0xe7, 0x7d, 0x78, 0x2e, 0x2f, 0x0a, 0x69, 0xc3, 0xda, - 0x91, 0x40, 0xfc, 0xf8, 0x60, 0xcf, 0xae, 0xe9, 0x83, 0xde, 0x16, 0xfa, 0x60, 0x69, 0x2a, 0xa3, - 0x59, 0x19, 0x35, 0x56, 0x1f, 0xfe, 0xdb, 0x80, 0xb6, 0x01, 0x92, 0xcf, 0xa1, 0x55, 0x3c, 0x1f, - 0xe4, 0xe5, 0x8a, 0x41, 0x9f, 0x3d, 0xb4, 0x5d, 0x77, 0x99, 0x38, 0x9b, 0xcf, 0x6f, 0xa0, 0x53, - 0x5e, 0x55, 0xc4, 0x2b, 0x59, 0x54, 0x2e, 0xc9, 0xee, 0xab, 0xb7, 0xea, 0x64, 0xae, 0xbf, 0xcf, - 0xdf, 0xfd, 0xd9, 0x82, 0x22, 0xaf, 0x2d, 0xbb, 0x96, 0x25, 0xf7, 0xaf, 0xdf, 0xa1, 0x95, 0x05, - 0x78, 0x92, 0x3f, 0xa3, 0xc6, 0x46, 0x20, 0xb7, 0xd8, 0x1a, 0x9b, 0xac, 0x7b, 0xff, 0x2e, 0xb5, - 0x2c, 0xc6, 0x19, 0x6c, 0x56, 0xdd, 0x6f, 0x32, 0x28, 0x4f, 0xfb, 0xf2, 0x05, 0xd2, 0x7d, 0xf3, - 0x7f, 0x68, 0xa6, 0xc1, 0x3e, 0x79, 0xef, 0xaf, 0x6b, 0xd7, 0xba, 0xba, 0x76, 0xad, 0x7f, 0xae, - 0x5d, 0xeb, 0x97, 0x1b, 0xb7, 0x76, 0x75, 0xe3, 0xd6, 0xfe, 0xbe, 0x71, 0x6b, 0xdf, 0xf6, 0x6e, - 0xfb, 0x23, 0xf6, 0x64, 0x35, 0xf9, 0x7a, 0xf4, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x48, 0x5e, - 0x22, 0xba, 0xaf, 0x09, 0x00, 0x00, + // 914 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xcf, 0x6e, 0xeb, 0x44, + 0x17, 0x8f, 0x93, 0xb4, 0xfd, 0x72, 0x5c, 0xe5, 0xba, 0xf3, 0xb5, 0xc5, 0x84, 0x60, 0x22, 0x03, + 0x97, 0x50, 0x50, 0xef, 0x55, 0x2e, 0x20, 0x10, 0x1b, 0x20, 0x5c, 0xa4, 0x20, 0x54, 0x2a, 0xb7, + 0x01, 0x01, 0x0b, 0xe4, 0xeb, 0x39, 0x6d, 0x47, 0x4d, 0x6d, 0x33, 0x33, 0xe1, 0xb6, 0x0b, 0x24, + 0x1e, 0x81, 0x15, 0x0b, 0x9e, 0x80, 0x05, 0x6f, 0xc0, 0x0b, 0xb0, 0xbc, 0x4b, 0x96, 0xa8, 0x95, + 0x78, 0x0e, 0x34, 0x63, 0x3b, 0x19, 0x27, 0x4e, 0x8b, 0xc4, 0x26, 0xc9, 0xf9, 0x9d, 0xff, 0x73, + 0x7e, 0x73, 0x26, 0xf0, 0x76, 0x94, 0x24, 0x9c, 0xb2, 0x38, 0x94, 0x09, 0x7f, 0x60, 0xfc, 0x4e, + 0x79, 0x22, 0x93, 0x07, 0xfa, 0x53, 0x98, 0xf8, 0xbe, 0x86, 0x88, 0x6d, 0x40, 0xfe, 0x2f, 0x16, + 0x38, 0x47, 0x69, 0x18, 0xe1, 0x11, 0x3b, 0x8d, 0x03, 0xfc, 0x6e, 0x8a, 0x42, 0x12, 0x17, 0x36, + 0x84, 0xc2, 0x46, 0xd4, 0xb5, 0x7a, 0x56, 0xbf, 0x15, 0x14, 0x22, 0xd9, 0x85, 0xf5, 0x33, 0x0c, + 0x29, 0x72, 0xb7, 0xde, 0xb3, 0xfa, 0x9b, 0x41, 0x2e, 0x91, 0x1e, 0xd8, 0xc9, 0x84, 0x8e, 0x28, + 0xc6, 0x92, 0xc9, 0x2b, 0xb7, 0xa1, 0x95, 0x26, 0x44, 0x06, 0xb0, 0x1d, 0xe3, 0xd3, 0x42, 0x54, + 0xd9, 0x42, 0x39, 0xe5, 0xe8, 0x36, 0xb5, 0x69, 0xa5, 0xce, 0x97, 0x40, 0xb2, 0xda, 0x64, 0x28, + 0xa7, 0xe2, 0x30, 0xbc, 0x9a, 0x24, 0x21, 0x25, 0x0f, 0x61, 0x5d, 0x68, 0x40, 0x17, 0xd7, 0x1e, + 0xb8, 0xfb, 0x66, 0x8f, 0x86, 0x43, 0x90, 0xdb, 0x91, 0x37, 0x61, 0x8b, 0xe2, 0x04, 0x25, 0x4b, + 0xe2, 0x63, 0x76, 0x81, 0x42, 0x86, 0x17, 0xa9, 0x6e, 0xa0, 0x11, 0x2c, 0x2b, 0xfc, 0x31, 0x6c, + 0x19, 0x27, 0x22, 0xd2, 0x24, 0x16, 0x48, 0x3e, 0x80, 0x0d, 0x8e, 0x11, 0xb2, 0x54, 0xea, 0xac, + 0xf6, 0xe0, 0xfe, 0x72, 0xd6, 0x20, 0x33, 0xf8, 0x92, 0xc9, 0xb3, 0x59, 0x0f, 0x41, 0xe1, 0xe6, + 0x9f, 0xc3, 0xf3, 0x2b, 0xad, 0xc8, 0x43, 0xf8, 0xbf, 0x30, 0x94, 0x79, 0xab, 0x3a, 0xd5, 0x66, + 0x50, 0xa5, 0x22, 0x5d, 0x68, 0x89, 0xd9, 0x21, 0x66, 0xc3, 0x98, 0x03, 0xfe, 0xaf, 0x16, 0x6c, + 0x9a, 0xd9, 0x6e, 0x1f, 0x69, 0x8a, 0xc8, 0x47, 0x54, 0x47, 0x69, 0x05, 0xb9, 0x44, 0xfa, 0x70, + 0x2f, 0x8c, 0xa2, 0x64, 0x1a, 0xcb, 0x85, 0xb1, 0x2e, 0xc2, 0xaa, 0x94, 0x18, 0xe5, 0xd3, 0x84, + 0x9f, 0x8f, 0xa8, 0x9e, 0x67, 0x2b, 0x98, 0x03, 0xc4, 0x03, 0xf8, 0x3e, 0x9c, 0x30, 0x3a, 0x8e, + 0x25, 0x9b, 0xb8, 0x6b, 0x3d, 0xab, 0xdf, 0x0c, 0x0c, 0xc4, 0xff, 0x06, 0x76, 0x3e, 0x61, 0x13, + 0xfc, 0x8c, 0x5d, 0x30, 0x39, 0x3c, 0xc3, 0xe8, 0xbc, 0x60, 0x61, 0x45, 0x01, 0x56, 0x75, 0x01, + 0x46, 0x73, 0xf5, 0x52, 0x73, 0xfe, 0x3e, 0xec, 0x2e, 0x06, 0xcf, 0x07, 0xba, 0x0d, 0x6b, 0x13, + 0x85, 0xea, 0x98, 0xcd, 0x20, 0x13, 0xfc, 0x47, 0xf0, 0x9c, 0x41, 0xa0, 0x52, 0x39, 0x2b, 0x4f, + 0xd0, 0x1f, 0x83, 0xbb, 0xec, 0x94, 0xa7, 0x79, 0x0f, 0x36, 0x52, 0x63, 0x98, 0xf6, 0xe0, 0xa5, + 0x55, 0x6c, 0xcd, 0x07, 0x1b, 0x14, 0xf6, 0xfe, 0xcf, 0xd6, 0x42, 0xdc, 0x30, 0x3e, 0xc5, 0xbb, + 0xaf, 0xe8, 0x1e, 0x38, 0x05, 0xa7, 0x33, 0x97, 0xd9, 0xa9, 0x2c, 0xe1, 0xe4, 0x2d, 0xd8, 0x29, + 0x63, 0x05, 0xf1, 0xb2, 0x49, 0x57, 0x2b, 0xfd, 0x2f, 0x72, 0x26, 0x97, 0xeb, 0xfa, 0xef, 0x0d, + 0xbf, 0x0f, 0x2f, 0x1c, 0x64, 0xb4, 0x19, 0x26, 0xf1, 0x09, 0x3b, 0x9d, 0xf2, 0x50, 0x25, 0x2f, + 0x5a, 0xee, 0x42, 0x2b, 0x9a, 0x72, 0x8e, 0x6a, 0xf0, 0x79, 0xd3, 0x73, 0xc0, 0xff, 0xdd, 0x82, + 0x6e, 0xb5, 0x77, 0x5e, 0x58, 0x1f, 0xee, 0x45, 0xa6, 0x62, 0x16, 0x64, 0x11, 0x2e, 0xf3, 0xb9, + 0xbe, 0xc8, 0xe7, 0xd7, 0x60, 0x2d, 0x4e, 0x28, 0x0a, 0xb7, 0xd1, 0x6b, 0xf4, 0xed, 0xc1, 0x56, + 0xa9, 0xbd, 0x83, 0x84, 0x62, 0x90, 0xe9, 0xd5, 0x20, 0x22, 0x8e, 0x61, 0xb1, 0x5c, 0xc6, 0x31, + 0xbb, 0xd4, 0xb7, 0xa3, 0x19, 0x2c, 0xe1, 0x3e, 0x83, 0xa6, 0x72, 0x35, 0x2e, 0xa3, 0x55, 0xba, + 0x8c, 0x5d, 0x68, 0x85, 0x94, 0x72, 0x14, 0x02, 0x85, 0x5b, 0xef, 0x35, 0x54, 0x49, 0x33, 0x80, + 0xbc, 0x01, 0x6b, 0xf2, 0x2a, 0xcd, 0x4b, 0x6a, 0x0f, 0x76, 0x96, 0x4a, 0x3a, 0xbe, 0x4a, 0x31, + 0xc8, 0x6c, 0xf6, 0x7e, 0xb4, 0x00, 0x1e, 0x73, 0x9e, 0xf0, 0xa1, 0xae, 0xb2, 0x0d, 0x30, 0x8e, + 0xf1, 0x32, 0xc5, 0x48, 0x22, 0x75, 0x6a, 0xc4, 0xc9, 0x17, 0xc7, 0xc7, 0x6a, 0xf4, 0x48, 0x1d, + 0x8b, 0xb8, 0xb0, 0x3d, 0x47, 0x58, 0x12, 0x1f, 0x62, 0x4c, 0x59, 0x7c, 0xea, 0xd4, 0x67, 0xb6, + 0x43, 0xd5, 0x0e, 0x52, 0xa7, 0x41, 0x08, 0xb4, 0x35, 0x72, 0x90, 0xc8, 0xc7, 0x97, 0x4c, 0x48, + 0xe1, 0x34, 0x89, 0x03, 0xb6, 0xce, 0xf7, 0xf9, 0xc9, 0x89, 0x40, 0xe9, 0xfc, 0x56, 0xdf, 0xfb, + 0x01, 0x6c, 0x83, 0x07, 0x64, 0xb7, 0xb4, 0xe6, 0x8b, 0x60, 0x35, 0xe2, 0x41, 0xc7, 0xa4, 0x4b, + 0x96, 0xb6, 0xa8, 0xc2, 0xb1, 0x16, 0xf4, 0x85, 0xe2, 0x48, 0x86, 0x5c, 0xf9, 0xd7, 0x17, 0xe2, + 0x16, 0x0d, 0x35, 0xf6, 0xde, 0x85, 0xff, 0x15, 0x87, 0x42, 0x6c, 0xd8, 0x38, 0xe6, 0x88, 0x1f, + 0x1e, 0x8e, 0x9c, 0x9a, 0x12, 0xd4, 0xb6, 0x50, 0x82, 0xa5, 0x5a, 0x19, 0xce, 0x8f, 0x51, 0x61, + 0xf5, 0xc1, 0xdf, 0x0d, 0xb0, 0x0d, 0x90, 0x7c, 0x0a, 0xad, 0xd9, 0x53, 0x41, 0x5e, 0xac, 0x20, + 0xfa, 0xfc, 0x51, 0xed, 0x78, 0xab, 0xd4, 0x39, 0x3f, 0xbf, 0x82, 0x76, 0x79, 0x55, 0x11, 0xbf, + 0xe4, 0x51, 0xb9, 0x24, 0x3b, 0x2f, 0xdf, 0x6a, 0x93, 0x87, 0xfe, 0xb6, 0x78, 0xe3, 0xe7, 0x0b, + 0x8a, 0xbc, 0xb2, 0xea, 0x5a, 0x96, 0xc2, 0xbf, 0x7a, 0x87, 0x55, 0x9e, 0xe0, 0x49, 0xf1, 0x64, + 0x1a, 0x1b, 0x81, 0xdc, 0xe2, 0x6b, 0x6c, 0xb2, 0xce, 0xfd, 0xbb, 0xcc, 0xf2, 0x1c, 0xe7, 0xb0, + 0x5d, 0x75, 0xbf, 0x49, 0xbf, 0xcc, 0xf6, 0xd5, 0x0b, 0xa4, 0xf3, 0xfa, 0xbf, 0xb0, 0xcc, 0x92, + 0x7d, 0xf4, 0xce, 0x1f, 0xd7, 0x9e, 0xf5, 0xec, 0xda, 0xb3, 0xfe, 0xba, 0xf6, 0xac, 0x9f, 0x6e, + 0xbc, 0xda, 0xb3, 0x1b, 0xaf, 0xf6, 0xe7, 0x8d, 0x57, 0xfb, 0xba, 0x7b, 0xdb, 0x9f, 0xae, 0x27, + 0xeb, 0xfa, 0xeb, 0xd1, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x0f, 0xcc, 0xe8, 0x2d, 0x9b, 0x09, + 0x00, 0x00, } func (m *SpaceSignRequest) Marshal() (dAtA []byte, err error) { @@ -1171,10 +1171,10 @@ func (m *SpaceReceipt) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x28 } - if len(m.ControlNodeIdentity) > 0 { - i -= len(m.ControlNodeIdentity) - copy(dAtA[i:], m.ControlNodeIdentity) - i = encodeVarintCoordinator(dAtA, i, uint64(len(m.ControlNodeIdentity))) + if len(m.NetworkId) > 0 { + i -= len(m.NetworkId) + copy(dAtA[i:], m.NetworkId) + i = encodeVarintCoordinator(dAtA, i, uint64(len(m.NetworkId))) i-- dAtA[i] = 0x22 } @@ -1653,7 +1653,7 @@ func (m *SpaceReceipt) Size() (n int) { if l > 0 { n += 1 + l + sovCoordinator(uint64(l)) } - l = len(m.ControlNodeIdentity) + l = len(m.NetworkId) if l > 0 { n += 1 + l + sovCoordinator(uint64(l)) } @@ -2428,9 +2428,9 @@ func (m *SpaceReceipt) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 4: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ControlNodeIdentity", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field NetworkId", wireType) } - var byteLen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowCoordinator @@ -2440,25 +2440,23 @@ func (m *SpaceReceipt) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if byteLen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthCoordinator } - postIndex := iNdEx + byteLen + postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthCoordinator } if postIndex > l { return io.ErrUnexpectedEOF } - m.ControlNodeIdentity = append(m.ControlNodeIdentity[:0], dAtA[iNdEx:postIndex]...) - if m.ControlNodeIdentity == nil { - m.ControlNodeIdentity = []byte{} - } + m.NetworkId = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 5: if wireType != 0 { diff --git a/coordinator/coordinatorproto/protos/coordinator.proto b/coordinator/coordinatorproto/protos/coordinator.proto index 804957a1..c6767eab 100644 --- a/coordinator/coordinatorproto/protos/coordinator.proto +++ b/coordinator/coordinatorproto/protos/coordinator.proto @@ -67,15 +67,15 @@ message SpaceReceiptWithSignature { // SpaceReceipt contains permission to SpacePush operation message SpaceReceipt { - // spaceId + // SpaceId is the identifier of space string spaceId = 1; - // peerId of receipt requester + // PeerId of receipt requester string peerId = 2; - // identity of space owner + // AccountIdentity is an identity of a space owner bytes accountIdentity = 3; - // identity of control node - bytes controlNodeIdentity = 4; - // unix-timestamp with a deadline time of receipt validity + // NetworkId is the id of a network where the receipt is issued + string networkId = 4; + // ValidUntil is a unix-timestamp with a deadline time of receipt validity uint64 validUntil = 5; } diff --git a/coordinator/coordinatorproto/receipt.go b/coordinator/coordinatorproto/receipt.go index 67287d9d..d011038f 100644 --- a/coordinator/coordinatorproto/receipt.go +++ b/coordinator/coordinatorproto/receipt.go @@ -3,43 +3,38 @@ package coordinatorproto import ( "bytes" "errors" - "fmt" "github.com/anytypeio/any-sync/util/crypto" + "github.com/anytypeio/any-sync/util/strkey" "github.com/gogo/protobuf/proto" - "golang.org/x/exp/slices" "time" ) var ( errReceiptSignatureIncorrect = errors.New("receipt signature is incorrect") - errNoSuchCoordinatorNode = errors.New("no such coordinator node") + errNetworkIsIncorrect = errors.New("network is incorrect") errReceiptSpaceIdIncorrect = errors.New("receipt space id is incorrect") errReceiptPeerIdIncorrect = errors.New("receipt peer id is incorrect") errReceiptAccountIncorrect = errors.New("receipt account is incorrect") errReceiptExpired = errors.New("receipt is expired") ) -func PrepareSpaceReceipt(spaceId, peerId string, validPeriod time.Duration, accountPubKey crypto.PubKey, nodeKey crypto.PrivKey) (signedReceipt *SpaceReceiptWithSignature, err error) { +func PrepareSpaceReceipt(spaceId, peerId string, validPeriod time.Duration, accountPubKey crypto.PubKey, networkKey crypto.PrivKey) (signedReceipt *SpaceReceiptWithSignature, err error) { marshalledAccount, err := accountPubKey.Marshall() if err != nil { return } - marshalledNode, err := nodeKey.GetPublic().Marshall() - if err != nil { - return - } receipt := &SpaceReceipt{ - SpaceId: spaceId, - PeerId: peerId, - AccountIdentity: marshalledAccount, - ControlNodeIdentity: marshalledNode, - ValidUntil: uint64(time.Now().Add(validPeriod).Unix()), + SpaceId: spaceId, + PeerId: peerId, + AccountIdentity: marshalledAccount, + NetworkId: networkKey.GetPublic().Network(), + ValidUntil: uint64(time.Now().Add(validPeriod).Unix()), } receiptData, err := receipt.Marshal() if err != nil { return } - sign, err := nodeKey.Sign(receiptData) + sign, err := networkKey.Sign(receiptData) if err != nil { return } @@ -49,7 +44,7 @@ func PrepareSpaceReceipt(spaceId, peerId string, validPeriod time.Duration, acco }, nil } -func CheckReceipt(peerId, spaceId string, accountIdentity []byte, coordinators []string, receipt *SpaceReceiptWithSignature) (err error) { +func CheckReceipt(peerId, spaceId string, accountIdentity []byte, networkId string, receipt *SpaceReceiptWithSignature) (err error) { payload := &SpaceReceipt{} err = proto.Unmarshal(receipt.GetSpaceReceiptPayload(), payload) if err != nil { @@ -72,9 +67,9 @@ func CheckReceipt(peerId, spaceId string, accountIdentity []byte, coordinators [ if !bytes.Equal(protoRaw.Storage(), accountRaw.Storage()) { return errReceiptAccountIncorrect } - err = checkCoordinator( - coordinators, - payload.ControlNodeIdentity, + err = checkNetwork( + networkId, + payload.NetworkId, receipt.GetSpaceReceiptPayload(), receipt.GetSignature()) if err != nil { @@ -86,16 +81,19 @@ func CheckReceipt(peerId, spaceId string, accountIdentity []byte, coordinators [ return } -func checkCoordinator(coordinators []string, identity []byte, payload, signature []byte) (err error) { - coordinatorKey, err := crypto.UnmarshalEd25519PublicKeyProto(identity) +func checkNetwork(networkId, payloadNetworkId string, payload, signature []byte) (err error) { + if networkId != payloadNetworkId { + return errNetworkIsIncorrect + } + networkIdentity, err := strkey.Decode(strkey.NetworkAddressVersionByte, networkId) if err != nil { return } - receiptCoordinator := coordinatorKey.PeerId() - if !slices.Contains(coordinators, coordinatorKey.PeerId()) { - return fmt.Errorf("got coordinator %s: %w", receiptCoordinator, errNoSuchCoordinatorNode) + networkKey, err := crypto.UnmarshalEd25519PublicKey(networkIdentity) + if err != nil { + return } - res, err := coordinatorKey.Verify(payload, signature) + res, err := networkKey.Verify(payload, signature) if err != nil { return } diff --git a/coordinator/coordinatorproto/receipt_test.go b/coordinator/coordinatorproto/receipt_test.go index fc2bc200..28bd9dc3 100644 --- a/coordinator/coordinatorproto/receipt_test.go +++ b/coordinator/coordinatorproto/receipt_test.go @@ -3,7 +3,6 @@ package coordinatorproto import ( "context" "crypto/rand" - "errors" "github.com/anytypeio/any-sync/util/crypto" "github.com/gogo/protobuf/proto" "github.com/stretchr/testify/require" @@ -12,7 +11,7 @@ import ( ) type fixture struct { - coordinatorKey crypto.PrivKey + networkKey crypto.PrivKey accountKey crypto.PubKey accountIdentity []byte ctx context.Context @@ -23,7 +22,7 @@ type fixture struct { } func newFixture(t *testing.T) *fixture { - coordinatorKey, _, err := crypto.GenerateEd25519Key(rand.Reader) + networkKey, _, err := crypto.GenerateEd25519Key(rand.Reader) require.NoError(t, err) accountKey, _, err := crypto.GenerateEd25519Key(rand.Reader) require.NoError(t, err) @@ -33,14 +32,14 @@ func newFixture(t *testing.T) *fixture { spaceId: "spaceId", peerId: "peerId", accountIdentity: accountKeyProto, - coordinatorKey: coordinatorKey, + networkKey: networkKey, accountKey: accountKey.GetPublic(), } } func (fx *fixture) prepareReceipt(t *testing.T, validPeriod time.Duration) { var err error - fx.signedReceipt, err = PrepareSpaceReceipt(fx.spaceId, fx.peerId, validPeriod, fx.accountKey, fx.coordinatorKey) + fx.signedReceipt, err = PrepareSpaceReceipt(fx.spaceId, fx.peerId, validPeriod, fx.accountKey, fx.networkKey) require.NoError(t, err) fx.originalReceipt = &SpaceReceipt{} err = proto.Unmarshal(fx.signedReceipt.SpaceReceiptPayload, fx.originalReceipt) @@ -52,7 +51,7 @@ func (fx *fixture) updateReceipt(t *testing.T, update func(t *testing.T, receipt update(t, fx.originalReceipt) marshalled, err := proto.Marshal(fx.originalReceipt) require.NoError(t, err) - signature, err := fx.coordinatorKey.Sign(marshalled) + signature, err := fx.networkKey.Sign(marshalled) require.NoError(t, err) fx.signedReceipt = &SpaceReceiptWithSignature{ SpaceReceiptPayload: marshalled, @@ -63,52 +62,50 @@ func (fx *fixture) updateReceipt(t *testing.T, update func(t *testing.T, receipt func TestReceiptValid(t *testing.T) { fx := newFixture(t) fx.prepareReceipt(t, time.Second) - err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, fx.networkKey.GetPublic().Network(), fx.signedReceipt) require.NoError(t, err) } func TestReceiptIncorrectSpaceId(t *testing.T) { fx := newFixture(t) fx.prepareReceipt(t, time.Second) - err := CheckReceipt(fx.peerId, "otherId", fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + err := CheckReceipt(fx.peerId, "otherId", fx.accountIdentity, fx.networkKey.GetPublic().Network(), fx.signedReceipt) require.Error(t, errReceiptSpaceIdIncorrect, err) } func TestReceiptIncorrectPeerId(t *testing.T) { fx := newFixture(t) fx.prepareReceipt(t, time.Second) - err := CheckReceipt("otherId", fx.spaceId, fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + err := CheckReceipt("otherId", fx.spaceId, fx.accountIdentity, fx.networkKey.GetPublic().Network(), fx.signedReceipt) require.Error(t, errReceiptPeerIdIncorrect, err) } func TestReceiptIncorrectAccountIdentity(t *testing.T) { fx := newFixture(t) fx.prepareReceipt(t, time.Second) - err := CheckReceipt(fx.peerId, fx.spaceId, []byte("some identity"), []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + err := CheckReceipt(fx.peerId, fx.spaceId, []byte("some identity"), fx.networkKey.GetPublic().Network(), fx.signedReceipt) require.Error(t, errReceiptAccountIncorrect, err) } -func TestReceiptIncorrectCoordinatorKey(t *testing.T) { +func TestReceiptIncorrectNetworkId(t *testing.T) { fx := newFixture(t) fx.prepareReceipt(t, time.Second) - t.Run("random key payload", func(t *testing.T) { + t.Run("random network id", func(t *testing.T) { fx.updateReceipt(t, func(t *testing.T, receipt *SpaceReceipt) { - receipt.ControlNodeIdentity = []byte("some random stuff") + receipt.NetworkId = "some random network id" }) - err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, fx.networkKey.GetPublic().Network(), fx.signedReceipt) require.Error(t, err) }) t.Run("random incorrect key", func(t *testing.T) { fx.updateReceipt(t, func(t *testing.T, receipt *SpaceReceipt) { randomKey, _, err := crypto.GenerateEd25519Key(rand.Reader) require.NoError(t, err) - keyBytes, err := randomKey.GetPublic().Marshall() - require.NoError(t, err) - receipt.ControlNodeIdentity = keyBytes + receipt.NetworkId = randomKey.GetPublic().Network() }) - err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) - require.True(t, errors.Is(err, errNoSuchCoordinatorNode)) + err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, fx.networkKey.GetPublic().Network(), fx.signedReceipt) + require.Error(t, errNetworkIsIncorrect, err) }) } @@ -116,7 +113,7 @@ func TestReceiptIncorrectSignature(t *testing.T) { fx := newFixture(t) fx.prepareReceipt(t, time.Second) fx.signedReceipt.Signature = []byte("random sig") - err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, fx.networkKey.GetPublic().Network(), fx.signedReceipt) require.Error(t, errReceiptSignatureIncorrect, err) } @@ -126,6 +123,6 @@ func TestReceiptExpired(t *testing.T) { fx.updateReceipt(t, func(t *testing.T, receipt *SpaceReceipt) { receipt.ValidUntil = uint64(time.Now().Add(-time.Second).Unix()) }) - err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, []string{fx.coordinatorKey.GetPublic().PeerId()}, fx.signedReceipt) + err := CheckReceipt(fx.peerId, fx.spaceId, fx.accountIdentity, fx.networkKey.GetPublic().Network(), fx.signedReceipt) require.Error(t, errReceiptExpired, err) } From 46de85a7f824379972c01f87c443784c154bedfc Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 25 Apr 2023 20:55:00 +0200 Subject: [PATCH 44/45] Check peerId when dialling --- net/dialer/dialer.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/net/dialer/dialer.go b/net/dialer/dialer.go index 00cdb996..4d85005b 100644 --- a/net/dialer/dialer.go +++ b/net/dialer/dialer.go @@ -24,7 +24,10 @@ import ( const CName = "common.net.dialer" -var ErrArrdsNotFound = errors.New("addrs for peer not found") +var ( + ErrAddrsNotFound = errors.New("addrs for peer not found") + ErrPeerIdIsUnexpected = errors.New("expected to connect with other peer id") +) var log = logger.NewNamed(CName) @@ -80,7 +83,7 @@ func (d *dialer) getPeerAddrs(peerId string) ([]string, error) { } addrs, ok := d.peerAddrs[peerId] if !ok || len(addrs) == 0 { - return nil, ErrArrdsNotFound + return nil, ErrAddrsNotFound } return addrs, nil } @@ -103,7 +106,7 @@ func (d *dialer) Dial(ctx context.Context, peerId string) (p peer.Peer, err erro ) log.InfoCtx(ctx, "dial", zap.String("peerId", peerId), zap.Strings("addrs", addrs)) for _, addr := range addrs { - conn, sc, err = d.handshake(ctx, addr) + conn, sc, err = d.handshake(ctx, addr, peerId) if err != nil { log.InfoCtx(ctx, "can't connect to host", zap.String("addr", addr), zap.Error(err)) } else { @@ -116,7 +119,7 @@ func (d *dialer) Dial(ctx context.Context, peerId string) (p peer.Peer, err erro return peer.NewPeer(sc, conn), nil } -func (d *dialer) handshake(ctx context.Context, addr string) (conn drpc.Conn, sc sec.SecureConn, err error) { +func (d *dialer) handshake(ctx context.Context, addr, peerId string) (conn drpc.Conn, sc sec.SecureConn, err error) { st := time.Now() // TODO: move dial timeout to config tcpConn, err := net.DialTimeout("tcp", addr, time.Second*3) @@ -129,6 +132,9 @@ func (d *dialer) handshake(ctx context.Context, addr string) (conn drpc.Conn, sc if err != nil { return nil, nil, fmt.Errorf("tls handshaeke error: %v; since start: %v", err, time.Since(st)) } + if peerId != sc.RemotePeer().String() { + return nil, nil, ErrPeerIdIsUnexpected + } log.Info("connected with remote host", zap.String("serverPeer", sc.RemotePeer().String()), zap.String("addr", addr)) conn = drpcconn.NewWithOptions(sc, drpcconn.Options{Manager: drpcmanager.Options{ Reader: drpcwire.ReaderOptions{MaximumBufferSize: d.config.Stream.MaxMsgSizeMb * (1 << 20)}, From e9c6f9599103e39bd1d9449194e1541c44a7b4d5 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 25 Apr 2023 21:41:18 +0200 Subject: [PATCH 45/45] Add reference to linear task in periodicsync todo --- util/periodicsync/periodicsync.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/periodicsync/periodicsync.go b/util/periodicsync/periodicsync.go index abe998c0..8838faad 100644 --- a/util/periodicsync/periodicsync.go +++ b/util/periodicsync/periodicsync.go @@ -16,7 +16,8 @@ type PeriodicSync interface { type SyncerFunc func(ctx context.Context) error func NewPeriodicSync(periodSeconds int, timeout time.Duration, caller SyncerFunc, l logger.CtxLogger) PeriodicSync { - // TODO: rename to PeriodicCall + // TODO: rename to PeriodicCall (including folders) and do PRs in all repos where we are using this + // https://linear.app/anytype/issue/GO-1241/change-periodicsync-component-to-periodiccall ctx, cancel := context.WithCancel(context.Background()) ctx = logger.CtxWithFields(ctx, zap.String("rootOp", "periodicCall")) return &periodicCall{