From 11e57a43f7e96f89c0e819310f0379b26ff0727d Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 10 Nov 2022 19:44:33 +0100 Subject: [PATCH 01/27] Create space with settings --- client/clientspace/clientcache/treecache.go | 7 +- client/document/textdocument/textdocument.go | 28 +- client/storage/keys.go | 14 +- client/storage/spacestorage.go | 66 ++- client/storage/spacestorage_test.go | 4 +- common/commonspace/deletionservice/service.go | 28 + common/commonspace/payloads.go | 4 +- common/commonspace/service.go | 4 +- .../spacesyncproto/protos/spacesync.proto | 10 +- .../spacesyncproto/spacesync.pb.go | 524 ++++++++++++------ common/commonspace/storage/storage.go | 12 +- node/storage/keys.go | 11 +- node/storage/spacestorage.go | 82 ++- node/storage/spacestorage_test.go | 4 +- 14 files changed, 546 insertions(+), 252 deletions(-) create mode 100644 common/commonspace/deletionservice/service.go diff --git a/client/clientspace/clientcache/treecache.go b/client/clientspace/clientcache/treecache.go index 5422b91c..ae3163cb 100644 --- a/client/clientspace/clientcache/treecache.go +++ b/client/clientspace/clientcache/treecache.go @@ -99,10 +99,5 @@ func (c *treeCache) GetDocument(ctx context.Context, spaceId, id string) (doc te } func (c *treeCache) GetTree(ctx context.Context, spaceId, id string) (tr tree.ObjectTree, err error) { - doc, err := c.GetDocument(ctx, spaceId, id) - if err != nil { - return - } - tr = doc.Tree() - return + return c.GetDocument(ctx, spaceId, id) } diff --git a/client/document/textdocument/textdocument.go b/client/document/textdocument/textdocument.go index 7b754670..954160d9 100644 --- a/client/document/textdocument/textdocument.go +++ b/client/document/textdocument/textdocument.go @@ -11,7 +11,7 @@ import ( ) type TextDocument interface { - Tree() tree.ObjectTree + tree.ObjectTree AddText(text string) error Text() (string, error) TreeDump() string @@ -19,7 +19,7 @@ type TextDocument interface { } type textDocument struct { - objTree tree.ObjectTree + tree.ObjectTree account account.Service } @@ -39,8 +39,8 @@ func CreateTextDocument( } return &textDocument{ - objTree: t, - account: account, + ObjectTree: t, + account: account, }, nil } @@ -50,15 +50,11 @@ func NewTextDocument(ctx context.Context, space commonspace.Space, id string, li return } return &textDocument{ - objTree: t, - account: account, + ObjectTree: t, + account: account, }, nil } -func (t *textDocument) Tree() tree.ObjectTree { - return t.objTree -} - func (t *textDocument) AddText(text string) (err error) { content := &testchanges.TextContent_TextAppend{ TextAppend: &testchanges.TextAppend{Text: text}, @@ -73,9 +69,9 @@ func (t *textDocument) AddText(text string) (err error) { if err != nil { return } - t.objTree.Lock() - defer t.objTree.Unlock() - _, err = t.objTree.AddContent(context.Background(), tree.SignableChangeContent{ + t.Lock() + defer t.Unlock() + _, err = t.AddContent(context.Background(), tree.SignableChangeContent{ Data: res, Key: t.account.Account().SignKey, Identity: t.account.Account().Identity, @@ -85,10 +81,10 @@ func (t *textDocument) AddText(text string) (err error) { } func (t *textDocument) Text() (text string, err error) { - t.objTree.RLock() - defer t.objTree.RUnlock() + t.RLock() + defer t.RUnlock() - err = t.objTree.Iterate( + err = t.Iterate( func(decrypted []byte) (any, error) { textChange := &testchanges.TextData{} err = proto.Unmarshal(decrypted, textChange) diff --git a/client/storage/keys.go b/client/storage/keys.go index 05bf7e7f..ddf187ac 100644 --- a/client/storage/keys.go +++ b/client/storage/keys.go @@ -59,14 +59,16 @@ func (t treeKeys) RawChangeKey(id string) []byte { } type spaceKeys struct { - headerKey []byte - treePrefixKey []byte + headerKey []byte + treePrefixKey []byte + spaceSettingsIdKey []byte } func newSpaceKeys(spaceId string) spaceKeys { return spaceKeys{ - headerKey: storage.JoinStringsToBytes("space", "header", spaceId), - treePrefixKey: storage.JoinStringsToBytes("space", spaceId, "t", "rootId"), + headerKey: storage.JoinStringsToBytes("space", "header", spaceId), + treePrefixKey: storage.JoinStringsToBytes("space", spaceId, "t", "rootId"), + spaceSettingsIdKey: storage.JoinStringsToBytes("space", spaceId, "spaceSettingsId"), } } @@ -78,6 +80,10 @@ func (s spaceKeys) TreeRootPrefix() []byte { return s.treePrefixKey } +func (s spaceKeys) SpaceSettingsId() []byte { + return s.spaceSettingsIdKey +} + type storageServiceKeys struct { spacePrefix []byte } diff --git a/client/storage/spacestorage.go b/client/storage/spacestorage.go index 8d41b298..278abc0e 100644 --- a/client/storage/spacestorage.go +++ b/client/storage/spacestorage.go @@ -4,19 +4,23 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" spacestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" storage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/storage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/treechangeproto" "github.com/dgraph-io/badger/v3" "sync" ) type spaceStorage struct { - spaceId string - objDb *badger.DB - keys spaceKeys - aclStorage storage.ListStorage - header *spacesyncproto.RawSpaceHeaderWithId - mx sync.Mutex + spaceId string + spaceSettingsId string + objDb *badger.DB + keys spaceKeys + aclStorage storage.ListStorage + header *spacesyncproto.RawSpaceHeaderWithId + mx sync.Mutex } +var spaceValidationFunc = spacestorage.ValidateSpaceStorageCreatePayload + func newSpaceStorage(objDb *badger.DB, spaceId string) (store spacestorage.SpaceStorage, err error) { keys := newSpaceKeys(spaceId) err = objDb.View(func(txn *badger.Txn) error { @@ -30,10 +34,15 @@ func newSpaceStorage(objDb *badger.DB, spaceId string) (store spacestorage.Space return err } + spaceSettingsId, err := getTxn(txn, keys.SpaceSettingsId()) + if err != nil { + return err + } store = &spaceStorage{ - spaceId: spaceId, - objDb: objDb, - keys: keys, + spaceId: spaceId, + spaceSettingsId: string(spaceSettingsId), + objDb: objDb, + keys: keys, header: &spacesyncproto.RawSpaceHeaderWithId{ RawHeader: header, Id: spaceId, @@ -54,8 +63,32 @@ func createSpaceStorage(db *badger.DB, payload spacestorage.SpaceStorageCreatePa err = spacesyncproto.ErrSpaceExists return } + err = spaceValidationFunc(payload) + if err != nil { + return + } + + spaceStore := &spaceStorage{ + spaceId: payload.SpaceHeaderWithId.Id, + objDb: db, + keys: keys, + spaceSettingsId: payload.SpaceSettingsWithId.Id, + header: payload.SpaceHeaderWithId, + } + _, err = spaceStore.CreateTreeStorage(storage.TreeStorageCreatePayload{ + RootRawChange: payload.SpaceSettingsWithId, + Changes: []*treechangeproto.RawTreeChangeWithId{payload.SpaceSettingsWithId}, + Heads: []string{payload.SpaceHeaderWithId.Id}, + }) + if err != nil { + return + } err = db.Update(func(txn *badger.Txn) error { - aclStorage, err := createListStorage(payload.SpaceHeaderWithId.Id, db, txn, payload.RecWithId) + err = txn.Set(keys.SpaceSettingsId(), []byte(payload.SpaceSettingsWithId.Id)) + if err != nil { + return err + } + aclStorage, err := createListStorage(payload.SpaceHeaderWithId.Id, db, txn, payload.AclWithId) if err != nil { return err } @@ -65,15 +98,10 @@ func createSpaceStorage(db *badger.DB, payload spacestorage.SpaceStorageCreatePa return err } - store = &spaceStorage{ - spaceId: payload.SpaceHeaderWithId.Id, - objDb: db, - keys: keys, - aclStorage: aclStorage, - header: payload.SpaceHeaderWithId, - } + spaceStore.aclStorage = aclStorage return nil }) + store = spaceStore return } @@ -81,6 +109,10 @@ func (s *spaceStorage) Id() string { return s.spaceId } +func (s *spaceStorage) SpaceSettingsId() string { + return s.spaceSettingsId +} + func (s *spaceStorage) TreeStorage(id string) (storage.TreeStorage, error) { return newTreeStorage(s.objDb, s.spaceId, id) } diff --git a/client/storage/spacestorage_test.go b/client/storage/spacestorage_test.go index 8540d338..84665039 100644 --- a/client/storage/spacestorage_test.go +++ b/client/storage/spacestorage_test.go @@ -19,7 +19,7 @@ func spaceTestPayload() spacestorage.SpaceStorageCreatePayload { Id: "aclRootId", } return spacestorage.SpaceStorageCreatePayload{ - RecWithId: aclRoot, + AclWithId: aclRoot, SpaceHeaderWithId: header, } } @@ -31,7 +31,7 @@ func testSpace(t *testing.T, store spacestorage.SpaceStorage, payload spacestora aclStorage, err := store.ACLStorage() require.NoError(t, err) - testList(t, aclStorage, payload.RecWithId, payload.RecWithId.Id) + testList(t, aclStorage, payload.AclWithId, payload.AclWithId.Id) } func TestSpaceStorage_Create(t *testing.T) { diff --git a/common/commonspace/deletionservice/service.go b/common/commonspace/deletionservice/service.go new file mode 100644 index 00000000..f6f715a2 --- /dev/null +++ b/common/commonspace/deletionservice/service.go @@ -0,0 +1,28 @@ +package deletionservice + +import ( + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/account" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" +) + +type Service interface { +} + +const deletionChangeType = "space.deletionlist" + +type service struct { + account account.Service +} + +func New() Service { + return nil +} + +func deriveDeletionTreePayload(account account.Service, spaceId string) tree.ObjectTreeCreatePayload { + return tree.ObjectTreeCreatePayload{ + SignKey: account.Account().SignKey, + ChangeType: deletionChangeType, + SpaceId: spaceId, + Identity: account.Account().Identity, + } +} diff --git a/common/commonspace/payloads.go b/common/commonspace/payloads.go index 66d257ec..843d597c 100644 --- a/common/commonspace/payloads.go +++ b/common/commonspace/payloads.go @@ -84,7 +84,7 @@ func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload st // creating storage storagePayload = storage.SpaceStorageCreatePayload{ - RecWithId: rawWithId, + AclWithId: rawWithId, SpaceHeaderWithId: rawHeaderWithId, } return @@ -176,7 +176,7 @@ func storagePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload st // creating storage storagePayload = storage.SpaceStorageCreatePayload{ - RecWithId: rawWithId, + AclWithId: rawWithId, SpaceHeaderWithId: rawHeaderWithId, } return diff --git a/common/commonspace/service.go b/common/commonspace/service.go index f9ca00d8..6947fa83 100644 --- a/common/commonspace/service.go +++ b/common/commonspace/service.go @@ -94,7 +94,7 @@ func (s *service) AddSpace(ctx context.Context, spaceDescription SpaceDescriptio } payload := storage.SpaceStorageCreatePayload{ - RecWithId: &aclrecordproto.RawACLRecordWithId{ + AclWithId: &aclrecordproto.RawACLRecordWithId{ Payload: spaceDescription.AclPayload, Id: spaceDescription.AclId, }, @@ -162,7 +162,7 @@ func (s *service) getSpaceStorageFromRemote(ctx context.Context, id string) (st return } st, err = s.storageProvider.CreateSpaceStorage(storage.SpaceStorageCreatePayload{ - RecWithId: &aclrecordproto.RawACLRecordWithId{ + AclWithId: &aclrecordproto.RawACLRecordWithId{ Payload: res.AclPayload, Id: res.AclPayloadId, }, diff --git a/common/commonspace/spacesyncproto/protos/spacesync.proto b/common/commonspace/spacesyncproto/protos/spacesync.proto index 441c0992..cf3b11ed 100644 --- a/common/commonspace/spacesyncproto/protos/spacesync.proto +++ b/common/commonspace/spacesyncproto/protos/spacesync.proto @@ -64,9 +64,7 @@ message ObjectSyncMessage { // PushSpaceRequest is a request to add space on a node containing only one acl record message PushSpaceRequest { - RawSpaceHeaderWithId spaceHeader = 1; - bytes aclPayload = 2; - string aclPayloadId = 3; + SpacePayload payload = 1; } // PushSpaceResponse is an empty response @@ -79,9 +77,15 @@ message PullSpaceRequest { // PullSpaceResponse is a response with header and acl root message PullSpaceResponse { + SpacePayload payload = 1; +} + +message SpacePayload { RawSpaceHeaderWithId spaceHeader = 1; bytes aclPayload = 2; string aclPayloadId = 3; + bytes spaceSettingsPayload = 4; + string spaceSettingsPayloadId = 5; } // SpaceHeader is a header for a space diff --git a/common/commonspace/spacesyncproto/spacesync.pb.go b/common/commonspace/spacesyncproto/spacesync.pb.go index 9f73b956..07e96827 100644 --- a/common/commonspace/spacesyncproto/spacesync.pb.go +++ b/common/commonspace/spacesyncproto/spacesync.pb.go @@ -397,9 +397,7 @@ func (m *ObjectSyncMessage) GetObjectId() string { // PushSpaceRequest is a request to add space on a node containing only one acl record type PushSpaceRequest struct { - SpaceHeader *RawSpaceHeaderWithId `protobuf:"bytes,1,opt,name=spaceHeader,proto3" json:"spaceHeader,omitempty"` - AclPayload []byte `protobuf:"bytes,2,opt,name=aclPayload,proto3" json:"aclPayload,omitempty"` - AclPayloadId string `protobuf:"bytes,3,opt,name=aclPayloadId,proto3" json:"aclPayloadId,omitempty"` + Payload *SpacePayload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` } func (m *PushSpaceRequest) Reset() { *m = PushSpaceRequest{} } @@ -435,27 +433,13 @@ func (m *PushSpaceRequest) XXX_DiscardUnknown() { var xxx_messageInfo_PushSpaceRequest proto.InternalMessageInfo -func (m *PushSpaceRequest) GetSpaceHeader() *RawSpaceHeaderWithId { +func (m *PushSpaceRequest) GetPayload() *SpacePayload { if m != nil { - return m.SpaceHeader + return m.Payload } return nil } -func (m *PushSpaceRequest) GetAclPayload() []byte { - if m != nil { - return m.AclPayload - } - return nil -} - -func (m *PushSpaceRequest) GetAclPayloadId() string { - if m != nil { - return m.AclPayloadId - } - return "" -} - // PushSpaceResponse is an empty response type PushSpaceResponse struct { } @@ -540,9 +524,7 @@ func (m *PullSpaceRequest) GetId() string { // PullSpaceResponse is a response with header and acl root type PullSpaceResponse struct { - SpaceHeader *RawSpaceHeaderWithId `protobuf:"bytes,1,opt,name=spaceHeader,proto3" json:"spaceHeader,omitempty"` - AclPayload []byte `protobuf:"bytes,2,opt,name=aclPayload,proto3" json:"aclPayload,omitempty"` - AclPayloadId string `protobuf:"bytes,3,opt,name=aclPayloadId,proto3" json:"aclPayloadId,omitempty"` + Payload *SpacePayload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` } func (m *PullSpaceResponse) Reset() { *m = PullSpaceResponse{} } @@ -578,27 +560,89 @@ func (m *PullSpaceResponse) XXX_DiscardUnknown() { var xxx_messageInfo_PullSpaceResponse proto.InternalMessageInfo -func (m *PullSpaceResponse) GetSpaceHeader() *RawSpaceHeaderWithId { +func (m *PullSpaceResponse) GetPayload() *SpacePayload { + if m != nil { + return m.Payload + } + return nil +} + +type SpacePayload struct { + SpaceHeader *RawSpaceHeaderWithId `protobuf:"bytes,1,opt,name=spaceHeader,proto3" json:"spaceHeader,omitempty"` + AclPayload []byte `protobuf:"bytes,2,opt,name=aclPayload,proto3" json:"aclPayload,omitempty"` + AclPayloadId string `protobuf:"bytes,3,opt,name=aclPayloadId,proto3" json:"aclPayloadId,omitempty"` + SpaceSettingsPayload []byte `protobuf:"bytes,4,opt,name=spaceSettingsPayload,proto3" json:"spaceSettingsPayload,omitempty"` + SpaceSettingsPayloadId string `protobuf:"bytes,5,opt,name=spaceSettingsPayloadId,proto3" json:"spaceSettingsPayloadId,omitempty"` +} + +func (m *SpacePayload) Reset() { *m = SpacePayload{} } +func (m *SpacePayload) String() string { return proto.CompactTextString(m) } +func (*SpacePayload) ProtoMessage() {} +func (*SpacePayload) Descriptor() ([]byte, []int) { + return fileDescriptor_80e49f1f4ac27799, []int{10} +} +func (m *SpacePayload) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SpacePayload) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SpacePayload.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SpacePayload) XXX_Merge(src proto.Message) { + xxx_messageInfo_SpacePayload.Merge(m, src) +} +func (m *SpacePayload) XXX_Size() int { + return m.Size() +} +func (m *SpacePayload) XXX_DiscardUnknown() { + xxx_messageInfo_SpacePayload.DiscardUnknown(m) +} + +var xxx_messageInfo_SpacePayload proto.InternalMessageInfo + +func (m *SpacePayload) GetSpaceHeader() *RawSpaceHeaderWithId { if m != nil { return m.SpaceHeader } return nil } -func (m *PullSpaceResponse) GetAclPayload() []byte { +func (m *SpacePayload) GetAclPayload() []byte { if m != nil { return m.AclPayload } return nil } -func (m *PullSpaceResponse) GetAclPayloadId() string { +func (m *SpacePayload) GetAclPayloadId() string { if m != nil { return m.AclPayloadId } return "" } +func (m *SpacePayload) GetSpaceSettingsPayload() []byte { + if m != nil { + return m.SpaceSettingsPayload + } + return nil +} + +func (m *SpacePayload) GetSpaceSettingsPayloadId() string { + if m != nil { + return m.SpaceSettingsPayloadId + } + return "" +} + // SpaceHeader is a header for a space type SpaceHeader struct { Identity []byte `protobuf:"bytes,1,opt,name=identity,proto3" json:"identity,omitempty"` @@ -612,7 +656,7 @@ func (m *SpaceHeader) Reset() { *m = SpaceHeader{} } func (m *SpaceHeader) String() string { return proto.CompactTextString(m) } func (*SpaceHeader) ProtoMessage() {} func (*SpaceHeader) Descriptor() ([]byte, []int) { - return fileDescriptor_80e49f1f4ac27799, []int{10} + return fileDescriptor_80e49f1f4ac27799, []int{11} } func (m *SpaceHeader) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -685,7 +729,7 @@ func (m *RawSpaceHeader) Reset() { *m = RawSpaceHeader{} } func (m *RawSpaceHeader) String() string { return proto.CompactTextString(m) } func (*RawSpaceHeader) ProtoMessage() {} func (*RawSpaceHeader) Descriptor() ([]byte, []int) { - return fileDescriptor_80e49f1f4ac27799, []int{11} + return fileDescriptor_80e49f1f4ac27799, []int{12} } func (m *RawSpaceHeader) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -737,7 +781,7 @@ func (m *RawSpaceHeaderWithId) Reset() { *m = RawSpaceHeaderWithId{} } func (m *RawSpaceHeaderWithId) String() string { return proto.CompactTextString(m) } func (*RawSpaceHeaderWithId) ProtoMessage() {} func (*RawSpaceHeaderWithId) Descriptor() ([]byte, []int) { - return fileDescriptor_80e49f1f4ac27799, []int{12} + return fileDescriptor_80e49f1f4ac27799, []int{13} } func (m *RawSpaceHeaderWithId) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -792,6 +836,7 @@ func init() { proto.RegisterType((*PushSpaceResponse)(nil), "anySpace.PushSpaceResponse") proto.RegisterType((*PullSpaceRequest)(nil), "anySpace.PullSpaceRequest") proto.RegisterType((*PullSpaceResponse)(nil), "anySpace.PullSpaceResponse") + proto.RegisterType((*SpacePayload)(nil), "anySpace.SpacePayload") proto.RegisterType((*SpaceHeader)(nil), "anySpace.SpaceHeader") proto.RegisterType((*RawSpaceHeader)(nil), "anySpace.RawSpaceHeader") proto.RegisterType((*RawSpaceHeaderWithId)(nil), "anySpace.RawSpaceHeaderWithId") @@ -802,53 +847,56 @@ func init() { } var fileDescriptor_80e49f1f4ac27799 = []byte{ - // 721 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x55, 0xcd, 0x6e, 0xd3, 0x4a, - 0x14, 0x8e, 0xdd, 0xb4, 0x4d, 0x4e, 0xd2, 0x34, 0x9d, 0xdb, 0xab, 0xeb, 0x1b, 0x90, 0x89, 0xbc, - 0x40, 0x11, 0x8b, 0x16, 0x02, 0xbb, 0x6e, 0xf8, 0x69, 0x2a, 0x22, 0x54, 0x5a, 0x4d, 0x40, 0x48, - 0x88, 0xcd, 0xd4, 0x9e, 0x26, 0x46, 0xfe, 0xc3, 0x33, 0x51, 0xeb, 0x05, 0xef, 0xc0, 0x12, 0x36, - 0x48, 0xbc, 0x0d, 0xcb, 0x2e, 0x59, 0xa2, 0xf6, 0x45, 0xd0, 0x9c, 0xd8, 0xb1, 0x9d, 0xa6, 0x5d, - 0xb3, 0x71, 0xe7, 0x7c, 0xe7, 0xef, 0x9b, 0x6f, 0xce, 0x69, 0xe0, 0x91, 0x1d, 0xfa, 0x7e, 0x18, - 0x88, 0x88, 0xd9, 0x7c, 0x17, 0xbf, 0x22, 0x09, 0xec, 0x28, 0x0e, 0x65, 0xb8, 0x8b, 0x5f, 0x91, - 0xa3, 0x3b, 0x08, 0x90, 0x1a, 0x0b, 0x92, 0x91, 0xc2, 0xac, 0x21, 0x6c, 0xbc, 0xe4, 0xcc, 0x19, - 0x25, 0x81, 0x4d, 0x59, 0x30, 0xe6, 0x84, 0x40, 0xf5, 0x34, 0x0e, 0x7d, 0x43, 0xeb, 0x6a, 0xbd, - 0x2a, 0xc5, 0x33, 0x69, 0x81, 0x2e, 0x43, 0x43, 0x47, 0x44, 0x97, 0x21, 0xd9, 0x86, 0x55, 0xcf, - 0xf5, 0x5d, 0x69, 0xac, 0x74, 0xb5, 0xde, 0x06, 0x9d, 0x19, 0xd6, 0x19, 0xb4, 0xe6, 0xa5, 0xb8, - 0x98, 0x7a, 0x52, 0xd5, 0x9a, 0x30, 0x31, 0xc1, 0x5a, 0x4d, 0x8a, 0x67, 0xb2, 0x07, 0x35, 0xee, - 0x71, 0x9f, 0x07, 0x52, 0x18, 0x7a, 0x77, 0xa5, 0xd7, 0xe8, 0xdf, 0xdb, 0xc9, 0xd8, 0xec, 0x94, - 0xf3, 0x07, 0xb3, 0x38, 0x3a, 0x4f, 0x50, 0x8d, 0xed, 0x70, 0x1a, 0xcc, 0x1b, 0xa3, 0x61, 0xed, - 0xc1, 0xbf, 0x4b, 0x13, 0x15, 0x6f, 0xd7, 0xc1, 0xee, 0x75, 0xaa, 0xbb, 0x0e, 0xf2, 0xe1, 0xcc, - 0xc1, 0x9b, 0xd4, 0x29, 0x9e, 0xad, 0x0f, 0xb0, 0x99, 0x27, 0x7f, 0x9a, 0x72, 0x21, 0x89, 0x01, - 0xeb, 0x28, 0xd8, 0x30, 0xcb, 0xcd, 0x4c, 0xb2, 0x0b, 0x6b, 0xb1, 0x52, 0x29, 0xa3, 0xfe, 0xdf, - 0x12, 0xea, 0xca, 0x4f, 0xd3, 0x30, 0xeb, 0x00, 0xda, 0x05, 0x6a, 0x51, 0x18, 0x08, 0x4e, 0xfa, - 0xb0, 0x1e, 0x23, 0x4d, 0x61, 0x68, 0x58, 0xc5, 0xb8, 0x49, 0x00, 0x9a, 0x05, 0x5a, 0x9f, 0x61, - 0xeb, 0xe8, 0xe4, 0x23, 0xb7, 0xa5, 0x72, 0x1e, 0x72, 0x21, 0xd8, 0x98, 0xdf, 0xc2, 0xd3, 0x50, - 0x2d, 0x22, 0x2f, 0x19, 0x66, 0x77, 0xcd, 0x4c, 0xe5, 0x89, 0x58, 0xe2, 0x85, 0xcc, 0x41, 0x0d, - 0x9b, 0x34, 0x33, 0x49, 0x07, 0x6a, 0x21, 0xb6, 0x18, 0x3a, 0x46, 0x15, 0x93, 0xe6, 0xb6, 0xf5, - 0x55, 0x83, 0xf6, 0xf1, 0x54, 0x4c, 0x90, 0x64, 0x26, 0xd3, 0x53, 0x68, 0x60, 0x3f, 0xc5, 0x99, - 0xc7, 0x48, 0xa1, 0xd1, 0x37, 0xf3, 0xbb, 0x50, 0x76, 0x36, 0xca, 0xfd, 0xef, 0x5c, 0x39, 0x19, - 0x3a, 0xb4, 0x98, 0x42, 0x4c, 0x00, 0x66, 0x7b, 0xc7, 0x29, 0x1f, 0x1d, 0xf9, 0x14, 0x10, 0x62, - 0x41, 0x33, 0xb7, 0x86, 0x33, 0xc6, 0x75, 0x5a, 0xc2, 0xac, 0x7f, 0x60, 0xab, 0xc0, 0x6c, 0x26, - 0xb1, 0x65, 0x29, 0xba, 0x9e, 0x57, 0xa2, 0xbb, 0x30, 0x0c, 0xd6, 0x37, 0x4d, 0x65, 0xce, 0x83, - 0xd2, 0xc7, 0xf9, 0x3b, 0x2e, 0xf5, 0x43, 0x83, 0x46, 0xa1, 0x8d, 0x7a, 0x1b, 0xd7, 0xe1, 0x81, - 0x74, 0x65, 0x92, 0x2e, 0xd3, 0xdc, 0x26, 0x77, 0xa1, 0x2e, 0x5d, 0x9f, 0x0b, 0xc9, 0xfc, 0x08, - 0xdb, 0xad, 0xd0, 0x1c, 0x50, 0x5e, 0x24, 0xf7, 0x26, 0x89, 0x78, 0xda, 0x2a, 0x07, 0xc8, 0x7d, - 0x68, 0xa9, 0xc1, 0x70, 0x6d, 0x26, 0xdd, 0x30, 0x78, 0xc5, 0x13, 0x7c, 0xf9, 0x2a, 0x5d, 0x40, - 0xd5, 0xe2, 0x08, 0xce, 0x1d, 0x63, 0x75, 0xb6, 0xc8, 0xea, 0x6c, 0x1d, 0x43, 0xab, 0x2c, 0x06, - 0xe9, 0x5e, 0xd7, 0xae, 0x59, 0xd6, 0x46, 0xb1, 0x71, 0xc7, 0x01, 0x93, 0xd3, 0x98, 0xa7, 0xd2, - 0xe4, 0x80, 0xb5, 0x0f, 0xdb, 0xcb, 0xe4, 0x55, 0x59, 0x31, 0x3b, 0x2b, 0x55, 0xcd, 0x81, 0xf4, - 0x5d, 0xf5, 0xec, 0x5d, 0x1f, 0xbc, 0x86, 0xda, 0x20, 0x8e, 0x5f, 0x84, 0x0e, 0x17, 0xa4, 0x05, - 0xf0, 0x36, 0xe0, 0xe7, 0x11, 0xb7, 0x25, 0x77, 0xda, 0x15, 0xd2, 0x86, 0x26, 0x96, 0x3f, 0x74, - 0x85, 0x70, 0x83, 0x71, 0x5b, 0x23, 0x9b, 0xa9, 0xd0, 0x83, 0x73, 0x57, 0x48, 0xd1, 0xd6, 0x15, - 0x30, 0x88, 0xe3, 0x30, 0x3e, 0x3a, 0x3d, 0x15, 0x5c, 0xb6, 0x9d, 0xfe, 0x77, 0x1d, 0x56, 0x31, - 0x84, 0x3c, 0x83, 0x5a, 0xb6, 0x9f, 0xe4, 0xff, 0x65, 0x3b, 0x8b, 0x83, 0xd6, 0xe9, 0x2c, 0x5d, - 0xe7, 0xd9, 0x78, 0xed, 0x43, 0x7d, 0x3e, 0xad, 0xa4, 0x10, 0xb8, 0xb8, 0x5c, 0x9d, 0x3b, 0x4b, - 0x7d, 0xc5, 0x2a, 0xe9, 0xe4, 0x96, 0xab, 0x94, 0x67, 0xbe, 0x5c, 0x65, 0x71, 0xd4, 0x0f, 0x60, - 0x6d, 0x24, 0x63, 0xce, 0x7c, 0x52, 0x08, 0xbb, 0xf6, 0x5f, 0xa6, 0x73, 0x9b, 0xb3, 0xa7, 0x3d, - 0xd4, 0x9e, 0x3f, 0xf9, 0x79, 0x69, 0x6a, 0x17, 0x97, 0xa6, 0xf6, 0xfb, 0xd2, 0xd4, 0xbe, 0x5c, - 0x99, 0x95, 0x8b, 0x2b, 0xb3, 0xf2, 0xeb, 0xca, 0xac, 0xbc, 0xef, 0xdc, 0xfc, 0xcb, 0x74, 0xb2, - 0x86, 0x7f, 0x1e, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0xf7, 0x95, 0x5b, 0xbc, 0xbe, 0x06, 0x00, - 0x00, + // 770 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0xcd, 0x4e, 0x23, 0x47, + 0x10, 0xf6, 0x0c, 0x06, 0xec, 0xb2, 0x31, 0xa6, 0x43, 0xc8, 0xc4, 0x89, 0x26, 0xd6, 0x1c, 0x22, + 0x2b, 0x07, 0x20, 0x4e, 0x94, 0x0b, 0x97, 0xfc, 0x60, 0x14, 0x2b, 0x22, 0xa0, 0x76, 0xa2, 0x48, + 0x51, 0x2e, 0xcd, 0x4c, 0x63, 0x4f, 0x34, 0x7f, 0x3b, 0xdd, 0x16, 0xcc, 0x61, 0xa5, 0x7d, 0x84, + 0x7d, 0x82, 0x95, 0xf6, 0x6d, 0xf6, 0xc8, 0x71, 0x8f, 0x2b, 0x78, 0x91, 0x55, 0x97, 0xe7, 0xd7, + 0x0c, 0x48, 0x7b, 0x19, 0x4f, 0x7d, 0x55, 0xf5, 0x55, 0xd5, 0xd7, 0x5d, 0x63, 0xf8, 0xde, 0x0e, + 0x7d, 0x3f, 0x0c, 0x44, 0xc4, 0x6c, 0x7e, 0x84, 0x4f, 0x91, 0x04, 0x76, 0x14, 0x87, 0x32, 0x3c, + 0xc2, 0xa7, 0x28, 0xd0, 0x43, 0x04, 0x48, 0x8b, 0x05, 0xc9, 0x4c, 0x61, 0xd6, 0x14, 0x76, 0x7e, + 0xe7, 0xcc, 0x99, 0x25, 0x81, 0x4d, 0x59, 0x30, 0xe7, 0x84, 0x40, 0xf3, 0x3a, 0x0e, 0x7d, 0x43, + 0x1b, 0x6a, 0xa3, 0x26, 0xc5, 0x77, 0xd2, 0x03, 0x5d, 0x86, 0x86, 0x8e, 0x88, 0x2e, 0x43, 0xb2, + 0x0f, 0x9b, 0x9e, 0xeb, 0xbb, 0xd2, 0xd8, 0x18, 0x6a, 0xa3, 0x1d, 0xba, 0x32, 0xac, 0x1b, 0xe8, + 0xe5, 0x54, 0x5c, 0x2c, 0x3d, 0xa9, 0xb8, 0x16, 0x4c, 0x2c, 0x90, 0xab, 0x4b, 0xf1, 0x9d, 0x9c, + 0x40, 0x8b, 0x7b, 0xdc, 0xe7, 0x81, 0x14, 0x86, 0x3e, 0xdc, 0x18, 0x75, 0xc6, 0xdf, 0x1c, 0x66, + 0xdd, 0x1c, 0x56, 0xf3, 0x27, 0xab, 0x38, 0x9a, 0x27, 0xa8, 0xc2, 0x76, 0xb8, 0x0c, 0xf2, 0xc2, + 0x68, 0x58, 0x27, 0xf0, 0x79, 0x6d, 0xa2, 0xea, 0xdb, 0x75, 0xb0, 0x7a, 0x9b, 0xea, 0xae, 0x83, + 0xfd, 0x70, 0xe6, 0xe0, 0x24, 0x6d, 0x8a, 0xef, 0xd6, 0x7f, 0xb0, 0x5b, 0x24, 0xbf, 0x58, 0x72, + 0x21, 0x89, 0x01, 0xdb, 0x28, 0xd8, 0x34, 0xcb, 0xcd, 0x4c, 0x72, 0x04, 0x5b, 0xb1, 0x52, 0x29, + 0x6b, 0xfd, 0x8b, 0x9a, 0xd6, 0x95, 0x9f, 0xa6, 0x61, 0xd6, 0x19, 0xf4, 0x4b, 0xad, 0x45, 0x61, + 0x20, 0x38, 0x19, 0xc3, 0x76, 0x8c, 0x6d, 0x0a, 0x43, 0x43, 0x16, 0xe3, 0x29, 0x01, 0x68, 0x16, + 0x68, 0xbd, 0x84, 0xbd, 0x8b, 0xab, 0xff, 0xb9, 0x2d, 0x95, 0xf3, 0x9c, 0x0b, 0xc1, 0xe6, 0xfc, + 0x99, 0x3e, 0x0d, 0x55, 0x22, 0xf2, 0x92, 0x69, 0x36, 0x6b, 0x66, 0x2a, 0x4f, 0xc4, 0x12, 0x2f, + 0x64, 0x0e, 0x6a, 0xd8, 0xa5, 0x99, 0x49, 0x06, 0xd0, 0x0a, 0xb1, 0xc4, 0xd4, 0x31, 0x9a, 0x98, + 0x94, 0xdb, 0xd6, 0x29, 0xf4, 0x2f, 0x97, 0x62, 0x81, 0x3d, 0x66, 0x2a, 0x1d, 0x17, 0x4c, 0xaa, + 0x7a, 0x67, 0x7c, 0x50, 0x8c, 0x81, 0xcf, 0xcb, 0x95, 0x37, 0xaf, 0x60, 0x7d, 0x06, 0x7b, 0x25, + 0x96, 0x95, 0x1a, 0x96, 0xa5, 0xa8, 0x3d, 0xaf, 0x42, 0xbd, 0x76, 0x6e, 0xd6, 0x44, 0x25, 0xe6, + 0x31, 0xa9, 0x8c, 0x9f, 0x5e, 0xff, 0x95, 0x0e, 0xdd, 0xb2, 0x87, 0xfc, 0x0c, 0x1d, 0x54, 0x4c, + 0xa9, 0xce, 0xe3, 0x94, 0xc6, 0x2c, 0x68, 0x28, 0xbb, 0x99, 0x15, 0xfe, 0x7f, 0x5c, 0xb9, 0x98, + 0x3a, 0xb4, 0x9c, 0x42, 0x4c, 0x00, 0x66, 0x7b, 0x29, 0x1f, 0x6a, 0xdd, 0xa5, 0x25, 0x84, 0x58, + 0xd0, 0x2d, 0xac, 0xe9, 0x4a, 0xf3, 0x36, 0xad, 0x60, 0x64, 0x0c, 0xfb, 0x48, 0x39, 0xe3, 0x52, + 0xba, 0xc1, 0x5c, 0x64, 0x6c, 0x4d, 0x64, 0xab, 0xf5, 0x91, 0x9f, 0xe0, 0xa0, 0x0e, 0x9f, 0x3a, + 0xc6, 0x26, 0x56, 0x78, 0xc2, 0x6b, 0xbd, 0xd5, 0xa0, 0x53, 0x1a, 0x49, 0x1d, 0xba, 0xeb, 0xf0, + 0x40, 0xba, 0x32, 0x49, 0xb7, 0x34, 0xb7, 0xc9, 0xd7, 0xd0, 0x96, 0xae, 0xcf, 0x85, 0x64, 0x7e, + 0x84, 0xa3, 0x6d, 0xd0, 0x02, 0x50, 0x5e, 0xac, 0xf1, 0x57, 0x12, 0xf1, 0x74, 0xac, 0x02, 0x20, + 0xdf, 0x42, 0x4f, 0xdd, 0x38, 0xd7, 0x66, 0xd2, 0x0d, 0x83, 0x3f, 0x78, 0x82, 0xd3, 0x34, 0xe9, + 0x1a, 0xaa, 0x36, 0x52, 0x70, 0xbe, 0xea, 0xba, 0x4b, 0xf1, 0xdd, 0xba, 0x84, 0x5e, 0x55, 0x78, + 0x32, 0x7c, 0x7c, 0x4e, 0xdd, 0xea, 0x39, 0xa8, 0x6e, 0xdc, 0x79, 0xc0, 0xe4, 0x32, 0xe6, 0xe9, + 0x31, 0x14, 0x80, 0x75, 0x0a, 0xfb, 0x75, 0x47, 0xa9, 0xb2, 0x62, 0x76, 0x53, 0x61, 0x2d, 0x80, + 0xf4, 0x16, 0xea, 0xd9, 0x2d, 0xfc, 0xee, 0x4f, 0x68, 0x4d, 0xe2, 0xf8, 0xb7, 0xd0, 0xe1, 0x82, + 0xf4, 0x00, 0xfe, 0x0e, 0xf8, 0x6d, 0xc4, 0x6d, 0xc9, 0x9d, 0x7e, 0x83, 0xf4, 0xd3, 0x9b, 0x75, + 0xee, 0x0a, 0xe1, 0x06, 0xf3, 0xbe, 0x46, 0x76, 0x53, 0xa1, 0x27, 0xb7, 0xae, 0x90, 0xa2, 0xaf, + 0x2b, 0x60, 0x12, 0xc7, 0x61, 0x7c, 0x71, 0x7d, 0x2d, 0xb8, 0xec, 0x3b, 0xe3, 0x37, 0x3a, 0x6c, + 0x62, 0x08, 0xf9, 0x05, 0x5a, 0xd9, 0xe2, 0x93, 0x2f, 0xeb, 0x3e, 0x06, 0xb8, 0x16, 0x83, 0x41, + 0xed, 0x77, 0x62, 0xb5, 0x0d, 0xa7, 0xd0, 0xce, 0x77, 0x8b, 0x94, 0x02, 0xd7, 0xd7, 0x76, 0xf0, + 0x55, 0xad, 0xaf, 0xcc, 0x92, 0x2e, 0x5a, 0x95, 0xa5, 0xba, 0xa1, 0x55, 0x96, 0xf5, 0xcd, 0x3c, + 0x83, 0xad, 0x99, 0x8c, 0x39, 0xf3, 0x49, 0x29, 0xec, 0xd1, 0xe7, 0x6b, 0xf0, 0x9c, 0x73, 0xa4, + 0x1d, 0x6b, 0xbf, 0xfe, 0xf8, 0xee, 0xde, 0xd4, 0xee, 0xee, 0x4d, 0xed, 0xc3, 0xbd, 0xa9, 0xbd, + 0x7e, 0x30, 0x1b, 0x77, 0x0f, 0x66, 0xe3, 0xfd, 0x83, 0xd9, 0xf8, 0x77, 0xf0, 0xf4, 0x5f, 0xde, + 0xd5, 0x16, 0xfe, 0xfc, 0xf0, 0x31, 0x00, 0x00, 0xff, 0xff, 0xaa, 0x60, 0x9b, 0x89, 0x17, 0x07, + 0x00, 0x00, } func (m *HeadSyncRange) Marshal() (dAtA []byte, err error) { @@ -1127,23 +1175,9 @@ func (m *PushSpaceRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.AclPayloadId) > 0 { - i -= len(m.AclPayloadId) - copy(dAtA[i:], m.AclPayloadId) - i = encodeVarintSpacesync(dAtA, i, uint64(len(m.AclPayloadId))) - i-- - dAtA[i] = 0x1a - } - if len(m.AclPayload) > 0 { - i -= len(m.AclPayload) - copy(dAtA[i:], m.AclPayload) - i = encodeVarintSpacesync(dAtA, i, uint64(len(m.AclPayload))) - i-- - dAtA[i] = 0x12 - } - if m.SpaceHeader != nil { + if m.Payload != nil { { - size, err := m.SpaceHeader.MarshalToSizedBuffer(dAtA[:i]) + size, err := m.Payload.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } @@ -1229,6 +1263,55 @@ func (m *PullSpaceResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Payload != nil { + { + size, err := m.Payload.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintSpacesync(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SpacePayload) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpacePayload) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SpacePayload) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.SpaceSettingsPayloadId) > 0 { + i -= len(m.SpaceSettingsPayloadId) + copy(dAtA[i:], m.SpaceSettingsPayloadId) + i = encodeVarintSpacesync(dAtA, i, uint64(len(m.SpaceSettingsPayloadId))) + i-- + dAtA[i] = 0x2a + } + if len(m.SpaceSettingsPayload) > 0 { + i -= len(m.SpaceSettingsPayload) + copy(dAtA[i:], m.SpaceSettingsPayload) + i = encodeVarintSpacesync(dAtA, i, uint64(len(m.SpaceSettingsPayload))) + i-- + dAtA[i] = 0x22 + } if len(m.AclPayloadId) > 0 { i -= len(m.AclPayloadId) copy(dAtA[i:], m.AclPayloadId) @@ -1519,16 +1602,8 @@ func (m *PushSpaceRequest) Size() (n int) { } var l int _ = l - if m.SpaceHeader != nil { - l = m.SpaceHeader.Size() - n += 1 + l + sovSpacesync(uint64(l)) - } - l = len(m.AclPayload) - if l > 0 { - n += 1 + l + sovSpacesync(uint64(l)) - } - l = len(m.AclPayloadId) - if l > 0 { + if m.Payload != nil { + l = m.Payload.Size() n += 1 + l + sovSpacesync(uint64(l)) } return n @@ -1557,6 +1632,19 @@ func (m *PullSpaceRequest) Size() (n int) { } func (m *PullSpaceResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Payload != nil { + l = m.Payload.Size() + n += 1 + l + sovSpacesync(uint64(l)) + } + return n +} + +func (m *SpacePayload) Size() (n int) { if m == nil { return 0 } @@ -1574,6 +1662,14 @@ func (m *PullSpaceResponse) Size() (n int) { if l > 0 { n += 1 + l + sovSpacesync(uint64(l)) } + l = len(m.SpaceSettingsPayload) + if l > 0 { + n += 1 + l + sovSpacesync(uint64(l)) + } + l = len(m.SpaceSettingsPayloadId) + if l > 0 { + n += 1 + l + sovSpacesync(uint64(l)) + } return n } @@ -2413,7 +2509,7 @@ func (m *PushSpaceRequest) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SpaceHeader", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Payload", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -2440,79 +2536,13 @@ func (m *PushSpaceRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.SpaceHeader == nil { - m.SpaceHeader = &RawSpaceHeaderWithId{} + if m.Payload == nil { + m.Payload = &SpacePayload{} } - if err := m.SpaceHeader.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.Payload.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field AclPayload", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowSpacesync - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthSpacesync - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthSpacesync - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.AclPayload = append(m.AclPayload[:0], dAtA[iNdEx:postIndex]...) - if m.AclPayload == nil { - m.AclPayload = []byte{} - } - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field AclPayloadId", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowSpacesync - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthSpacesync - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthSpacesync - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.AclPayloadId = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipSpacesync(dAtA[iNdEx:]) @@ -2695,6 +2725,92 @@ func (m *PullSpaceResponse) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: PullSpaceResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Payload", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpacesync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthSpacesync + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthSpacesync + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Payload == nil { + m.Payload = &SpacePayload{} + } + if err := m.Payload.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipSpacesync(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthSpacesync + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SpacePayload) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpacesync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpacePayload: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpacePayload: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field SpaceHeader", wireType) @@ -2797,6 +2913,72 @@ func (m *PullSpaceResponse) Unmarshal(dAtA []byte) error { } m.AclPayloadId = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceSettingsPayload", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpacesync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthSpacesync + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthSpacesync + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceSettingsPayload = append(m.SpaceSettingsPayload[:0], dAtA[iNdEx:postIndex]...) + if m.SpaceSettingsPayload == nil { + m.SpaceSettingsPayload = []byte{} + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceSettingsPayloadId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpacesync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthSpacesync + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthSpacesync + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceSettingsPayloadId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipSpacesync(dAtA[iNdEx:]) diff --git a/common/commonspace/storage/storage.go b/common/commonspace/storage/storage.go index a18b9c16..ba9d0e26 100644 --- a/common/commonspace/storage/storage.go +++ b/common/commonspace/storage/storage.go @@ -7,6 +7,7 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/aclrecordproto" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/storage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/treechangeproto" ) const CName = "commonspace.storage" @@ -17,6 +18,7 @@ var ErrSpaceStorageMissing = errors.New("space storage missing") type SpaceStorage interface { storage.Provider Id() string + SpaceSettingsId() string ACLStorage() (storage.ListStorage, error) SpaceHeader() (*spacesyncproto.RawSpaceHeaderWithId, error) StoredIds() ([]string, error) @@ -24,8 +26,9 @@ type SpaceStorage interface { } type SpaceStorageCreatePayload struct { - RecWithId *aclrecordproto.RawACLRecordWithId - SpaceHeaderWithId *spacesyncproto.RawSpaceHeaderWithId + AclWithId *aclrecordproto.RawACLRecordWithId + SpaceHeaderWithId *spacesyncproto.RawSpaceHeaderWithId + SpaceSettingsWithId *treechangeproto.RawTreeChangeWithId } type SpaceStorageProvider interface { @@ -33,3 +36,8 @@ type SpaceStorageProvider interface { SpaceStorage(id string) (SpaceStorage, error) CreateSpaceStorage(payload SpaceStorageCreatePayload) (SpaceStorage, error) } + +func ValidateSpaceStorageCreatePayload(payload SpaceStorageCreatePayload) (err error) { + // TODO: add proper validation + return nil +} diff --git a/node/storage/keys.go b/node/storage/keys.go index e21a7140..77b58e22 100644 --- a/node/storage/keys.go +++ b/node/storage/keys.go @@ -51,7 +51,10 @@ func newSpaceKeys(spaceId string) spaceKeys { return spaceKeys{headerKey: storage.JoinStringsToBytes("s", spaceId)} } -var spaceIdKey = []byte("spaceId") +var ( + spaceIdKey = []byte("spaceId") + spaceSettingsIdKey = []byte("spaceSettingsId") +) func (s spaceKeys) SpaceIdKey() []byte { return spaceIdKey @@ -61,7 +64,11 @@ func (s spaceKeys) HeaderKey() []byte { return s.headerKey } -func isRootIdKey(key string) bool { +func (s spaceKeys) SpaceSettingsIdKey() []byte { + return spaceSettingsIdKey +} + +func isTreeHeadsKey(key string) bool { return strings.HasPrefix(key, "t/") && strings.HasSuffix(key, "/heads") } diff --git a/node/storage/spacestorage.go b/node/storage/spacestorage.go index 018c6b04..e700e0da 100644 --- a/node/storage/spacestorage.go +++ b/node/storage/spacestorage.go @@ -6,25 +6,27 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" spacestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/storage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/treechangeproto" "go.uber.org/zap" "path" "sync" "time" ) -var defPogrebOptions = &pogreb.Options{ - BackgroundCompactionInterval: time.Minute * 5, -} - -var log = logger.NewNamed("storage.spacestorage") +var ( + defPogrebOptions = &pogreb.Options{BackgroundCompactionInterval: time.Minute * 5} + log = logger.NewNamed("storage.spacestorage") + spaceValidationFunc = spacestorage.ValidateSpaceStorageCreatePayload +) type spaceStorage struct { - spaceId string - objDb *pogreb.DB - keys spaceKeys - aclStorage storage.ListStorage - header *spacesyncproto.RawSpaceHeaderWithId - mx sync.Mutex + spaceId string + spaceSettingsId string + objDb *pogreb.DB + keys spaceKeys + aclStorage storage.ListStorage + header *spacesyncproto.RawSpaceHeaderWithId + mx sync.Mutex } func newSpaceStorage(rootPath string, spaceId string) (store spacestorage.SpaceStorage, err error) { @@ -61,15 +63,25 @@ func newSpaceStorage(rootPath string, spaceId string) (store spacestorage.SpaceS return } + spaceSettingsId, err := objDb.Get(keys.SpaceSettingsIdKey()) + if err != nil { + return + } + if spaceSettingsId == nil { + err = spacestorage.ErrSpaceStorageMissing + return + } + aclStorage, err := newListStorage(objDb) if err != nil { return } store = &spaceStorage{ - spaceId: spaceId, - objDb: objDb, - keys: keys, + spaceId: spaceId, + spaceSettingsId: string(spaceSettingsId), + objDb: objDb, + keys: keys, header: &spacesyncproto.RawSpaceHeaderWithId{ RawHeader: header, Id: spaceId, @@ -103,8 +115,35 @@ func createSpaceStorage(rootPath string, payload spacestorage.SpaceStorageCreate err = spacesyncproto.ErrSpaceExists return } + err = spaceValidationFunc(payload) + if err != nil { + return + } - aclStorage, err := createListStorage(db, payload.RecWithId) + aclStorage, err := createListStorage(db, payload.AclWithId) + if err != nil { + return + } + + store = &spaceStorage{ + spaceId: payload.SpaceHeaderWithId.Id, + objDb: db, + keys: keys, + aclStorage: aclStorage, + spaceSettingsId: payload.SpaceSettingsWithId.Id, + header: payload.SpaceHeaderWithId, + } + + _, err = store.CreateTreeStorage(storage.TreeStorageCreatePayload{ + RootRawChange: payload.SpaceSettingsWithId, + Changes: []*treechangeproto.RawTreeChangeWithId{payload.SpaceSettingsWithId}, + Heads: []string{payload.SpaceHeaderWithId.Id}, + }) + if err != nil { + return + } + + err = db.Put(keys.SpaceSettingsIdKey(), []byte(payload.SpaceSettingsWithId.Id)) if err != nil { return } @@ -119,13 +158,6 @@ func createSpaceStorage(rootPath string, payload spacestorage.SpaceStorageCreate return } - store = &spaceStorage{ - spaceId: payload.SpaceHeaderWithId.Id, - objDb: db, - keys: keys, - aclStorage: aclStorage, - header: payload.SpaceHeaderWithId, - } return } @@ -133,6 +165,10 @@ func (s *spaceStorage) Id() string { return s.spaceId } +func (s *spaceStorage) SpaceSettingsId() string { + return s.spaceSettingsId +} + func (s *spaceStorage) TreeStorage(id string) (storage.TreeStorage, error) { return newTreeStorage(s.objDb, id) } @@ -159,7 +195,7 @@ func (s *spaceStorage) StoredIds() (ids []string, err error) { key, _, err := index.Next() for err == nil { strKey := string(key) - if isRootIdKey(strKey) { + if isTreeHeadsKey(strKey) { ids = append(ids, getRootId(strKey)) } key, _, err = index.Next() diff --git a/node/storage/spacestorage_test.go b/node/storage/spacestorage_test.go index 33243e0a..4009bd1f 100644 --- a/node/storage/spacestorage_test.go +++ b/node/storage/spacestorage_test.go @@ -20,7 +20,7 @@ func spaceTestPayload() spacestorage.SpaceStorageCreatePayload { Id: "aclRootId", } return spacestorage.SpaceStorageCreatePayload{ - RecWithId: aclRoot, + AclWithId: aclRoot, SpaceHeaderWithId: header, } } @@ -32,7 +32,7 @@ func testSpace(t *testing.T, store spacestorage.SpaceStorage, payload spacestora aclStorage, err := store.ACLStorage() require.NoError(t, err) - testList(t, aclStorage, payload.RecWithId, payload.RecWithId.Id) + testList(t, aclStorage, payload.AclWithId, payload.AclWithId.Id) } func TestSpaceStorage_Create(t *testing.T) { From 020b1f1d71e15c24fc095d52e80c93795c4c02b5 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 10 Nov 2022 20:10:57 +0100 Subject: [PATCH 02/27] Change pullspace and pushspace methods and handlers --- client/clientspace/rpchandler.go | 25 ++++++++++----- common/commonspace/diffservice/diffsyncer.go | 20 ++++++++++-- common/commonspace/service.go | 12 ++++++-- common/commonspace/space.go | 32 ++++++++++++++------ common/commonspace/synctree/synctree.go | 3 +- node/nodespace/rpchandler.go | 25 ++++++++++----- 6 files changed, 86 insertions(+), 31 deletions(-) diff --git a/client/clientspace/rpchandler.go b/client/clientspace/rpchandler.go index 8f6a7989..bef22138 100644 --- a/client/clientspace/rpchandler.go +++ b/client/clientspace/rpchandler.go @@ -19,20 +19,31 @@ func (r *rpcHandler) PullSpace(ctx context.Context, request *spacesyncproto.Pull return } - description := sp.Description() + spaceDesc, err := sp.Description() + if err != nil { + err = spacesyncproto.ErrUnexpected + return + } + resp = &spacesyncproto.PullSpaceResponse{ - SpaceHeader: description.SpaceHeader, - AclPayload: description.AclPayload, - AclPayloadId: description.AclId, + Payload: &spacesyncproto.SpacePayload{ + SpaceHeader: spaceDesc.SpaceHeader, + AclPayloadId: spaceDesc.AclId, + AclPayload: spaceDesc.AclPayload, + SpaceSettingsPayload: spaceDesc.SpaceSettingsPayload, + SpaceSettingsPayloadId: spaceDesc.SpaceSettingsId, + }, } return } func (r *rpcHandler) PushSpace(ctx context.Context, req *spacesyncproto.PushSpaceRequest) (resp *spacesyncproto.PushSpaceResponse, err error) { description := commonspace.SpaceDescription{ - SpaceHeader: req.SpaceHeader, - AclId: req.AclPayloadId, - AclPayload: req.AclPayload, + SpaceHeader: req.Payload.SpaceHeader, + AclId: req.Payload.AclPayloadId, + AclPayload: req.Payload.AclPayload, + SpaceSettingsPayload: req.Payload.SpaceSettingsPayload, + SpaceSettingsId: req.Payload.SpaceSettingsPayloadId, } err = r.s.AddSpace(ctx, description) if err != nil { diff --git a/common/commonspace/diffservice/diffsyncer.go b/common/commonspace/diffservice/diffsyncer.go index f9ca9f26..9610a787 100644 --- a/common/commonspace/diffservice/diffsyncer.go +++ b/common/commonspace/diffservice/diffsyncer.go @@ -108,10 +108,24 @@ func (d *diffSyncer) sendPushSpaceRequest(ctx context.Context, cl spacesyncproto return } + spaceSettingsTreeStorage, err := d.storage.TreeStorage(d.storage.SpaceSettingsId()) + if err != nil { + return + } + spaceSettingsRoot, err := spaceSettingsTreeStorage.Root() + if err != nil { + return + } + + spacePayload := &spacesyncproto.SpacePayload{ + SpaceHeader: header, + AclPayload: root.Payload, + AclPayloadId: root.Id, + SpaceSettingsPayload: spaceSettingsRoot.RawChange, + SpaceSettingsPayloadId: spaceSettingsRoot.Id, + } _, err = cl.PushSpace(ctx, &spacesyncproto.PushSpaceRequest{ - SpaceHeader: header, - AclPayload: root.Payload, - AclPayloadId: root.Id, + Payload: spacePayload, }) return } diff --git a/common/commonspace/service.go b/common/commonspace/service.go index 6947fa83..dc4d4b83 100644 --- a/common/commonspace/service.go +++ b/common/commonspace/service.go @@ -15,6 +15,7 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/net/pool" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/nodeconf" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/aclrecordproto" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/treechangeproto" ) const CName = "common.commonspace" @@ -161,12 +162,17 @@ func (s *service) getSpaceStorageFromRemote(ctx context.Context, id string) (st if err != nil { return } + st, err = s.storageProvider.CreateSpaceStorage(storage.SpaceStorageCreatePayload{ AclWithId: &aclrecordproto.RawACLRecordWithId{ - Payload: res.AclPayload, - Id: res.AclPayloadId, + Payload: res.Payload.AclPayload, + Id: res.Payload.AclPayloadId, }, - SpaceHeaderWithId: res.SpaceHeader, + SpaceSettingsWithId: &treechangeproto.RawTreeChangeWithId{ + RawChange: res.Payload.SpaceSettingsPayload, + Id: res.Payload.SpaceSettingsPayloadId, + }, + SpaceHeaderWithId: res.Payload.SpaceHeader, }) return } diff --git a/common/commonspace/space.go b/common/commonspace/space.go index 8b72008d..c63f854c 100644 --- a/common/commonspace/space.go +++ b/common/commonspace/space.go @@ -48,9 +48,11 @@ type SpaceDerivePayload struct { } type SpaceDescription struct { - SpaceHeader *spacesyncproto.RawSpaceHeaderWithId - AclId string - AclPayload []byte + SpaceHeader *spacesyncproto.RawSpaceHeaderWithId + AclId string + AclPayload []byte + SpaceSettingsId string + SpaceSettingsPayload []byte } func NewSpaceId(id string, repKey uint64) string { @@ -62,7 +64,7 @@ type Space interface { Init(ctx context.Context) error StoredIds() []string - Description() SpaceDescription + Description() (SpaceDescription, error) SpaceSyncRpc() RpcHandler @@ -99,13 +101,25 @@ func (s *space) Id() string { return s.id } -func (s *space) Description() SpaceDescription { +func (s *space) Description() (desc SpaceDescription, err error) { root := s.aclList.Root() - return SpaceDescription{ - SpaceHeader: s.header, - AclId: root.Id, - AclPayload: root.Payload, + settingsStorage, err := s.storage.TreeStorage(s.storage.SpaceSettingsId()) + if err != nil { + return } + settingsRoot, err := settingsStorage.Root() + if err != nil { + return + } + + desc = SpaceDescription{ + SpaceHeader: s.header, + AclId: root.Id, + AclPayload: root.Payload, + SpaceSettingsId: settingsRoot.Id, + SpaceSettingsPayload: settingsRoot.RawChange, + } + return } func (s *space) Init(ctx context.Context) (err error) { diff --git a/common/commonspace/synctree/synctree.go b/common/commonspace/synctree/synctree.go index c3d09d96..cb549bd5 100644 --- a/common/commonspace/synctree/synctree.go +++ b/common/commonspace/synctree/synctree.go @@ -171,7 +171,6 @@ func BuildSyncTreeOrGetRemote(ctx context.Context, id string, deps BuildDeps) (t } func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t tree.ObjectTree, err error) { - t, err = buildObjectTree(deps.TreeStorage, deps.AclList) if err != nil { return @@ -204,7 +203,7 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t tr } func (s *SyncTree) AddContent(ctx context.Context, content tree.SignableChangeContent) (res tree.AddResult, err error) { - if s.isClosed { + if s.isClosed { // checkAlive err err = ErrSyncTreeClosed return } diff --git a/node/nodespace/rpchandler.go b/node/nodespace/rpchandler.go index da8deca1..901f0a3e 100644 --- a/node/nodespace/rpchandler.go +++ b/node/nodespace/rpchandler.go @@ -19,20 +19,31 @@ func (r *rpcHandler) PullSpace(ctx context.Context, request *spacesyncproto.Pull return } - description := sp.Description() + spaceDesc, err := sp.Description() + if err != nil { + err = spacesyncproto.ErrUnexpected + return + } + resp = &spacesyncproto.PullSpaceResponse{ - SpaceHeader: description.SpaceHeader, - AclPayload: description.AclPayload, - AclPayloadId: description.AclId, + Payload: &spacesyncproto.SpacePayload{ + SpaceHeader: spaceDesc.SpaceHeader, + AclPayloadId: spaceDesc.AclId, + AclPayload: spaceDesc.AclPayload, + SpaceSettingsPayload: spaceDesc.SpaceSettingsPayload, + SpaceSettingsPayloadId: spaceDesc.SpaceSettingsId, + }, } return } func (r *rpcHandler) PushSpace(ctx context.Context, req *spacesyncproto.PushSpaceRequest) (resp *spacesyncproto.PushSpaceResponse, err error) { description := commonspace.SpaceDescription{ - SpaceHeader: req.SpaceHeader, - AclId: req.AclPayloadId, - AclPayload: req.AclPayload, + SpaceHeader: req.Payload.SpaceHeader, + AclId: req.Payload.AclPayloadId, + AclPayload: req.Payload.AclPayload, + SpaceSettingsPayload: req.Payload.SpaceSettingsPayload, + SpaceSettingsId: req.Payload.SpaceSettingsPayloadId, } err = r.s.AddSpace(ctx, description) if err != nil { From 72d83ca171ce6f5c1d7213cc216e1d3ca822a2dd Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 11 Nov 2022 12:37:45 +0100 Subject: [PATCH 03/27] WIP document deletion --- client/storage/keys.go | 22 ++-- client/storage/treestorage.go | 42 +++++++ common/commonspace/deletionservice/service.go | 28 ----- common/commonspace/payloads.go | 73 +++++++++--- common/commonspace/settingsservice/service.go | 22 ++++ common/commonspace/space.go | 1 + common/commonspace/synctree/synctree.go | 110 ++++++++++++------ common/pkg/acl/storage/inmemory.go | 4 + common/pkg/acl/storage/treestorage.go | 1 + common/pkg/acl/tree/changebuilder.go | 2 +- common/pkg/acl/tree/objecttree.go | 7 +- common/pkg/acl/tree/objecttree_test.go | 2 +- common/pkg/acl/tree/objecttreefactory.go | 2 +- node/storage/keys.go | 7 ++ node/storage/treestorage.go | 33 ++++++ 15 files changed, 263 insertions(+), 93 deletions(-) delete mode 100644 common/commonspace/deletionservice/service.go create mode 100644 common/commonspace/settingsservice/service.go diff --git a/client/storage/keys.go b/client/storage/keys.go index ddf187ac..45eb8cb0 100644 --- a/client/storage/keys.go +++ b/client/storage/keys.go @@ -31,18 +31,20 @@ func (a aclKeys) RawRecordKey(id string) []byte { } type treeKeys struct { - id string - spaceId string - headsKey []byte - rootKey []byte + id string + spaceId string + headsKey []byte + rootKey []byte + rawChangePrefix []byte } func newTreeKeys(spaceId, id string) treeKeys { return treeKeys{ - id: id, - spaceId: spaceId, - headsKey: storage.JoinStringsToBytes("space", spaceId, "t", id, "heads"), - rootKey: storage.JoinStringsToBytes("space", spaceId, "t", "rootId", id), + id: id, + spaceId: spaceId, + headsKey: storage.JoinStringsToBytes("space", spaceId, "t", id, "heads"), + rootKey: storage.JoinStringsToBytes("space", spaceId, "t", "rootId", id), + rawChangePrefix: storage.JoinStringsToBytes("space", spaceId, "t", id), } } @@ -58,6 +60,10 @@ func (t treeKeys) RawChangeKey(id string) []byte { return storage.JoinStringsToBytes("space", t.spaceId, "t", t.id, id) } +func (t treeKeys) RawChangePrefix() []byte { + return t.rawChangePrefix +} + type spaceKeys struct { headerKey []byte treePrefixKey []byte diff --git a/client/storage/treestorage.go b/client/storage/treestorage.go index 2c555555..d74d376c 100644 --- a/client/storage/treestorage.go +++ b/client/storage/treestorage.go @@ -136,3 +136,45 @@ func (t *treeStorage) GetRawChange(ctx context.Context, id string) (raw *treecha func (t *treeStorage) HasChange(ctx context.Context, id string) (bool, error) { return hasDB(t.db, t.keys.RawChangeKey(id)), nil } + +func (t *treeStorage) Delete() (err error) { + storedKeys, err := t.storedKeys() + if err != nil { + return + } + err = t.db.Update(func(txn *badger.Txn) error { + for _, k := range storedKeys { + err = txn.Delete(k) + if err != nil { + return err + } + } + return nil + }) + return +} + +func (t *treeStorage) storedKeys() (keys [][]byte, err error) { + err = t.db.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.PrefetchValues = false + opts.Prefix = t.keys.RawChangePrefix() + + it := txn.NewIterator(opts) + defer it.Close() + + for it.Rewind(); it.Valid(); it.Next() { + item := it.Item() + key := item.Key() + // if it is a heads key + if len(key) <= len(t.keys.HeadsKey()) { + continue + } + keyCopy := make([]byte, 0, len(key)) + keyCopy = item.KeyCopy(key) + keys = append(keys, keyCopy) + } + return nil + }) + return +} diff --git a/common/commonspace/deletionservice/service.go b/common/commonspace/deletionservice/service.go deleted file mode 100644 index f6f715a2..00000000 --- a/common/commonspace/deletionservice/service.go +++ /dev/null @@ -1,28 +0,0 @@ -package deletionservice - -import ( - "github.com/anytypeio/go-anytype-infrastructure-experiments/common/account" - "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" -) - -type Service interface { -} - -const deletionChangeType = "space.deletionlist" - -type service struct { - account account.Service -} - -func New() Service { - return nil -} - -func deriveDeletionTreePayload(account account.Service, spaceId string) tree.ObjectTreeCreatePayload { - return tree.ObjectTreeCreatePayload{ - SignKey: account.Account().SignKey, - ChangeType: deletionChangeType, - SpaceId: spaceId, - Identity: account.Account().Identity, - } -} diff --git a/common/commonspace/payloads.go b/common/commonspace/payloads.go index 843d597c..cd6a4c1f 100644 --- a/common/commonspace/payloads.go +++ b/common/commonspace/payloads.go @@ -3,7 +3,9 @@ package commonspace import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" - aclrecordproto2 "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/aclrecordproto" + aclrecordproto "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/aclrecordproto" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/common" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/util/cid" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/util/keys/asymmetric/signingkey" "hash/fnv" @@ -11,6 +13,11 @@ import ( "time" ) +const ( + SpaceSettingsChangeType = "reserved.spacesettings" + SpaceDerivationScheme = "derivation.standard" +) + func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload storage.SpaceStorageCreatePayload, err error) { // unmarshalling signing and encryption keys identity, err := payload.SigningKey.GetPublic().Raw() @@ -23,8 +30,8 @@ func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload st } // preparing header and space id - bytes := make([]byte, 32) - _, err = rand.Read(bytes) + spaceHeaderSeed := make([]byte, 32) + _, err = rand.Read(spaceHeaderSeed) if err != nil { return } @@ -33,7 +40,7 @@ func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload st Timestamp: time.Now().UnixNano(), SpaceType: payload.SpaceType, ReplicationKey: payload.ReplicationKey, - Seed: bytes, + Seed: spaceHeaderSeed, } marshalled, err := header.Marshal() if err != nil { @@ -68,12 +75,11 @@ func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload st } // preparing acl - aclRoot := &aclrecordproto2.ACLRoot{ + aclRoot := &aclrecordproto.ACLRoot{ Identity: identity, EncryptionKey: encPubKey, SpaceId: spaceId, EncryptedReadKey: encReadKey, - DerivationScheme: "", CurrentReadKeyHash: readKeyHash, Timestamp: time.Now().UnixNano(), } @@ -82,10 +88,31 @@ func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload st return } + builder := tree.NewChangeBuilder(common.NewKeychain(), nil) + spaceSettingsSeed := make([]byte, 32) + _, err = rand.Read(spaceSettingsSeed) + if err != nil { + return + } + + _, settingsRoot, err := builder.BuildInitialContent(tree.InitialContent{ + AclHeadId: rawWithId.Id, + Identity: aclRoot.Identity, + SigningKey: payload.SigningKey, + SpaceId: spaceId, + Seed: spaceSettingsSeed, + ChangeType: SpaceSettingsChangeType, + Timestamp: time.Now().UnixNano(), + }) + if err != nil { + return + } + // creating storage storagePayload = storage.SpaceStorageCreatePayload{ - AclWithId: rawWithId, - SpaceHeaderWithId: rawHeaderWithId, + AclWithId: rawWithId, + SpaceHeaderWithId: rawHeaderWithId, + SpaceSettingsWithId: settingsRoot, } return } @@ -144,7 +171,7 @@ func storagePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload st } // deriving and encrypting read key - readKey, err := aclrecordproto2.ACLReadKeyDerive(signPrivKey, encPrivKey) + readKey, err := aclrecordproto.ACLReadKeyDerive(signPrivKey, encPrivKey) if err != nil { return } @@ -160,29 +187,41 @@ func storagePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload st } // preparing acl - aclRoot := &aclrecordproto2.ACLRoot{ + aclRoot := &aclrecordproto.ACLRoot{ Identity: identity, EncryptionKey: encPubKey, SpaceId: spaceId, EncryptedReadKey: encReadKey, - DerivationScheme: "", + DerivationScheme: SpaceDerivationScheme, CurrentReadKeyHash: readKeyHash, - Timestamp: time.Now().UnixNano(), } rawWithId, err := marshalACLRoot(aclRoot, payload.SigningKey) if err != nil { return } + builder := tree.NewChangeBuilder(common.NewKeychain(), nil) + _, settingsRoot, err := builder.BuildInitialContent(tree.InitialContent{ + AclHeadId: rawWithId.Id, + Identity: aclRoot.Identity, + SigningKey: payload.SigningKey, + SpaceId: spaceId, + ChangeType: SpaceSettingsChangeType, + }) + if err != nil { + return + } + // creating storage storagePayload = storage.SpaceStorageCreatePayload{ - AclWithId: rawWithId, - SpaceHeaderWithId: rawHeaderWithId, + AclWithId: rawWithId, + SpaceHeaderWithId: rawHeaderWithId, + SpaceSettingsWithId: settingsRoot, } return } -func marshalACLRoot(aclRoot *aclrecordproto2.ACLRoot, key signingkey.PrivKey) (rawWithId *aclrecordproto2.RawACLRecordWithId, err error) { +func marshalACLRoot(aclRoot *aclrecordproto.ACLRoot, key signingkey.PrivKey) (rawWithId *aclrecordproto.RawACLRecordWithId, err error) { marshalledRoot, err := aclRoot.Marshal() if err != nil { return @@ -191,7 +230,7 @@ func marshalACLRoot(aclRoot *aclrecordproto2.ACLRoot, key signingkey.PrivKey) (r if err != nil { return } - raw := &aclrecordproto2.RawACLRecord{ + raw := &aclrecordproto.RawACLRecord{ Payload: marshalledRoot, Signature: signature, } @@ -203,7 +242,7 @@ func marshalACLRoot(aclRoot *aclrecordproto2.ACLRoot, key signingkey.PrivKey) (r if err != nil { return } - rawWithId = &aclrecordproto2.RawACLRecordWithId{ + rawWithId = &aclrecordproto.RawACLRecordWithId{ Payload: marshalledRaw, Id: aclHeadId, } diff --git a/common/commonspace/settingsservice/service.go b/common/commonspace/settingsservice/service.go new file mode 100644 index 00000000..02c83bf5 --- /dev/null +++ b/common/commonspace/settingsservice/service.go @@ -0,0 +1,22 @@ +package settingsservice + +import ( + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/account" +) + +type Service interface { +} + +const deletionChangeType = "space.deletionlist" + +type service struct { + account account.Service +} + +func New() Service { + return nil +} + +type DeletedDocumentNotifiable interface { + NotifyDeleted(id string) +} diff --git a/common/commonspace/space.go b/common/commonspace/space.go index c63f854c..ef6845f4 100644 --- a/common/commonspace/space.go +++ b/common/commonspace/space.go @@ -141,6 +141,7 @@ func (s *space) Init(ctx context.Context) (err error) { if err != nil { return } + s.aclList = syncacl.NewSyncACL(aclList, s.syncService.StreamPool()) objectGetter := newCommonSpaceGetter(s.id, s.aclList, s.cache) s.syncService.Init(objectGetter) diff --git a/common/commonspace/synctree/synctree.go b/common/commonspace/synctree/synctree.go index cb549bd5..09a3a403 100644 --- a/common/commonspace/synctree/synctree.go +++ b/common/commonspace/synctree/synctree.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/logger" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/diffservice" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsservice" spacestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/syncservice" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/syncservice/synchandler" @@ -20,15 +21,20 @@ import ( "go.uber.org/zap" ) -var ErrSyncTreeClosed = errors.New("sync tree is closed") +var ( + ErrSyncTreeClosed = errors.New("sync tree is closed") + ErrSyncTreeDeleted = errors.New("sync tree is deleted") +) // SyncTree sends head updates to sync service and also sends new changes to update listener type SyncTree struct { tree.ObjectTree synchandler.SyncHandler - syncClient SyncClient - listener updatelistener.UpdateListener - isClosed bool + syncClient SyncClient + listener updatelistener.UpdateListener + deletedNotifiable settingsservice.DeletedDocumentNotifiable + isClosed bool + isDeleted bool } var log = logger.NewNamed("commonspace.synctree").Sugar() @@ -39,25 +45,27 @@ var buildObjectTree = tree.BuildObjectTree var createSyncClient = newSyncClient type CreateDeps struct { - SpaceId string - Payload tree.ObjectTreeCreatePayload - Configuration nodeconf.Configuration - HeadNotifiable diffservice.HeadNotifiable - StreamPool syncservice.StreamPool - Listener updatelistener.UpdateListener - AclList list.ACLList - CreateStorage storage.TreeStorageCreatorFunc + SpaceId string + Payload tree.ObjectTreeCreatePayload + Configuration nodeconf.Configuration + HeadNotifiable diffservice.HeadNotifiable + StreamPool syncservice.StreamPool + Listener updatelistener.UpdateListener + AclList list.ACLList + CreateStorage storage.TreeStorageCreatorFunc + DeletedNotifiable settingsservice.DeletedDocumentNotifiable } type BuildDeps struct { - SpaceId string - StreamPool syncservice.StreamPool - Configuration nodeconf.Configuration - HeadNotifiable diffservice.HeadNotifiable - Listener updatelistener.UpdateListener - AclList list.ACLList - SpaceStorage spacestorage.SpaceStorage - TreeStorage storage.TreeStorage + SpaceId string + StreamPool syncservice.StreamPool + Configuration nodeconf.Configuration + HeadNotifiable diffservice.HeadNotifiable + Listener updatelistener.UpdateListener + AclList list.ACLList + SpaceStorage spacestorage.SpaceStorage + TreeStorage storage.TreeStorage + DeletedNotifiable settingsservice.DeletedDocumentNotifiable } func DeriveSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, err error) { @@ -72,9 +80,10 @@ func DeriveSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, er sharedFactory, deps.Configuration) syncTree := &SyncTree{ - ObjectTree: t, - syncClient: syncClient, - listener: deps.Listener, + ObjectTree: t, + syncClient: syncClient, + listener: deps.Listener, + deletedNotifiable: deps.DeletedNotifiable, } syncHandler := newSyncTreeHandler(syncTree, syncClient) syncTree.SyncHandler = syncHandler @@ -97,9 +106,10 @@ func CreateSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, er GetRequestFactory(), deps.Configuration) syncTree := &SyncTree{ - ObjectTree: t, - syncClient: syncClient, - listener: deps.Listener, + ObjectTree: t, + syncClient: syncClient, + listener: deps.Listener, + deletedNotifiable: deps.DeletedNotifiable, } syncHandler := newSyncTreeHandler(syncTree, syncClient) syncTree.SyncHandler = syncHandler @@ -182,9 +192,10 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t tr GetRequestFactory(), deps.Configuration) syncTree := &SyncTree{ - ObjectTree: t, - syncClient: syncClient, - listener: deps.Listener, + ObjectTree: t, + syncClient: syncClient, + listener: deps.Listener, + deletedNotifiable: deps.DeletedNotifiable, } syncHandler := newSyncTreeHandler(syncTree, syncClient) syncTree.SyncHandler = syncHandler @@ -203,8 +214,7 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t tr } func (s *SyncTree) AddContent(ctx context.Context, content tree.SignableChangeContent) (res tree.AddResult, err error) { - if s.isClosed { // checkAlive err - err = ErrSyncTreeClosed + if err = s.checkAlive(); err != nil { return } res, err = s.ObjectTree.AddContent(ctx, content) @@ -217,8 +227,7 @@ func (s *SyncTree) AddContent(ctx context.Context, content tree.SignableChangeCo } func (s *SyncTree) AddRawChanges(ctx context.Context, changes ...*treechangeproto.RawTreeChangeWithId) (res tree.AddResult, err error) { - if s.isClosed { - err = ErrSyncTreeClosed + if err = s.checkAlive(); err != nil { return } res, err = s.ObjectTree.AddRawChanges(ctx, changes...) @@ -242,15 +251,44 @@ func (s *SyncTree) AddRawChanges(ctx context.Context, changes ...*treechangeprot return } +func (s *SyncTree) Delete() (err error) { + log.With("id", s.ID()).Debug("deleting sync tree") + s.Lock() + defer func() { + s.Unlock() + if err == nil { + s.deletedNotifiable.NotifyDeleted(s.ID()) + } + }() + if err = s.checkAlive(); err != nil { + return + } + err = s.ObjectTree.Delete() + if err != nil { + return + } + s.isDeleted = true + + return +} + func (s *SyncTree) Close() (err error) { log.With("id", s.ID()).Debug("closing sync tree") s.Lock() defer s.Unlock() - log.With("id", s.ID()).Debug("taken lock on sync tree") - if s.isClosed { - err = ErrSyncTreeClosed + if err = s.checkAlive(); err != nil { return } s.isClosed = true return } + +func (s *SyncTree) checkAlive() (err error) { + if s.isClosed { + err = ErrSyncTreeClosed + } + if s.isDeleted { + err = ErrSyncTreeDeleted + } + return +} diff --git a/common/pkg/acl/storage/inmemory.go b/common/pkg/acl/storage/inmemory.go index 273f6c83..a59eeff3 100644 --- a/common/pkg/acl/storage/inmemory.go +++ b/common/pkg/acl/storage/inmemory.go @@ -157,6 +157,10 @@ func (t *inMemoryTreeStorage) GetRawChange(ctx context.Context, changeId string) return nil, fmt.Errorf("could not get change with id: %s", changeId) } +func (t *inMemoryTreeStorage) Delete() error { + return nil +} + type inMemoryStorageProvider struct { objects map[string]TreeStorage sync.RWMutex diff --git a/common/pkg/acl/storage/treestorage.go b/common/pkg/acl/storage/treestorage.go index 549e872a..dcb99fd0 100644 --- a/common/pkg/acl/storage/treestorage.go +++ b/common/pkg/acl/storage/treestorage.go @@ -14,6 +14,7 @@ type TreeStorage interface { AddRawChange(change *treechangeproto.RawTreeChangeWithId) error GetRawChange(ctx context.Context, id string) (*treechangeproto.RawTreeChangeWithId, error) HasChange(ctx context.Context, id string) (bool, error) + Delete() error } type TreeStorageCreatorFunc = func(payload TreeStorageCreatePayload) (TreeStorage, error) diff --git a/common/pkg/acl/tree/changebuilder.go b/common/pkg/acl/tree/changebuilder.go index 05d37721..a3c9d3e2 100644 --- a/common/pkg/acl/tree/changebuilder.go +++ b/common/pkg/acl/tree/changebuilder.go @@ -48,7 +48,7 @@ type changeBuilder struct { keys *common.Keychain } -func newChangeBuilder(keys *common.Keychain, rootChange *treechangeproto.RawTreeChangeWithId) ChangeBuilder { +func NewChangeBuilder(keys *common.Keychain, rootChange *treechangeproto.RawTreeChangeWithId) ChangeBuilder { return &changeBuilder{keys: keys, rootChange: rootChange} } diff --git a/common/pkg/acl/tree/objecttree.go b/common/pkg/acl/tree/objecttree.go index a89b3a29..52bbc36b 100644 --- a/common/pkg/acl/tree/objecttree.go +++ b/common/pkg/acl/tree/objecttree.go @@ -57,6 +57,7 @@ type ObjectTree interface { AddContent(ctx context.Context, content SignableChangeContent) (AddResult, error) AddRawChanges(ctx context.Context, changes ...*treechangeproto.RawTreeChangeWithId) (AddResult, error) + Delete() error Close() error } @@ -100,7 +101,7 @@ func defaultObjectTreeDeps( aclList list2.ACLList) objectTreeDeps { keychain := common.NewKeychain() - changeBuilder := newChangeBuilder(keychain, rootChange) + changeBuilder := NewChangeBuilder(keychain, rootChange) treeBuilder := newTreeBuilder(treeStorage, changeBuilder) return objectTreeDeps{ changeBuilder: changeBuilder, @@ -508,6 +509,10 @@ func (ot *objectTree) Close() error { return nil } +func (ot *objectTree) Delete() error { + return nil +} + func (ot *objectTree) SnapshotPath() []string { // TODO: Add error as return parameter if ot.snapshotPathIsActual() { diff --git a/common/pkg/acl/tree/objecttree_test.go b/common/pkg/acl/tree/objecttree_test.go index da9f2364..337f7903 100644 --- a/common/pkg/acl/tree/objecttree_test.go +++ b/common/pkg/acl/tree/objecttree_test.go @@ -116,7 +116,7 @@ func prepareTreeContext(t *testing.T, aclList list.ACLList) testTreeContext { treeStorage := changeCreator.createNewTreeStorage("0", aclList.Head().Id) root, _ := treeStorage.Root() changeBuilder := &mockChangeBuilder{ - originalBuilder: newChangeBuilder(nil, root), + originalBuilder: NewChangeBuilder(nil, root), } deps := objectTreeDeps{ changeBuilder: changeBuilder, diff --git a/common/pkg/acl/tree/objecttreefactory.go b/common/pkg/acl/tree/objecttreefactory.go index 26912377..58e6ca65 100644 --- a/common/pkg/acl/tree/objecttreefactory.go +++ b/common/pkg/acl/tree/objecttreefactory.go @@ -71,7 +71,7 @@ func createObjectTree( Seed: seed, } - _, raw, err := newChangeBuilder(common.NewKeychain(), nil).BuildInitialContent(cnt) + _, raw, err := NewChangeBuilder(common.NewKeychain(), nil).BuildInitialContent(cnt) if err != nil { return } diff --git a/node/storage/keys.go b/node/storage/keys.go index 77b58e22..5dabc5da 100644 --- a/node/storage/keys.go +++ b/node/storage/keys.go @@ -1,6 +1,7 @@ package storage import ( + "fmt" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/storage" "strings" ) @@ -25,6 +26,7 @@ func (a aclKeys) RawRecordKey(id string) []byte { type treeKeys struct { id string + prefix string headsKey []byte } @@ -32,6 +34,7 @@ func newTreeKeys(id string) treeKeys { return treeKeys{ id: id, headsKey: storage.JoinStringsToBytes("t", id, "heads"), + prefix: fmt.Sprintf("t/%s", id), } } @@ -43,6 +46,10 @@ func (t treeKeys) RawChangeKey(id string) []byte { return storage.JoinStringsToBytes("t", t.id, id) } +func (t treeKeys) isTreeRecordKey(key string) bool { + return strings.HasPrefix(key, t.prefix) && !strings.HasSuffix(key, "/heads") +} + type spaceKeys struct { headerKey []byte } diff --git a/node/storage/treestorage.go b/node/storage/treestorage.go index 77234d2f..5ab47451 100644 --- a/node/storage/treestorage.go +++ b/node/storage/treestorage.go @@ -135,3 +135,36 @@ func (t *treeStorage) GetRawChange(ctx context.Context, id string) (raw *treecha func (t *treeStorage) HasChange(ctx context.Context, id string) (bool, error) { return t.db.Has(t.keys.RawChangeKey(id)) } + +func (t *treeStorage) Delete() (err error) { + storedKeys, err := t.storedKeys() + if err != nil { + return + } + for _, k := range storedKeys { + err = t.db.Delete(k) + if err != nil { + return + } + } + return +} + +func (t *treeStorage) storedKeys() (keys [][]byte, err error) { + index := t.db.Items() + + key, _, err := index.Next() + for err == nil { + strKey := string(key) + if t.keys.isTreeRecordKey(strKey) { + keys = append(keys, key) + } + key, _, err = index.Next() + } + + if err != pogreb.ErrIterationDone { + return + } + err = nil + return +} From fed41ba3aa0d1864ea46f4ebddee2c9ae762f6ff Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Sat, 12 Nov 2022 13:29:00 +0100 Subject: [PATCH 04/27] Add non-encrypted changes and settings document logic --- client/clientspace/clientcache/treecache.go | 13 + common/commonspace/commongetter.go | 9 +- .../commonspace/settingsdocument/provider.go | 45 + .../settingsdocument/settingsdocument.go | 136 +++ common/commonspace/settingsservice/service.go | 22 - common/commonspace/space.go | 31 +- .../spacesyncproto/protos/spacesync.proto | 20 + .../spacesyncproto/spacesync.pb.go | 926 +++++++++++++++++- common/commonspace/synctree/synctree.go | 87 +- common/commonspace/treegetter/treegetter.go | 1 + common/pkg/acl/tree/changebuilder.go | 13 +- common/pkg/acl/tree/objecttree.go | 48 +- common/pkg/acl/tree/objecttreefactory.go | 9 +- common/pkg/acl/tree/signablecontent.go | 9 +- node/nodespace/nodecache/treecache.go | 13 + 15 files changed, 1231 insertions(+), 151 deletions(-) create mode 100644 common/commonspace/settingsdocument/provider.go create mode 100644 common/commonspace/settingsdocument/settingsdocument.go delete mode 100644 common/commonspace/settingsservice/service.go diff --git a/client/clientspace/clientcache/treecache.go b/client/clientspace/clientcache/treecache.go index ae3163cb..abdd931a 100644 --- a/client/clientspace/clientcache/treecache.go +++ b/client/clientspace/clientcache/treecache.go @@ -101,3 +101,16 @@ func (c *treeCache) GetDocument(ctx context.Context, spaceId, id string) (doc te func (c *treeCache) GetTree(ctx context.Context, spaceId, id string) (tr tree.ObjectTree, err error) { return c.GetDocument(ctx, spaceId, id) } + +func (c *treeCache) DeleteTree(ctx context.Context, spaceId, treeId string) (err error) { + tr, err := c.GetTree(ctx, spaceId, treeId) + if err != nil { + return + } + err = tr.Delete() + if err != nil { + return + } + _, err = c.cache.Remove(treeId) + return +} diff --git a/common/commonspace/commongetter.go b/common/commonspace/commongetter.go index da40bfff..6b6a857e 100644 --- a/common/commonspace/commongetter.go +++ b/common/commonspace/commongetter.go @@ -3,6 +3,7 @@ package commonspace import ( "context" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/objectgetter" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/syncacl" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter" ) @@ -11,13 +12,15 @@ type commonSpaceGetter struct { spaceId string aclList *syncacl.SyncACL treeGetter treegetter.TreeGetter + settings settingsdocument.SettingsDocument } -func newCommonSpaceGetter(spaceId string, aclList *syncacl.SyncACL, treeGetter treegetter.TreeGetter) objectgetter.ObjectGetter { +func newCommonSpaceGetter(spaceId string, aclList *syncacl.SyncACL, treeGetter treegetter.TreeGetter, settings settingsdocument.SettingsDocument) objectgetter.ObjectGetter { return &commonSpaceGetter{ spaceId: spaceId, aclList: aclList, treeGetter: treeGetter, + settings: settings, } } @@ -26,6 +29,10 @@ func (c *commonSpaceGetter) GetObject(ctx context.Context, objectId string) (obj obj = c.aclList return } + if c.settings.ID() == objectId { + obj = c.settings.(objectgetter.Object) + return + } t, err := c.treeGetter.GetTree(ctx, c.spaceId, objectId) if err != nil { return diff --git a/common/commonspace/settingsdocument/provider.go b/common/commonspace/settingsdocument/provider.go new file mode 100644 index 00000000..c25c3b8a --- /dev/null +++ b/common/commonspace/settingsdocument/provider.go @@ -0,0 +1,45 @@ +package settingsdocument + +import ( + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" + "github.com/gogo/protobuf/proto" +) + +type deletedIdsProvider interface { + ProvideIds(tr tree.ObjectTree, startId string) (ids []string, lastId string, err error) +} + +type provider struct{} + +func (p *provider) convert(decrypted []byte) (res any, err error) { + deleteChange := &spacesyncproto.SettingsData{} + err = proto.Unmarshal(decrypted, deleteChange) + if err != nil { + return nil, err + } + return deleteChange, nil +} + +func (p *provider) ProvideIds(tr tree.ObjectTree, startId string) (ids []string, lastId string, err error) { + processChange := func(change *tree.Change) bool { + // ignoring first change if startId is not "" + if change.Id == startId { + return true + } + deleteChange := change.Model.(*spacesyncproto.SettingsData) + for _, cnt := range deleteChange.Content { + if cnt.GetObjectDelete() != nil { + ids = append(ids, cnt.GetObjectDelete().GetId()) + } + } + lastId = change.Id + return true + } + if startId == "" { + err = tr.IterateFrom(tr.ID(), p.convert, processChange) + } else { + err = tr.IterateFrom(startId, p.convert, processChange) + } + return +} diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go new file mode 100644 index 00000000..5c9333e4 --- /dev/null +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -0,0 +1,136 @@ +package settingsdocument + +import ( + "context" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/account" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" + spacestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree/updatelistener" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" +) + +type DeletionState int + +const ( + DeletionStateQueued DeletionState = iota + DeletionStateDeleted +) + +type SettingsDocument interface { + tree.ObjectTree + DeleteObject(id string) (err error) +} + +type BuildTreeFunc func(ctx context.Context, id string, listener updatelistener.UpdateListener) (t tree.ObjectTree, err error) + +type Deps struct { + BuildFunc BuildTreeFunc + Account account.Service + TreeGetter treegetter.TreeGetter + Store spacestorage.SpaceStorage + prov deletedIdsProvider +} + +type settingsDocument struct { + tree.ObjectTree + account account.Service + spaceId string + deletionState map[string]DeletionState + treeGetter treegetter.TreeGetter + store spacestorage.SpaceStorage + lastChangeId string + prov deletedIdsProvider +} + +func NewSettingsDocument(ctx context.Context, deps Deps, spaceId string) (doc SettingsDocument, err error) { + s := &settingsDocument{ + account: deps.Account, + spaceId: spaceId, + deletionState: map[string]DeletionState{}, + treeGetter: deps.TreeGetter, + store: deps.Store, + } + s.ObjectTree, err = deps.BuildFunc(ctx, deps.Store.SpaceSettingsId(), s) + if err != nil { + return + } + // this is needed mainly for testing + if deps.prov == nil { + s.prov = &provider{} + } + doc = s + return +} + +func (s *settingsDocument) Update(tr tree.ObjectTree) { + ids, lastId, err := s.prov.ProvideIds(tr, s.lastChangeId) + if err != nil { + return + } + s.lastChangeId = lastId + s.toBeDeleted(ids) +} + +func (s *settingsDocument) Rebuild(tr tree.ObjectTree) { + ids, lastId, err := s.prov.ProvideIds(tr, "") + if err != nil { + return + } + s.lastChangeId = lastId + s.toBeDeleted(ids) +} + +func (s *settingsDocument) Init() { + s.Lock() + defer s.Unlock() + s.Rebuild(s) +} + +func (s *settingsDocument) toBeDeleted(ids []string) { + for _, id := range ids { + if state, exists := s.deletionState[id]; exists && state == DeletionStateDeleted { + continue + } + // if not already deleted + if _, err := s.store.TreeStorage(id); err == nil { + s.deletionState[id] = DeletionStateQueued + err := s.treeGetter.DeleteTree(context.Background(), s.spaceId, id) + if err != nil { + // TODO: some errors may tell us that the tree is actually deleted, so we should have more checks here + // TODO: add logging + continue + } + } + s.deletionState[id] = DeletionStateDeleted + } +} + +func (s *settingsDocument) DeleteObject(id string) (err error) { + content := &spacesyncproto.SpaceSettingsContent_ObjectDelete{ + ObjectDelete: &spacesyncproto.ObjectDelete{Id: id}, + } + change := &spacesyncproto.SettingsData{ + Content: []*spacesyncproto.SpaceSettingsContent{ + {content}, + }, + Snapshot: nil, + } + res, err := change.Marshal() + if err != nil { + return + } + s.Lock() + defer s.Unlock() + _, err = s.AddContent(context.Background(), tree.SignableChangeContent{ + Data: res, + Key: s.account.Account().SignKey, + Identity: s.account.Account().Identity, + IsSnapshot: false, + IsEncrypted: false, + }) + if err == nil { + s.Update(s) + } + return +} diff --git a/common/commonspace/settingsservice/service.go b/common/commonspace/settingsservice/service.go deleted file mode 100644 index 02c83bf5..00000000 --- a/common/commonspace/settingsservice/service.go +++ /dev/null @@ -1,22 +0,0 @@ -package settingsservice - -import ( - "github.com/anytypeio/go-anytype-infrastructure-experiments/common/account" -) - -type Service interface { -} - -const deletionChangeType = "space.deletionlist" - -type service struct { - account account.Service -} - -func New() Service { - return nil -} - -type DeletedDocumentNotifiable interface { - NotifyDeleted(id string) -} diff --git a/common/commonspace/space.go b/common/commonspace/space.go index ef6845f4..f9638cad 100644 --- a/common/commonspace/space.go +++ b/common/commonspace/space.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/account" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/diffservice" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/syncacl" @@ -82,13 +83,14 @@ type space struct { rpc *rpcHandler - syncService syncservice.SyncService - diffService diffservice.DiffService - storage storage.SpaceStorage - cache treegetter.TreeGetter - account account.Service - aclList *syncacl.SyncACL - configuration nodeconf.Configuration + syncService syncservice.SyncService + diffService diffservice.DiffService + storage storage.SpaceStorage + cache treegetter.TreeGetter + account account.Service + aclList *syncacl.SyncACL + configuration nodeconf.Configuration + settingsDocument settingsdocument.SettingsDocument isClosed atomic.Bool } @@ -142,8 +144,18 @@ func (s *space) Init(ctx context.Context) (err error) { return } + deps := settingsdocument.Deps{ + BuildFunc: s.BuildTree, + Account: s.account, + TreeGetter: s.cache, + Store: s.storage, + } + s.settingsDocument, err = settingsdocument.NewSettingsDocument(context.Background(), deps, s.id) + if err != nil { + return + } s.aclList = syncacl.NewSyncACL(aclList, s.syncService.StreamPool()) - objectGetter := newCommonSpaceGetter(s.id, s.aclList, s.cache) + objectGetter := newCommonSpaceGetter(s.id, s.aclList, s.cache, s.settingsDocument) s.syncService.Init(objectGetter) s.diffService.Init(initialIds) return nil @@ -231,6 +243,9 @@ func (s *space) Close() error { if err := s.syncService.Close(); err != nil { mError.Add(err) } + if err := s.settingsDocument.Close(); err != nil { + mError.Add(err) + } if err := s.aclList.Close(); err != nil { mError.Add(err) } diff --git a/common/commonspace/spacesyncproto/protos/spacesync.proto b/common/commonspace/spacesyncproto/protos/spacesync.proto index cf3b11ed..0582fffc 100644 --- a/common/commonspace/spacesyncproto/protos/spacesync.proto +++ b/common/commonspace/spacesyncproto/protos/spacesync.proto @@ -106,3 +106,23 @@ message RawSpaceHeaderWithId { bytes rawHeader = 1; string id = 2; } + +message SpaceSettingsContent { + oneof value { + ObjectDelete objectDelete = 1; + } +} + +message ObjectDelete { + string id = 1; +} + +message SpaceSettingsSnapshot { + repeated string deletedIds = 1; +} + +message SettingsData { + repeated SpaceSettingsContent content = 1; + SpaceSettingsSnapshot snapshot = 2; +} + diff --git a/common/commonspace/spacesyncproto/spacesync.pb.go b/common/commonspace/spacesyncproto/spacesync.pb.go index 07e96827..d0672626 100644 --- a/common/commonspace/spacesyncproto/spacesync.pb.go +++ b/common/commonspace/spacesyncproto/spacesync.pb.go @@ -824,6 +824,219 @@ func (m *RawSpaceHeaderWithId) GetId() string { return "" } +type SpaceSettingsContent struct { + // Types that are valid to be assigned to Value: + // + // *SpaceSettingsContent_ObjectDelete + Value isSpaceSettingsContent_Value `protobuf_oneof:"value"` +} + +func (m *SpaceSettingsContent) Reset() { *m = SpaceSettingsContent{} } +func (m *SpaceSettingsContent) String() string { return proto.CompactTextString(m) } +func (*SpaceSettingsContent) ProtoMessage() {} +func (*SpaceSettingsContent) Descriptor() ([]byte, []int) { + return fileDescriptor_80e49f1f4ac27799, []int{14} +} +func (m *SpaceSettingsContent) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SpaceSettingsContent) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SpaceSettingsContent.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SpaceSettingsContent) XXX_Merge(src proto.Message) { + xxx_messageInfo_SpaceSettingsContent.Merge(m, src) +} +func (m *SpaceSettingsContent) XXX_Size() int { + return m.Size() +} +func (m *SpaceSettingsContent) XXX_DiscardUnknown() { + xxx_messageInfo_SpaceSettingsContent.DiscardUnknown(m) +} + +var xxx_messageInfo_SpaceSettingsContent proto.InternalMessageInfo + +type isSpaceSettingsContent_Value interface { + isSpaceSettingsContent_Value() + MarshalTo([]byte) (int, error) + Size() int +} + +type SpaceSettingsContent_ObjectDelete struct { + ObjectDelete *ObjectDelete `protobuf:"bytes,1,opt,name=objectDelete,proto3,oneof" json:"objectDelete,omitempty"` +} + +func (*SpaceSettingsContent_ObjectDelete) isSpaceSettingsContent_Value() {} + +func (m *SpaceSettingsContent) GetValue() isSpaceSettingsContent_Value { + if m != nil { + return m.Value + } + return nil +} + +func (m *SpaceSettingsContent) GetObjectDelete() *ObjectDelete { + if x, ok := m.GetValue().(*SpaceSettingsContent_ObjectDelete); ok { + return x.ObjectDelete + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*SpaceSettingsContent) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*SpaceSettingsContent_ObjectDelete)(nil), + } +} + +type ObjectDelete struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (m *ObjectDelete) Reset() { *m = ObjectDelete{} } +func (m *ObjectDelete) String() string { return proto.CompactTextString(m) } +func (*ObjectDelete) ProtoMessage() {} +func (*ObjectDelete) Descriptor() ([]byte, []int) { + return fileDescriptor_80e49f1f4ac27799, []int{15} +} +func (m *ObjectDelete) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ObjectDelete) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ObjectDelete.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ObjectDelete) XXX_Merge(src proto.Message) { + xxx_messageInfo_ObjectDelete.Merge(m, src) +} +func (m *ObjectDelete) XXX_Size() int { + return m.Size() +} +func (m *ObjectDelete) XXX_DiscardUnknown() { + xxx_messageInfo_ObjectDelete.DiscardUnknown(m) +} + +var xxx_messageInfo_ObjectDelete proto.InternalMessageInfo + +func (m *ObjectDelete) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +type SpaceSettingsSnapshot struct { + DeletedIds []string `protobuf:"bytes,1,rep,name=deletedIds,proto3" json:"deletedIds,omitempty"` +} + +func (m *SpaceSettingsSnapshot) Reset() { *m = SpaceSettingsSnapshot{} } +func (m *SpaceSettingsSnapshot) String() string { return proto.CompactTextString(m) } +func (*SpaceSettingsSnapshot) ProtoMessage() {} +func (*SpaceSettingsSnapshot) Descriptor() ([]byte, []int) { + return fileDescriptor_80e49f1f4ac27799, []int{16} +} +func (m *SpaceSettingsSnapshot) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SpaceSettingsSnapshot) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SpaceSettingsSnapshot.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SpaceSettingsSnapshot) XXX_Merge(src proto.Message) { + xxx_messageInfo_SpaceSettingsSnapshot.Merge(m, src) +} +func (m *SpaceSettingsSnapshot) XXX_Size() int { + return m.Size() +} +func (m *SpaceSettingsSnapshot) XXX_DiscardUnknown() { + xxx_messageInfo_SpaceSettingsSnapshot.DiscardUnknown(m) +} + +var xxx_messageInfo_SpaceSettingsSnapshot proto.InternalMessageInfo + +func (m *SpaceSettingsSnapshot) GetDeletedIds() []string { + if m != nil { + return m.DeletedIds + } + return nil +} + +type SettingsData struct { + Content []*SpaceSettingsContent `protobuf:"bytes,1,rep,name=content,proto3" json:"content,omitempty"` + Snapshot *SpaceSettingsSnapshot `protobuf:"bytes,2,opt,name=snapshot,proto3" json:"snapshot,omitempty"` +} + +func (m *SettingsData) Reset() { *m = SettingsData{} } +func (m *SettingsData) String() string { return proto.CompactTextString(m) } +func (*SettingsData) ProtoMessage() {} +func (*SettingsData) Descriptor() ([]byte, []int) { + return fileDescriptor_80e49f1f4ac27799, []int{17} +} +func (m *SettingsData) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SettingsData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SettingsData.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SettingsData) XXX_Merge(src proto.Message) { + xxx_messageInfo_SettingsData.Merge(m, src) +} +func (m *SettingsData) XXX_Size() int { + return m.Size() +} +func (m *SettingsData) XXX_DiscardUnknown() { + xxx_messageInfo_SettingsData.DiscardUnknown(m) +} + +var xxx_messageInfo_SettingsData proto.InternalMessageInfo + +func (m *SettingsData) GetContent() []*SpaceSettingsContent { + if m != nil { + return m.Content + } + return nil +} + +func (m *SettingsData) GetSnapshot() *SpaceSettingsSnapshot { + if m != nil { + return m.Snapshot + } + return nil +} + func init() { proto.RegisterEnum("anySpace.ErrCodes", ErrCodes_name, ErrCodes_value) proto.RegisterType((*HeadSyncRange)(nil), "anySpace.HeadSyncRange") @@ -840,6 +1053,10 @@ func init() { proto.RegisterType((*SpaceHeader)(nil), "anySpace.SpaceHeader") proto.RegisterType((*RawSpaceHeader)(nil), "anySpace.RawSpaceHeader") proto.RegisterType((*RawSpaceHeaderWithId)(nil), "anySpace.RawSpaceHeaderWithId") + proto.RegisterType((*SpaceSettingsContent)(nil), "anySpace.SpaceSettingsContent") + proto.RegisterType((*ObjectDelete)(nil), "anySpace.ObjectDelete") + proto.RegisterType((*SpaceSettingsSnapshot)(nil), "anySpace.SpaceSettingsSnapshot") + proto.RegisterType((*SettingsData)(nil), "anySpace.SettingsData") } func init() { @@ -847,55 +1064,62 @@ func init() { } var fileDescriptor_80e49f1f4ac27799 = []byte{ - // 770 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0xcd, 0x4e, 0x23, 0x47, - 0x10, 0xf6, 0x0c, 0x06, 0xec, 0xb2, 0x31, 0xa6, 0x43, 0xc8, 0xc4, 0x89, 0x26, 0xd6, 0x1c, 0x22, - 0x2b, 0x07, 0x20, 0x4e, 0x94, 0x0b, 0x97, 0xfc, 0x60, 0x14, 0x2b, 0x22, 0xa0, 0x76, 0xa2, 0x48, - 0x51, 0x2e, 0xcd, 0x4c, 0x63, 0x4f, 0x34, 0x7f, 0x3b, 0xdd, 0x16, 0xcc, 0x61, 0xa5, 0x7d, 0x84, - 0x7d, 0x82, 0x95, 0xf6, 0x6d, 0xf6, 0xc8, 0x71, 0x8f, 0x2b, 0x78, 0x91, 0x55, 0x97, 0xe7, 0xd7, - 0x0c, 0x48, 0x7b, 0x19, 0x4f, 0x7d, 0x55, 0xf5, 0x55, 0xd5, 0xd7, 0x5d, 0x63, 0xf8, 0xde, 0x0e, - 0x7d, 0x3f, 0x0c, 0x44, 0xc4, 0x6c, 0x7e, 0x84, 0x4f, 0x91, 0x04, 0x76, 0x14, 0x87, 0x32, 0x3c, - 0xc2, 0xa7, 0x28, 0xd0, 0x43, 0x04, 0x48, 0x8b, 0x05, 0xc9, 0x4c, 0x61, 0xd6, 0x14, 0x76, 0x7e, - 0xe7, 0xcc, 0x99, 0x25, 0x81, 0x4d, 0x59, 0x30, 0xe7, 0x84, 0x40, 0xf3, 0x3a, 0x0e, 0x7d, 0x43, - 0x1b, 0x6a, 0xa3, 0x26, 0xc5, 0x77, 0xd2, 0x03, 0x5d, 0x86, 0x86, 0x8e, 0x88, 0x2e, 0x43, 0xb2, - 0x0f, 0x9b, 0x9e, 0xeb, 0xbb, 0xd2, 0xd8, 0x18, 0x6a, 0xa3, 0x1d, 0xba, 0x32, 0xac, 0x1b, 0xe8, - 0xe5, 0x54, 0x5c, 0x2c, 0x3d, 0xa9, 0xb8, 0x16, 0x4c, 0x2c, 0x90, 0xab, 0x4b, 0xf1, 0x9d, 0x9c, - 0x40, 0x8b, 0x7b, 0xdc, 0xe7, 0x81, 0x14, 0x86, 0x3e, 0xdc, 0x18, 0x75, 0xc6, 0xdf, 0x1c, 0x66, - 0xdd, 0x1c, 0x56, 0xf3, 0x27, 0xab, 0x38, 0x9a, 0x27, 0xa8, 0xc2, 0x76, 0xb8, 0x0c, 0xf2, 0xc2, - 0x68, 0x58, 0x27, 0xf0, 0x79, 0x6d, 0xa2, 0xea, 0xdb, 0x75, 0xb0, 0x7a, 0x9b, 0xea, 0xae, 0x83, - 0xfd, 0x70, 0xe6, 0xe0, 0x24, 0x6d, 0x8a, 0xef, 0xd6, 0x7f, 0xb0, 0x5b, 0x24, 0xbf, 0x58, 0x72, - 0x21, 0x89, 0x01, 0xdb, 0x28, 0xd8, 0x34, 0xcb, 0xcd, 0x4c, 0x72, 0x04, 0x5b, 0xb1, 0x52, 0x29, - 0x6b, 0xfd, 0x8b, 0x9a, 0xd6, 0x95, 0x9f, 0xa6, 0x61, 0xd6, 0x19, 0xf4, 0x4b, 0xad, 0x45, 0x61, - 0x20, 0x38, 0x19, 0xc3, 0x76, 0x8c, 0x6d, 0x0a, 0x43, 0x43, 0x16, 0xe3, 0x29, 0x01, 0x68, 0x16, - 0x68, 0xbd, 0x84, 0xbd, 0x8b, 0xab, 0xff, 0xb9, 0x2d, 0x95, 0xf3, 0x9c, 0x0b, 0xc1, 0xe6, 0xfc, - 0x99, 0x3e, 0x0d, 0x55, 0x22, 0xf2, 0x92, 0x69, 0x36, 0x6b, 0x66, 0x2a, 0x4f, 0xc4, 0x12, 0x2f, - 0x64, 0x0e, 0x6a, 0xd8, 0xa5, 0x99, 0x49, 0x06, 0xd0, 0x0a, 0xb1, 0xc4, 0xd4, 0x31, 0x9a, 0x98, - 0x94, 0xdb, 0xd6, 0x29, 0xf4, 0x2f, 0x97, 0x62, 0x81, 0x3d, 0x66, 0x2a, 0x1d, 0x17, 0x4c, 0xaa, - 0x7a, 0x67, 0x7c, 0x50, 0x8c, 0x81, 0xcf, 0xcb, 0x95, 0x37, 0xaf, 0x60, 0x7d, 0x06, 0x7b, 0x25, - 0x96, 0x95, 0x1a, 0x96, 0xa5, 0xa8, 0x3d, 0xaf, 0x42, 0xbd, 0x76, 0x6e, 0xd6, 0x44, 0x25, 0xe6, - 0x31, 0xa9, 0x8c, 0x9f, 0x5e, 0xff, 0x95, 0x0e, 0xdd, 0xb2, 0x87, 0xfc, 0x0c, 0x1d, 0x54, 0x4c, - 0xa9, 0xce, 0xe3, 0x94, 0xc6, 0x2c, 0x68, 0x28, 0xbb, 0x99, 0x15, 0xfe, 0x7f, 0x5c, 0xb9, 0x98, - 0x3a, 0xb4, 0x9c, 0x42, 0x4c, 0x00, 0x66, 0x7b, 0x29, 0x1f, 0x6a, 0xdd, 0xa5, 0x25, 0x84, 0x58, - 0xd0, 0x2d, 0xac, 0xe9, 0x4a, 0xf3, 0x36, 0xad, 0x60, 0x64, 0x0c, 0xfb, 0x48, 0x39, 0xe3, 0x52, - 0xba, 0xc1, 0x5c, 0x64, 0x6c, 0x4d, 0x64, 0xab, 0xf5, 0x91, 0x9f, 0xe0, 0xa0, 0x0e, 0x9f, 0x3a, - 0xc6, 0x26, 0x56, 0x78, 0xc2, 0x6b, 0xbd, 0xd5, 0xa0, 0x53, 0x1a, 0x49, 0x1d, 0xba, 0xeb, 0xf0, - 0x40, 0xba, 0x32, 0x49, 0xb7, 0x34, 0xb7, 0xc9, 0xd7, 0xd0, 0x96, 0xae, 0xcf, 0x85, 0x64, 0x7e, - 0x84, 0xa3, 0x6d, 0xd0, 0x02, 0x50, 0x5e, 0xac, 0xf1, 0x57, 0x12, 0xf1, 0x74, 0xac, 0x02, 0x20, - 0xdf, 0x42, 0x4f, 0xdd, 0x38, 0xd7, 0x66, 0xd2, 0x0d, 0x83, 0x3f, 0x78, 0x82, 0xd3, 0x34, 0xe9, - 0x1a, 0xaa, 0x36, 0x52, 0x70, 0xbe, 0xea, 0xba, 0x4b, 0xf1, 0xdd, 0xba, 0x84, 0x5e, 0x55, 0x78, - 0x32, 0x7c, 0x7c, 0x4e, 0xdd, 0xea, 0x39, 0xa8, 0x6e, 0xdc, 0x79, 0xc0, 0xe4, 0x32, 0xe6, 0xe9, - 0x31, 0x14, 0x80, 0x75, 0x0a, 0xfb, 0x75, 0x47, 0xa9, 0xb2, 0x62, 0x76, 0x53, 0x61, 0x2d, 0x80, - 0xf4, 0x16, 0xea, 0xd9, 0x2d, 0xfc, 0xee, 0x4f, 0x68, 0x4d, 0xe2, 0xf8, 0xb7, 0xd0, 0xe1, 0x82, - 0xf4, 0x00, 0xfe, 0x0e, 0xf8, 0x6d, 0xc4, 0x6d, 0xc9, 0x9d, 0x7e, 0x83, 0xf4, 0xd3, 0x9b, 0x75, - 0xee, 0x0a, 0xe1, 0x06, 0xf3, 0xbe, 0x46, 0x76, 0x53, 0xa1, 0x27, 0xb7, 0xae, 0x90, 0xa2, 0xaf, - 0x2b, 0x60, 0x12, 0xc7, 0x61, 0x7c, 0x71, 0x7d, 0x2d, 0xb8, 0xec, 0x3b, 0xe3, 0x37, 0x3a, 0x6c, - 0x62, 0x08, 0xf9, 0x05, 0x5a, 0xd9, 0xe2, 0x93, 0x2f, 0xeb, 0x3e, 0x06, 0xb8, 0x16, 0x83, 0x41, - 0xed, 0x77, 0x62, 0xb5, 0x0d, 0xa7, 0xd0, 0xce, 0x77, 0x8b, 0x94, 0x02, 0xd7, 0xd7, 0x76, 0xf0, - 0x55, 0xad, 0xaf, 0xcc, 0x92, 0x2e, 0x5a, 0x95, 0xa5, 0xba, 0xa1, 0x55, 0x96, 0xf5, 0xcd, 0x3c, - 0x83, 0xad, 0x99, 0x8c, 0x39, 0xf3, 0x49, 0x29, 0xec, 0xd1, 0xe7, 0x6b, 0xf0, 0x9c, 0x73, 0xa4, - 0x1d, 0x6b, 0xbf, 0xfe, 0xf8, 0xee, 0xde, 0xd4, 0xee, 0xee, 0x4d, 0xed, 0xc3, 0xbd, 0xa9, 0xbd, - 0x7e, 0x30, 0x1b, 0x77, 0x0f, 0x66, 0xe3, 0xfd, 0x83, 0xd9, 0xf8, 0x77, 0xf0, 0xf4, 0x5f, 0xde, - 0xd5, 0x16, 0xfe, 0xfc, 0xf0, 0x31, 0x00, 0x00, 0xff, 0xff, 0xaa, 0x60, 0x9b, 0x89, 0x17, 0x07, + // 882 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x56, 0x4f, 0x6f, 0x1b, 0x45, + 0x14, 0xf7, 0x6e, 0x9c, 0xc4, 0x7e, 0xd9, 0xba, 0xee, 0x90, 0x96, 0xc5, 0x45, 0x4b, 0x34, 0x07, + 0x14, 0x71, 0x48, 0x8a, 0x41, 0x80, 0x54, 0x0e, 0xd0, 0xda, 0x55, 0x2d, 0x54, 0x12, 0x8d, 0x41, + 0x48, 0x08, 0x0e, 0xd3, 0xdd, 0x89, 0xbd, 0x68, 0xbd, 0xb3, 0xec, 0x8c, 0x49, 0x7d, 0x40, 0xe2, + 0xc2, 0x9d, 0x4f, 0x80, 0xc4, 0xb7, 0xe1, 0xd8, 0x23, 0x47, 0x94, 0x7c, 0x11, 0x34, 0x6f, 0xff, + 0x3b, 0x9b, 0x4a, 0x5c, 0xd6, 0xf3, 0xfe, 0xfd, 0xde, 0x9f, 0x79, 0xf3, 0x93, 0xe1, 0x43, 0x5f, + 0xae, 0x56, 0x32, 0x56, 0x09, 0xf7, 0xc5, 0x29, 0x7e, 0xd5, 0x26, 0xf6, 0x93, 0x54, 0x6a, 0x79, + 0x8a, 0x5f, 0x55, 0x69, 0x4f, 0x50, 0x41, 0x7a, 0x3c, 0xde, 0xcc, 0x8d, 0x8e, 0xce, 0xe0, 0xce, + 0x73, 0xc1, 0x83, 0xf9, 0x26, 0xf6, 0x19, 0x8f, 0x17, 0x82, 0x10, 0xe8, 0x5e, 0xa4, 0x72, 0xe5, + 0x5a, 0x47, 0xd6, 0x71, 0x97, 0xe1, 0x99, 0x0c, 0xc0, 0xd6, 0xd2, 0xb5, 0x51, 0x63, 0x6b, 0x49, + 0x0e, 0x61, 0x37, 0x0a, 0x57, 0xa1, 0x76, 0x77, 0x8e, 0xac, 0xe3, 0x3b, 0x2c, 0x13, 0xe8, 0x25, + 0x0c, 0x4a, 0x28, 0xa1, 0xd6, 0x91, 0x36, 0x58, 0x4b, 0xae, 0x96, 0x88, 0xe5, 0x30, 0x3c, 0x93, + 0xc7, 0xd0, 0x13, 0x91, 0x58, 0x89, 0x58, 0x2b, 0xd7, 0x3e, 0xda, 0x39, 0x3e, 0x18, 0xbf, 0x77, + 0x52, 0x54, 0x73, 0xd2, 0x8c, 0x9f, 0x66, 0x7e, 0xac, 0x0c, 0x30, 0x89, 0x7d, 0xb9, 0x8e, 0xcb, + 0xc4, 0x28, 0xd0, 0xc7, 0x70, 0xbf, 0x35, 0xd0, 0xd4, 0x1d, 0x06, 0x98, 0xbd, 0xcf, 0xec, 0x30, + 0xc0, 0x7a, 0x04, 0x0f, 0xb0, 0x93, 0x3e, 0xc3, 0x33, 0xfd, 0x01, 0xee, 0x56, 0xc1, 0x3f, 0xaf, + 0x85, 0xd2, 0xc4, 0x85, 0x7d, 0x1c, 0xd8, 0xac, 0x88, 0x2d, 0x44, 0x72, 0x0a, 0x7b, 0xa9, 0x99, + 0x52, 0x51, 0xfa, 0xdb, 0x2d, 0xa5, 0x1b, 0x3b, 0xcb, 0xdd, 0xe8, 0x33, 0x18, 0xd6, 0x4a, 0x4b, + 0x64, 0xac, 0x04, 0x19, 0xc3, 0x7e, 0x8a, 0x65, 0x2a, 0xd7, 0x42, 0x14, 0xf7, 0xb6, 0x01, 0xb0, + 0xc2, 0x91, 0xfe, 0x0a, 0xf7, 0xce, 0x5e, 0xfe, 0x24, 0x7c, 0x6d, 0x8c, 0x2f, 0x84, 0x52, 0x7c, + 0x21, 0xde, 0x50, 0xa7, 0x6b, 0x52, 0x24, 0xd1, 0x66, 0x56, 0xf4, 0x5a, 0x88, 0xc6, 0x92, 0xf0, + 0x4d, 0x24, 0x79, 0x80, 0x33, 0x74, 0x58, 0x21, 0x92, 0x11, 0xf4, 0x24, 0xa6, 0x98, 0x05, 0x6e, + 0x17, 0x83, 0x4a, 0x99, 0x4e, 0x60, 0x78, 0xbe, 0x56, 0x4b, 0xac, 0xb1, 0x98, 0xd2, 0xa3, 0x0a, + 0xc9, 0x64, 0x3f, 0x18, 0x3f, 0xa8, 0xda, 0xc0, 0xef, 0x79, 0x66, 0x2d, 0x33, 0xd0, 0xb7, 0xe0, + 0x5e, 0x0d, 0x25, 0x9b, 0x06, 0xa5, 0x06, 0x3a, 0x8a, 0x1a, 0xd0, 0x5b, 0xf7, 0x46, 0xa7, 0x26, + 0xb0, 0xf4, 0xc9, 0xc7, 0xf8, 0xff, 0xf3, 0xff, 0x66, 0x83, 0x53, 0xb7, 0x90, 0x2f, 0xe0, 0x00, + 0x27, 0x66, 0xa6, 0x2e, 0xd2, 0x1c, 0xc6, 0xab, 0x60, 0x18, 0xbf, 0x9c, 0x57, 0xf6, 0xef, 0x42, + 0xbd, 0x9c, 0x05, 0xac, 0x1e, 0x42, 0x3c, 0x00, 0xee, 0x47, 0x39, 0x1e, 0xce, 0xda, 0x61, 0x35, + 0x0d, 0xa1, 0xe0, 0x54, 0xd2, 0x2c, 0x9b, 0x79, 0x9f, 0x35, 0x74, 0x64, 0x0c, 0x87, 0x08, 0x39, + 0x17, 0x5a, 0x87, 0xf1, 0x42, 0x15, 0x68, 0x5d, 0x44, 0x6b, 0xb5, 0x91, 0x4f, 0xe0, 0x41, 0x9b, + 0x7e, 0x16, 0xb8, 0xbb, 0x98, 0xe1, 0x16, 0x2b, 0xfd, 0xcb, 0x82, 0x83, 0x5a, 0x4b, 0xe6, 0xd2, + 0xc3, 0x40, 0xc4, 0x3a, 0xd4, 0x9b, 0xfc, 0x95, 0x96, 0x32, 0x79, 0x17, 0xfa, 0x3a, 0x5c, 0x09, + 0xa5, 0xf9, 0x2a, 0xc1, 0xd6, 0x76, 0x58, 0xa5, 0x30, 0x56, 0xcc, 0xf1, 0xcd, 0x26, 0x11, 0x79, + 0x5b, 0x95, 0x82, 0xbc, 0x0f, 0x03, 0xb3, 0x71, 0xa1, 0xcf, 0x75, 0x28, 0xe3, 0xaf, 0xc4, 0x06, + 0xbb, 0xe9, 0xb2, 0x2d, 0xad, 0x79, 0x91, 0x4a, 0x88, 0xac, 0x6a, 0x87, 0xe1, 0x99, 0x9e, 0xc3, + 0xa0, 0x39, 0x78, 0x72, 0x74, 0xf3, 0x9e, 0x9c, 0xe6, 0x3d, 0x98, 0x6a, 0xc2, 0x45, 0xcc, 0xf5, + 0x3a, 0x15, 0xf9, 0x35, 0x54, 0x0a, 0x3a, 0x81, 0xc3, 0xb6, 0xab, 0x34, 0x51, 0x29, 0xbf, 0x6c, + 0xa0, 0x56, 0x8a, 0x7c, 0x0b, 0xed, 0x72, 0x0b, 0x7f, 0x84, 0xc3, 0x79, 0x7d, 0xaa, 0x4f, 0x65, + 0xac, 0x0d, 0xcb, 0x7c, 0x0e, 0x4e, 0xf6, 0x50, 0x26, 0x22, 0x12, 0x5a, 0xdc, 0xdc, 0xc6, 0xb3, + 0x9a, 0xf5, 0x79, 0x87, 0x35, 0xbc, 0x9f, 0xec, 0xc3, 0xee, 0x2f, 0x3c, 0x5a, 0x0b, 0xea, 0x81, + 0x53, 0x77, 0xbc, 0xf1, 0x08, 0x3e, 0x85, 0xfb, 0x8d, 0xf4, 0xf3, 0x98, 0x27, 0x6a, 0x29, 0xb5, + 0xd9, 0xc1, 0x00, 0x43, 0x82, 0x59, 0x90, 0x51, 0x4a, 0x9f, 0xd5, 0x34, 0xf4, 0x77, 0x0b, 0x9c, + 0x22, 0x68, 0xc2, 0x35, 0x27, 0x9f, 0xc1, 0xbe, 0x9f, 0xd5, 0x9e, 0x13, 0x90, 0xb7, 0xf5, 0x72, + 0xb6, 0x3a, 0x64, 0x85, 0xbb, 0x21, 0x6f, 0x95, 0xa7, 0xc5, 0xc1, 0x34, 0xc8, 0xbb, 0xb5, 0x3a, + 0x56, 0x06, 0x7c, 0xf0, 0x35, 0xf4, 0xa6, 0x69, 0xfa, 0x54, 0x06, 0x42, 0x91, 0x01, 0xc0, 0xb7, + 0xb1, 0x78, 0x95, 0x08, 0x5f, 0x8b, 0x60, 0xd8, 0x21, 0xc3, 0xfc, 0x65, 0xbe, 0x08, 0x95, 0x0a, + 0xe3, 0xc5, 0xd0, 0x22, 0x77, 0xf3, 0x45, 0x9d, 0xbe, 0x0a, 0x95, 0x56, 0x43, 0xdb, 0x28, 0xa6, + 0x69, 0x2a, 0xd3, 0xb3, 0x8b, 0x0b, 0x25, 0xf4, 0x30, 0x18, 0xff, 0x69, 0xc3, 0x2e, 0xba, 0x90, + 0x2f, 0xa1, 0x57, 0x10, 0x27, 0x79, 0xa7, 0x8d, 0x4c, 0x91, 0x56, 0x46, 0xa3, 0x56, 0x9e, 0xcd, + 0xd8, 0x64, 0x02, 0xfd, 0x92, 0x9b, 0x48, 0xcd, 0x71, 0x9b, 0xf6, 0x46, 0x0f, 0x5b, 0x6d, 0x75, + 0x94, 0x9c, 0xa8, 0x9a, 0x28, 0x4d, 0x86, 0x6b, 0xa2, 0x6c, 0x33, 0xdb, 0x33, 0xd8, 0x9b, 0xeb, + 0x54, 0xf0, 0x15, 0x79, 0xb8, 0xbd, 0x44, 0x35, 0xfa, 0x1f, 0xbd, 0xc9, 0x78, 0x6c, 0x3d, 0xb2, + 0x9e, 0x7c, 0xfc, 0xf7, 0x95, 0x67, 0xbd, 0xbe, 0xf2, 0xac, 0x7f, 0xaf, 0x3c, 0xeb, 0x8f, 0x6b, + 0xaf, 0xf3, 0xfa, 0xda, 0xeb, 0xfc, 0x73, 0xed, 0x75, 0xbe, 0x1f, 0xdd, 0xfe, 0x97, 0xe1, 0xe5, + 0x1e, 0xfe, 0x7c, 0xf4, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x4e, 0x86, 0xe7, 0xfa, 0x57, 0x08, 0x00, 0x00, } @@ -1469,6 +1693,170 @@ func (m *RawSpaceHeaderWithId) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *SpaceSettingsContent) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceSettingsContent) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SpaceSettingsContent) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Value != nil { + { + size := m.Value.Size() + i -= size + if _, err := m.Value.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + } + } + return len(dAtA) - i, nil +} + +func (m *SpaceSettingsContent_ObjectDelete) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SpaceSettingsContent_ObjectDelete) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.ObjectDelete != nil { + { + size, err := m.ObjectDelete.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintSpacesync(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} +func (m *ObjectDelete) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ObjectDelete) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ObjectDelete) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Id) > 0 { + i -= len(m.Id) + copy(dAtA[i:], m.Id) + i = encodeVarintSpacesync(dAtA, i, uint64(len(m.Id))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SpaceSettingsSnapshot) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceSettingsSnapshot) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SpaceSettingsSnapshot) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.DeletedIds) > 0 { + for iNdEx := len(m.DeletedIds) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.DeletedIds[iNdEx]) + copy(dAtA[i:], m.DeletedIds[iNdEx]) + i = encodeVarintSpacesync(dAtA, i, uint64(len(m.DeletedIds[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *SettingsData) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SettingsData) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SettingsData) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Snapshot != nil { + { + size, err := m.Snapshot.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintSpacesync(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + if len(m.Content) > 0 { + for iNdEx := len(m.Content) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Content[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintSpacesync(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func encodeVarintSpacesync(dAtA []byte, offset int, v uint64) int { offset -= sovSpacesync(v) base := offset @@ -1734,6 +2122,77 @@ func (m *RawSpaceHeaderWithId) Size() (n int) { return n } +func (m *SpaceSettingsContent) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Value != nil { + n += m.Value.Size() + } + return n +} + +func (m *SpaceSettingsContent_ObjectDelete) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.ObjectDelete != nil { + l = m.ObjectDelete.Size() + n += 1 + l + sovSpacesync(uint64(l)) + } + return n +} +func (m *ObjectDelete) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Id) + if l > 0 { + n += 1 + l + sovSpacesync(uint64(l)) + } + return n +} + +func (m *SpaceSettingsSnapshot) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.DeletedIds) > 0 { + for _, s := range m.DeletedIds { + l = len(s) + n += 1 + l + sovSpacesync(uint64(l)) + } + } + return n +} + +func (m *SettingsData) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Content) > 0 { + for _, e := range m.Content { + l = e.Size() + n += 1 + l + sovSpacesync(uint64(l)) + } + } + if m.Snapshot != nil { + l = m.Snapshot.Size() + n += 1 + l + sovSpacesync(uint64(l)) + } + return n +} + func sovSpacesync(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -3422,6 +3881,375 @@ func (m *RawSpaceHeaderWithId) Unmarshal(dAtA []byte) error { } return nil } +func (m *SpaceSettingsContent) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpacesync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceSettingsContent: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceSettingsContent: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ObjectDelete", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpacesync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthSpacesync + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthSpacesync + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &ObjectDelete{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Value = &SpaceSettingsContent_ObjectDelete{v} + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipSpacesync(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthSpacesync + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ObjectDelete) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpacesync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ObjectDelete: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ObjectDelete: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Id", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpacesync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthSpacesync + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthSpacesync + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Id = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipSpacesync(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthSpacesync + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SpaceSettingsSnapshot) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpacesync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceSettingsSnapshot: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceSettingsSnapshot: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DeletedIds", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpacesync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthSpacesync + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthSpacesync + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DeletedIds = append(m.DeletedIds, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipSpacesync(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthSpacesync + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SettingsData) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpacesync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SettingsData: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SettingsData: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Content", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpacesync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthSpacesync + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthSpacesync + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Content = append(m.Content, &SpaceSettingsContent{}) + if err := m.Content[len(m.Content)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Snapshot", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpacesync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthSpacesync + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthSpacesync + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Snapshot == nil { + m.Snapshot = &SpaceSettingsSnapshot{} + } + if err := m.Snapshot.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipSpacesync(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthSpacesync + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipSpacesync(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/common/commonspace/synctree/synctree.go b/common/commonspace/synctree/synctree.go index 09a3a403..90a346ae 100644 --- a/common/commonspace/synctree/synctree.go +++ b/common/commonspace/synctree/synctree.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/logger" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/diffservice" - "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsservice" spacestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/syncservice" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/syncservice/synchandler" @@ -30,11 +29,10 @@ var ( type SyncTree struct { tree.ObjectTree synchandler.SyncHandler - syncClient SyncClient - listener updatelistener.UpdateListener - deletedNotifiable settingsservice.DeletedDocumentNotifiable - isClosed bool - isDeleted bool + syncClient SyncClient + listener updatelistener.UpdateListener + isClosed bool + isDeleted bool } var log = logger.NewNamed("commonspace.synctree").Sugar() @@ -45,27 +43,25 @@ var buildObjectTree = tree.BuildObjectTree var createSyncClient = newSyncClient type CreateDeps struct { - SpaceId string - Payload tree.ObjectTreeCreatePayload - Configuration nodeconf.Configuration - HeadNotifiable diffservice.HeadNotifiable - StreamPool syncservice.StreamPool - Listener updatelistener.UpdateListener - AclList list.ACLList - CreateStorage storage.TreeStorageCreatorFunc - DeletedNotifiable settingsservice.DeletedDocumentNotifiable + SpaceId string + Payload tree.ObjectTreeCreatePayload + Configuration nodeconf.Configuration + HeadNotifiable diffservice.HeadNotifiable + StreamPool syncservice.StreamPool + Listener updatelistener.UpdateListener + AclList list.ACLList + CreateStorage storage.TreeStorageCreatorFunc } type BuildDeps struct { - SpaceId string - StreamPool syncservice.StreamPool - Configuration nodeconf.Configuration - HeadNotifiable diffservice.HeadNotifiable - Listener updatelistener.UpdateListener - AclList list.ACLList - SpaceStorage spacestorage.SpaceStorage - TreeStorage storage.TreeStorage - DeletedNotifiable settingsservice.DeletedDocumentNotifiable + SpaceId string + StreamPool syncservice.StreamPool + Configuration nodeconf.Configuration + HeadNotifiable diffservice.HeadNotifiable + Listener updatelistener.UpdateListener + AclList list.ACLList + SpaceStorage spacestorage.SpaceStorage + TreeStorage storage.TreeStorage } func DeriveSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, err error) { @@ -80,10 +76,9 @@ func DeriveSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, er sharedFactory, deps.Configuration) syncTree := &SyncTree{ - ObjectTree: t, - syncClient: syncClient, - listener: deps.Listener, - deletedNotifiable: deps.DeletedNotifiable, + ObjectTree: t, + syncClient: syncClient, + listener: deps.Listener, } syncHandler := newSyncTreeHandler(syncTree, syncClient) syncTree.SyncHandler = syncHandler @@ -106,10 +101,9 @@ func CreateSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, er GetRequestFactory(), deps.Configuration) syncTree := &SyncTree{ - ObjectTree: t, - syncClient: syncClient, - listener: deps.Listener, - deletedNotifiable: deps.DeletedNotifiable, + ObjectTree: t, + syncClient: syncClient, + listener: deps.Listener, } syncHandler := newSyncTreeHandler(syncTree, syncClient) syncTree.SyncHandler = syncHandler @@ -192,10 +186,9 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t tr GetRequestFactory(), deps.Configuration) syncTree := &SyncTree{ - ObjectTree: t, - syncClient: syncClient, - listener: deps.Listener, - deletedNotifiable: deps.DeletedNotifiable, + ObjectTree: t, + syncClient: syncClient, + listener: deps.Listener, } syncHandler := newSyncTreeHandler(syncTree, syncClient) syncTree.SyncHandler = syncHandler @@ -213,6 +206,20 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t tr return } +func (s *SyncTree) IterateFrom(id string, convert tree.ChangeConvertFunc, iterate tree.ChangeIterateFunc) (err error) { + if err = s.checkAlive(); err != nil { + return + } + return s.ObjectTree.IterateFrom(id, convert, iterate) +} + +func (s *SyncTree) Iterate(convert tree.ChangeConvertFunc, iterate tree.ChangeIterateFunc) (err error) { + if err = s.checkAlive(); err != nil { + return + } + return s.ObjectTree.Iterate(convert, iterate) +} + func (s *SyncTree) AddContent(ctx context.Context, content tree.SignableChangeContent) (res tree.AddResult, err error) { if err = s.checkAlive(); err != nil { return @@ -254,12 +261,7 @@ func (s *SyncTree) AddRawChanges(ctx context.Context, changes ...*treechangeprot func (s *SyncTree) Delete() (err error) { log.With("id", s.ID()).Debug("deleting sync tree") s.Lock() - defer func() { - s.Unlock() - if err == nil { - s.deletedNotifiable.NotifyDeleted(s.ID()) - } - }() + defer s.Unlock() if err = s.checkAlive(); err != nil { return } @@ -268,7 +270,6 @@ func (s *SyncTree) Delete() (err error) { return } s.isDeleted = true - return } diff --git a/common/commonspace/treegetter/treegetter.go b/common/commonspace/treegetter/treegetter.go index 9047e273..79e2913f 100644 --- a/common/commonspace/treegetter/treegetter.go +++ b/common/commonspace/treegetter/treegetter.go @@ -15,4 +15,5 @@ var ErrSpaceNotFound = errors.New("space not found") type TreeGetter interface { app.ComponentRunnable GetTree(ctx context.Context, spaceId, treeId string) (tree.ObjectTree, error) + DeleteTree(ctx context.Context, spaceId, treeId string) error } diff --git a/common/pkg/acl/tree/changebuilder.go b/common/pkg/acl/tree/changebuilder.go index a3c9d3e2..ace60b23 100644 --- a/common/pkg/acl/tree/changebuilder.go +++ b/common/pkg/acl/tree/changebuilder.go @@ -155,12 +155,15 @@ func (c *changeBuilder) BuildContent(payload BuilderContent) (ch *Change, rawIdC Identity: payload.Identity, IsSnapshot: payload.IsSnapshot, } - - encrypted, err := payload.ReadKey.Encrypt(payload.Content) - if err != nil { - return + if payload.ReadKey != nil { + encrypted, err := payload.ReadKey.Encrypt(payload.Content) + if err != nil { + return + } + change.ChangesData = encrypted + } else { + change.ChangesData = payload.Content } - change.ChangesData = encrypted marshalledChange, err := proto.Marshal(change) if err != nil { diff --git a/common/pkg/acl/tree/objecttree.go b/common/pkg/acl/tree/objecttree.go index 52bbc36b..31cbd0e7 100644 --- a/common/pkg/acl/tree/objecttree.go +++ b/common/pkg/acl/tree/objecttree.go @@ -5,7 +5,7 @@ import ( "context" "errors" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/common" - list2 "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/list" + list "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/list" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/treechangeproto" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/util/keys/symmetric" @@ -67,7 +67,7 @@ type objectTree struct { validator ObjectTreeValidator rawChangeLoader *rawChangeLoader treeBuilder *treeBuilder - aclList list2.ACLList + aclList list.ACLList id string root *treechangeproto.RawTreeChangeWithId @@ -92,13 +92,13 @@ type objectTreeDeps struct { treeStorage storage.TreeStorage validator ObjectTreeValidator rawChangeLoader *rawChangeLoader - aclList list2.ACLList + aclList list.ACLList } func defaultObjectTreeDeps( rootChange *treechangeproto.RawTreeChangeWithId, treeStorage storage.TreeStorage, - aclList list2.ACLList) objectTreeDeps { + aclList list.ACLList) objectTreeDeps { keychain := common.NewKeychain() changeBuilder := NewChangeBuilder(keychain, rootChange) @@ -187,16 +187,23 @@ func (ot *objectTree) prepareBuilderContent(content SignableChangeContent) (cnt ot.aclList.RLock() defer ot.aclList.RUnlock() - state := ot.aclList.ACLState() // special method for own keys - readKey, err := state.CurrentReadKey() - if err != nil { - return + var ( + state = ot.aclList.ACLState() // special method for own keys + readKey *symmetric.Key + readKeyHash uint64 + ) + if content.IsEncrypted { + readKeyHash = state.CurrentReadKeyHash() + readKey, err = state.CurrentReadKey() + if err != nil { + return + } } cnt = BuilderContent{ TreeHeadIds: ot.tree.Heads(), AclHeadId: ot.aclList.Head().Id, SnapshotBaseId: ot.tree.RootId(), - CurrentReadKeyHash: state.CurrentReadKeyHash(), + CurrentReadKeyHash: readKeyHash, Identity: content.Identity, IsSnapshot: content.IsSnapshot, SigningKey: content.Key, @@ -440,9 +447,25 @@ func (ot *objectTree) IterateFrom(id string, convert ChangeConvertFunc, iterate ot.tree.Iterate(id, iterate) return } + decrypt := func(c *Change) (decrypted []byte, err error) { + // the change is not encrypted + if c.ReadKeyHash == 0 { + decrypted = c.Data + return + } + readKey, exists := ot.keys[c.ReadKeyHash] + if !exists { + err = list.ErrNoReadKey + return + } + + decrypted, err = readKey.Decrypt(c.Data) + return + } ot.tree.Iterate(ot.tree.RootId(), func(c *Change) (isContinue bool) { var model any + // if already saved as a model if c.Model != nil { return iterate(c) } @@ -450,14 +473,9 @@ func (ot *objectTree) IterateFrom(id string, convert ChangeConvertFunc, iterate if c.Id == ot.id { return iterate(c) } - readKey, exists := ot.keys[c.ReadKeyHash] - if !exists { - err = list2.ErrNoReadKey - return false - } var decrypted []byte - decrypted, err = readKey.Decrypt(c.Data) + decrypted, err = decrypt(c) if err != nil { return false } diff --git a/common/pkg/acl/tree/objecttreefactory.go b/common/pkg/acl/tree/objecttreefactory.go index 58e6ca65..ac655814 100644 --- a/common/pkg/acl/tree/objecttreefactory.go +++ b/common/pkg/acl/tree/objecttreefactory.go @@ -14,10 +14,11 @@ import ( ) type ObjectTreeCreatePayload struct { - SignKey signingkey.PrivKey - ChangeType string - SpaceId string - Identity []byte + SignKey signingkey.PrivKey + ChangeType string + SpaceId string + Identity []byte + IsEncrypted bool } func BuildObjectTree(treeStorage storage.TreeStorage, aclList list.ACLList) (ObjectTree, error) { diff --git a/common/pkg/acl/tree/signablecontent.go b/common/pkg/acl/tree/signablecontent.go index 086d1b06..4be96d1d 100644 --- a/common/pkg/acl/tree/signablecontent.go +++ b/common/pkg/acl/tree/signablecontent.go @@ -5,8 +5,9 @@ import ( ) type SignableChangeContent struct { - Data []byte - Key signingkey.PrivKey - Identity []byte - IsSnapshot bool + Data []byte + Key signingkey.PrivKey + Identity []byte + IsSnapshot bool + IsEncrypted bool } diff --git a/node/nodespace/nodecache/treecache.go b/node/nodespace/nodecache/treecache.go index 79d3ac01..e14f9247 100644 --- a/node/nodespace/nodecache/treecache.go +++ b/node/nodespace/nodecache/treecache.go @@ -70,3 +70,16 @@ func (c *treeCache) GetTree(ctx context.Context, spaceId, id string) (tr tree.Ob tr = value.(tree.ObjectTree) return } + +func (c *treeCache) DeleteTree(ctx context.Context, spaceId, treeId string) (err error) { + tr, err := c.GetTree(ctx, spaceId, treeId) + if err != nil { + return + } + err = tr.Delete() + if err != nil { + return + } + _, err = c.cache.Remove(treeId) + return +} From 6c38474c408304d6e372b838c8e53340d60497d2 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Sat, 12 Nov 2022 16:24:06 +0100 Subject: [PATCH 05/27] Diff service notify logic --- common/commonspace/diffservice/diffservice.go | 12 ++++--- common/commonspace/diffservice/diffsyncer.go | 36 ++++++++++++++++--- .../commonspace/settingsdocument/provider.go | 14 ++++++-- .../settingsdocument/settingsdocument.go | 31 +++++++++------- common/commonspace/space.go | 7 ++++ common/pkg/acl/tree/objecttree.go | 2 +- 6 files changed, 78 insertions(+), 24 deletions(-) diff --git a/common/commonspace/diffservice/diffservice.go b/common/commonspace/diffservice/diffservice.go index 71fe9bef..b4d05667 100644 --- a/common/commonspace/diffservice/diffservice.go +++ b/common/commonspace/diffservice/diffservice.go @@ -16,7 +16,7 @@ import ( type DiffService interface { HeadNotifiable HandleRangeRequest(ctx context.Context, req *spacesyncproto.HeadSyncRequest) (resp *spacesyncproto.HeadSyncResponse, err error) - RemoveObject(id string) + RemoveObjects(ids []string) AllIds() []string Init(objectIds []string) @@ -29,6 +29,7 @@ type diffService struct { storage storage.SpaceStorage diff ldiff.Diff log *zap.Logger + syncer DiffSyncer syncPeriod int } @@ -50,6 +51,7 @@ func NewDiffService( return &diffService{ spaceId: spaceId, storage: storage, + syncer: syncer, periodicSync: periodicSync, diff: diff, log: log, @@ -77,9 +79,11 @@ func (d *diffService) AllIds() []string { return d.diff.Ids() } -func (d *diffService) RemoveObject(id string) { - // TODO: add space document to remove ids - d.diff.RemoveId(id) +func (d *diffService) RemoveObjects(ids []string) { + for _, id := range ids { + d.diff.RemoveId(id) + } + d.syncer.RemoveObjects(ids) } func (d *diffService) Close() (err error) { diff --git a/common/commonspace/diffservice/diffsyncer.go b/common/commonspace/diffservice/diffsyncer.go index 9610a787..abe360f5 100644 --- a/common/commonspace/diffservice/diffsyncer.go +++ b/common/commonspace/diffservice/diffsyncer.go @@ -11,11 +11,13 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/nodeconf" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/ldiff" "go.uber.org/zap" + "sync" "time" ) type DiffSyncer interface { Sync(ctx context.Context) error + RemoveObjects(ids []string) } func newDiffSyncer( @@ -34,10 +36,12 @@ func newDiffSyncer( confConnector: confConnector, clientFactory: clientFactory, log: log, + removedIds: map[string]struct{}{}, } } type diffSyncer struct { + sync.Mutex spaceId string diff ldiff.Diff confConnector nodeconf.ConfConnector @@ -45,6 +49,15 @@ type diffSyncer struct { storage storage.SpaceStorage clientFactory spacesyncproto.ClientFactory log *zap.Logger + removedIds map[string]struct{} +} + +func (d *diffSyncer) RemoveObjects(ids []string) { + d.Lock() + defer d.Unlock() + for _, id := range ids { + d.removedIds[id] = struct{}{} + } } func (d *diffSyncer) Sync(ctx context.Context) error { @@ -74,15 +87,30 @@ func (d *diffSyncer) syncWithPeer(ctx context.Context, p peer.Peer) (err error) if err == spacesyncproto.ErrSpaceMissing { return d.sendPushSpaceRequest(ctx, cl) } + var afterFilterIds []string + filter := func(ids []string) { + for _, id := range ids { + if _, exists := d.removedIds[id]; !exists { + afterFilterIds = append(afterFilterIds, id) + } + } + } + d.Lock() + totalLen := 0 + // not syncing ids which were removed through settings document + for _, ids := range [][]string{newIds, changedIds, removedIds} { + totalLen += len(ids) + filter(ids) + } + d.Unlock() ctx = peer.CtxWithPeerId(ctx, p.Id()) - d.pingTreesInCache(ctx, newIds) - d.pingTreesInCache(ctx, changedIds) - d.pingTreesInCache(ctx, removedIds) + d.pingTreesInCache(ctx, afterFilterIds) d.log.Info("sync done:", zap.Int("newIds", len(newIds)), zap.Int("changedIds", len(changedIds)), - zap.Int("removedIds", len(removedIds))) + zap.Int("removedIds", len(removedIds)), + zap.Int("filteredIds", totalLen-len(afterFilterIds))) return } diff --git a/common/commonspace/settingsdocument/provider.go b/common/commonspace/settingsdocument/provider.go index c25c3b8a..bc64da57 100644 --- a/common/commonspace/settingsdocument/provider.go +++ b/common/commonspace/settingsdocument/provider.go @@ -23,17 +23,25 @@ func (p *provider) convert(decrypted []byte) (res any, err error) { func (p *provider) ProvideIds(tr tree.ObjectTree, startId string) (ids []string, lastId string, err error) { processChange := func(change *tree.Change) bool { - // ignoring first change if startId is not "" - if change.Id == startId { + // ignoring root change which has empty model or startId change + lastId = change.Id + if change.Model == nil || (change.Id == startId && startId != "") { return true } + deleteChange := change.Model.(*spacesyncproto.SettingsData) + // getting data from snapshot if we start from it + if change.Id == tr.Root().Id { + ids = deleteChange.Snapshot.DeletedIds + return true + } + + // otherwise getting data from content for _, cnt := range deleteChange.Content { if cnt.GetObjectDelete() != nil { ids = append(ids, cnt.GetObjectDelete().GetId()) } } - lastId = change.Id return true } if startId == "" { diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go index 5c9333e4..b4a4c8b5 100644 --- a/common/commonspace/settingsdocument/settingsdocument.go +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -19,37 +19,42 @@ const ( type SettingsDocument interface { tree.ObjectTree + Init() DeleteObject(id string) (err error) } type BuildTreeFunc func(ctx context.Context, id string, listener updatelistener.UpdateListener) (t tree.ObjectTree, err error) +type RemoveObjectsFunc func([]string) type Deps struct { BuildFunc BuildTreeFunc Account account.Service TreeGetter treegetter.TreeGetter Store spacestorage.SpaceStorage + RemoveFunc RemoveObjectsFunc prov deletedIdsProvider } type settingsDocument struct { tree.ObjectTree - account account.Service - spaceId string - deletionState map[string]DeletionState - treeGetter treegetter.TreeGetter - store spacestorage.SpaceStorage - lastChangeId string - prov deletedIdsProvider + account account.Service + spaceId string + deletionState map[string]DeletionState + treeGetter treegetter.TreeGetter + store spacestorage.SpaceStorage + lastChangeId string + prov deletedIdsProvider + removeNotifyFunc RemoveObjectsFunc } func NewSettingsDocument(ctx context.Context, deps Deps, spaceId string) (doc SettingsDocument, err error) { s := &settingsDocument{ - account: deps.Account, - spaceId: spaceId, - deletionState: map[string]DeletionState{}, - treeGetter: deps.TreeGetter, - store: deps.Store, + account: deps.Account, + spaceId: spaceId, + deletionState: map[string]DeletionState{}, + treeGetter: deps.TreeGetter, + store: deps.Store, + removeNotifyFunc: deps.RemoveFunc, } s.ObjectTree, err = deps.BuildFunc(ctx, deps.Store.SpaceSettingsId(), s) if err != nil { @@ -104,6 +109,8 @@ func (s *settingsDocument) toBeDeleted(ids []string) { } s.deletionState[id] = DeletionStateDeleted } + // notifying about removal + s.removeNotifyFunc(ids) } func (s *settingsDocument) DeleteObject(id string) (err error) { diff --git a/common/commonspace/space.go b/common/commonspace/space.go index f9638cad..f6d4087d 100644 --- a/common/commonspace/space.go +++ b/common/commonspace/space.go @@ -72,6 +72,7 @@ type Space interface { DeriveTree(ctx context.Context, payload tree.ObjectTreeCreatePayload, listener updatelistener.UpdateListener) (tree.ObjectTree, error) CreateTree(ctx context.Context, payload tree.ObjectTreeCreatePayload, listener updatelistener.UpdateListener) (tree.ObjectTree, error) BuildTree(ctx context.Context, id string, listener updatelistener.UpdateListener) (tree.ObjectTree, error) + DeleteTree(ctx context.Context, id string) (err error) Close() error } @@ -149,11 +150,13 @@ func (s *space) Init(ctx context.Context) (err error) { Account: s.account, TreeGetter: s.cache, Store: s.storage, + RemoveFunc: s.diffService.RemoveObjects, } s.settingsDocument, err = settingsdocument.NewSettingsDocument(context.Background(), deps, s.id) if err != nil { return } + s.settingsDocument.Init() s.aclList = syncacl.NewSyncACL(aclList, s.syncService.StreamPool()) objectGetter := newCommonSpaceGetter(s.id, s.aclList, s.cache, s.settingsDocument) s.syncService.Init(objectGetter) @@ -230,6 +233,10 @@ func (s *space) BuildTree(ctx context.Context, id string, listener updatelistene return synctree.BuildSyncTreeOrGetRemote(ctx, id, deps) } +func (s *space) DeleteTree(ctx context.Context, id string) (err error) { + return s.settingsDocument.DeleteObject(id) +} + func (s *space) Close() error { log.With(zap.String("id", s.id)).Debug("space is closing") defer func() { diff --git a/common/pkg/acl/tree/objecttree.go b/common/pkg/acl/tree/objecttree.go index 31cbd0e7..00b009c2 100644 --- a/common/pkg/acl/tree/objecttree.go +++ b/common/pkg/acl/tree/objecttree.go @@ -463,7 +463,7 @@ func (ot *objectTree) IterateFrom(id string, convert ChangeConvertFunc, iterate return } - ot.tree.Iterate(ot.tree.RootId(), func(c *Change) (isContinue bool) { + ot.tree.Iterate(id, func(c *Change) (isContinue bool) { var model any // if already saved as a model if c.Model != nil { From 62d26fbafde2beba47b307614fcaa89658d4f6de Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Sat, 12 Nov 2022 17:49:25 +0100 Subject: [PATCH 06/27] WIP more deletion checks --- client/api/controller.go | 6 +++ client/document/service.go | 12 +++++- common/commonspace/diffservice/diffservice.go | 8 +--- common/commonspace/diffservice/diffsyncer.go | 14 ++++++ .../settingsdocument/settingsdocument.go | 43 +++++++++++++------ common/commonspace/space.go | 2 +- 6 files changed, 63 insertions(+), 22 deletions(-) diff --git a/client/api/controller.go b/client/api/controller.go index 9e897dac..0e6b886c 100644 --- a/client/api/controller.go +++ b/client/api/controller.go @@ -23,6 +23,8 @@ type Controller interface { // CreateDocument creates new document in space CreateDocument(spaceId string) (id string, err error) + // DeleteDocument deletes a document from space + DeleteDocument(spaceId, documentId string) (err error) // AllDocumentIds gets all ids of documents in space AllDocumentIds(spaceId string) (ids []string, err error) // AddText adds text to space document @@ -97,6 +99,10 @@ func (c *controller) CreateDocument(spaceId string) (id string, err error) { return c.docService.CreateDocument(spaceId) } +func (c *controller) DeleteDocument(spaceId, documentId string) (err error) { + return c.docService.DeleteDocument(spaceId, documentId) +} + func (c *controller) AllDocumentIds(spaceId string) (ids []string, err error) { return c.docService.AllDocumentIds(spaceId) } diff --git a/client/document/service.go b/client/document/service.go index 399363b8..57e1cd98 100644 --- a/client/document/service.go +++ b/client/document/service.go @@ -53,10 +53,18 @@ func (s *service) CreateDocument(spaceId string) (id string, err error) { if err != nil { return } - id = doc.Tree().ID() + id = doc.ID() return } +func (s *service) DeleteDocument(spaceId, documentId string) (err error) { + space, err := s.spaceService.GetSpace(context.Background(), spaceId) + if err != nil { + return + } + return space.DeleteTree(context.Background(), documentId) +} + func (s *service) AllDocumentIds(spaceId string) (ids []string, err error) { space, err := s.spaceService.GetSpace(context.Background(), spaceId) if err != nil { @@ -79,5 +87,5 @@ func (s *service) DumpDocumentTree(spaceId, documentId string) (dump string, err if err != nil { return } - return doc.Tree().DebugDump() + return doc.DebugDump() } diff --git a/common/commonspace/diffservice/diffservice.go b/common/commonspace/diffservice/diffservice.go index b4d05667..a0369ece 100644 --- a/common/commonspace/diffservice/diffservice.go +++ b/common/commonspace/diffservice/diffservice.go @@ -69,10 +69,7 @@ func (d *diffService) HandleRangeRequest(ctx context.Context, req *spacesyncprot } func (d *diffService) UpdateHeads(id string, heads []string) { - d.diff.Set(ldiff.Element{ - Id: id, - Head: concatStrings(heads), - }) + d.syncer.UpdateHeads(id, heads) } func (d *diffService) AllIds() []string { @@ -80,9 +77,6 @@ func (d *diffService) AllIds() []string { } func (d *diffService) RemoveObjects(ids []string) { - for _, id := range ids { - d.diff.RemoveId(id) - } d.syncer.RemoveObjects(ids) } diff --git a/common/commonspace/diffservice/diffsyncer.go b/common/commonspace/diffservice/diffsyncer.go index abe360f5..31c5071a 100644 --- a/common/commonspace/diffservice/diffsyncer.go +++ b/common/commonspace/diffservice/diffsyncer.go @@ -18,6 +18,7 @@ import ( type DiffSyncer interface { Sync(ctx context.Context) error RemoveObjects(ids []string) + UpdateHeads(id string, heads []string) } func newDiffSyncer( @@ -56,10 +57,23 @@ func (d *diffSyncer) RemoveObjects(ids []string) { d.Lock() defer d.Unlock() for _, id := range ids { + d.diff.RemoveId(id) d.removedIds[id] = struct{}{} } } +func (d *diffSyncer) UpdateHeads(id string, heads []string) { + d.Lock() + defer d.Unlock() + if _, exists := d.removedIds[id]; exists { + return + } + d.diff.Set(ldiff.Element{ + Id: id, + Head: concatStrings(heads), + }) +} + func (d *diffSyncer) Sync(ctx context.Context) error { st := time.Now() // diffing with responsible peers according to configuration diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go index b4a4c8b5..a93a4fae 100644 --- a/common/commonspace/settingsdocument/settingsdocument.go +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -8,6 +8,7 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree/updatelistener" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" + "sync" ) type DeletionState int @@ -19,7 +20,7 @@ const ( type SettingsDocument interface { tree.ObjectTree - Init() + Refresh() DeleteObject(id string) (err error) } @@ -37,14 +38,15 @@ type Deps struct { type settingsDocument struct { tree.ObjectTree - account account.Service - spaceId string - deletionState map[string]DeletionState - treeGetter treegetter.TreeGetter - store spacestorage.SpaceStorage - lastChangeId string - prov deletedIdsProvider - removeNotifyFunc RemoveObjectsFunc + account account.Service + spaceId string + deletionState map[string]DeletionState + treeGetter treegetter.TreeGetter + store spacestorage.SpaceStorage + lastChangeId string + prov deletedIdsProvider + removeNotifyFunc RemoveObjectsFunc + deletionStateLock sync.Mutex } func NewSettingsDocument(ctx context.Context, deps Deps, spaceId string) (doc SettingsDocument, err error) { @@ -68,6 +70,14 @@ func NewSettingsDocument(ctx context.Context, deps Deps, spaceId string) (doc Se return } +func (s *settingsDocument) NotifyHeadsUpdate(id string) { + s.deletionStateLock.Lock() + if _, exists := s.deletionState[id]; exists { + s.deletionState[id] = DeletionStateQueued + } + s.deletionStateLock.Unlock() +} + func (s *settingsDocument) Update(tr tree.ObjectTree) { ids, lastId, err := s.prov.ProvideIds(tr, s.lastChangeId) if err != nil { @@ -86,7 +96,7 @@ func (s *settingsDocument) Rebuild(tr tree.ObjectTree) { s.toBeDeleted(ids) } -func (s *settingsDocument) Init() { +func (s *settingsDocument) Refresh() { s.Lock() defer s.Unlock() s.Rebuild(s) @@ -94,26 +104,37 @@ func (s *settingsDocument) Init() { func (s *settingsDocument) toBeDeleted(ids []string) { for _, id := range ids { + s.deletionStateLock.Lock() if state, exists := s.deletionState[id]; exists && state == DeletionStateDeleted { + s.deletionStateLock.Unlock() continue } // if not already deleted + // TODO: here we can possibly have problems if the document is synced later, maybe we should block syncing with deleted documents if _, err := s.store.TreeStorage(id); err == nil { s.deletionState[id] = DeletionStateQueued + s.deletionStateLock.Unlock() + // doing this without lock err := s.treeGetter.DeleteTree(context.Background(), s.spaceId, id) if err != nil { // TODO: some errors may tell us that the tree is actually deleted, so we should have more checks here // TODO: add logging continue } + // TODO: add loop to double check that everything that should be deleted is actually deleted + s.deletionStateLock.Lock() } + s.deletionState[id] = DeletionStateDeleted + s.deletionStateLock.Unlock() } // notifying about removal s.removeNotifyFunc(ids) } func (s *settingsDocument) DeleteObject(id string) (err error) { + s.Lock() + defer s.Unlock() content := &spacesyncproto.SpaceSettingsContent_ObjectDelete{ ObjectDelete: &spacesyncproto.ObjectDelete{Id: id}, } @@ -127,8 +148,6 @@ func (s *settingsDocument) DeleteObject(id string) (err error) { if err != nil { return } - s.Lock() - defer s.Unlock() _, err = s.AddContent(context.Background(), tree.SignableChangeContent{ Data: res, Key: s.account.Account().SignKey, diff --git a/common/commonspace/space.go b/common/commonspace/space.go index f6d4087d..55d9f338 100644 --- a/common/commonspace/space.go +++ b/common/commonspace/space.go @@ -156,7 +156,7 @@ func (s *space) Init(ctx context.Context) (err error) { if err != nil { return } - s.settingsDocument.Init() + s.settingsDocument.Refresh() s.aclList = syncacl.NewSyncACL(aclList, s.syncService.StreamPool()) objectGetter := newCommonSpaceGetter(s.id, s.aclList, s.cache, s.settingsDocument) s.syncService.Init(objectGetter) From b617cdc690b2fc51fa75b6cdcc0005c68632dc5d Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Sat, 12 Nov 2022 18:17:01 +0100 Subject: [PATCH 07/27] Add sync loop to settings document --- client/api/service.go | 13 ++++++++++ client/document/service.go | 1 + common/commonspace/diffservice/diffservice.go | 5 ++-- .../commonspace/diffservice/headnotifiable.go | 6 +++++ .../settingsdocument/settingsdocument.go | 17 ++++++------ common/commonspace/space.go | 23 +++++++++++++--- .../periodicsync}/periodicsync.go | 10 ++++--- .../periodicsync}/periodicsync_test.go | 26 +++++++++++++------ 8 files changed, 75 insertions(+), 26 deletions(-) rename common/{commonspace/diffservice => util/periodicsync}/periodicsync.go (83%) rename common/{commonspace/diffservice => util/periodicsync}/periodicsync_test.go (55%) diff --git a/client/api/service.go b/client/api/service.go index 1f7ed647..7155ea38 100644 --- a/client/api/service.go +++ b/client/api/service.go @@ -65,6 +65,7 @@ func (s *service) Run(ctx context.Context) (err error) { mux.HandleFunc("/loadSpace", s.loadSpace) mux.HandleFunc("/allSpaceIds", s.allSpaceIds) mux.HandleFunc("/createDocument", s.createDocument) + mux.HandleFunc("/deleteDocument", s.deleteDocument) mux.HandleFunc("/allDocumentIds", s.allDocumentIds) mux.HandleFunc("/addText", s.addText) mux.HandleFunc("/dumpDocumentTree", s.dumpDocumentTree) @@ -134,6 +135,18 @@ func (s *service) createDocument(w http.ResponseWriter, req *http.Request) { sendText(w, http.StatusOK, id) } +func (s *service) deleteDocument(w http.ResponseWriter, req *http.Request) { + query := req.URL.Query() + spaceId := query.Get("spaceId") + documentId := query.Get("documentId") + err := s.controller.DeleteDocument(spaceId, documentId) + if err != nil { + sendText(w, http.StatusInternalServerError, err.Error()) + return + } + sendText(w, http.StatusOK, documentId) +} + func (s *service) allDocumentIds(w http.ResponseWriter, req *http.Request) { query := req.URL.Query() spaceId := query.Get("spaceId") diff --git a/client/document/service.go b/client/document/service.go index 57e1cd98..d6f727ea 100644 --- a/client/document/service.go +++ b/client/document/service.go @@ -14,6 +14,7 @@ import ( type Service interface { app.Component CreateDocument(spaceId string) (id string, err error) + DeleteDocument(spaceId, documentId string) (err error) AllDocumentIds(spaceId string) (ids []string, err error) AddText(spaceId, documentId, text string) (err error) DumpDocumentTree(spaceId, documentId string) (dump string, err error) diff --git a/common/commonspace/diffservice/diffservice.go b/common/commonspace/diffservice/diffservice.go index a0369ece..8888a111 100644 --- a/common/commonspace/diffservice/diffservice.go +++ b/common/commonspace/diffservice/diffservice.go @@ -9,6 +9,7 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/nodeconf" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/ldiff" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/util/periodicsync" "go.uber.org/zap" "strings" ) @@ -25,7 +26,7 @@ type DiffService interface { type diffService struct { spaceId string - periodicSync PeriodicSync + periodicSync periodicsync.PeriodicSync storage storage.SpaceStorage diff ldiff.Diff log *zap.Logger @@ -46,7 +47,7 @@ func NewDiffService( l := log.With(zap.String("spaceId", spaceId)) factory := spacesyncproto.ClientFactoryFunc(spacesyncproto.NewDRPCSpaceClient) syncer := newDiffSyncer(spaceId, diff, confConnector, cache, storage, factory, l) - periodicSync := newPeriodicSync(syncPeriod, syncer, l) + periodicSync := periodicsync.NewPeriodicSync(syncPeriod, syncer.Sync, l) return &diffService{ spaceId: spaceId, diff --git a/common/commonspace/diffservice/headnotifiable.go b/common/commonspace/diffservice/headnotifiable.go index 80819bfe..8e987dc9 100644 --- a/common/commonspace/diffservice/headnotifiable.go +++ b/common/commonspace/diffservice/headnotifiable.go @@ -3,3 +3,9 @@ package diffservice type HeadNotifiable interface { UpdateHeads(id string, heads []string) } + +type HeadNotifiableFunc func(id string, heads []string) + +func (h HeadNotifiableFunc) UpdateHeads(id string, heads []string) { + h(id, heads) +} diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go index a93a4fae..5ba52732 100644 --- a/common/commonspace/settingsdocument/settingsdocument.go +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -22,6 +22,7 @@ type SettingsDocument interface { tree.ObjectTree Refresh() DeleteObject(id string) (err error) + NotifyObjectUpdate(id string) } type BuildTreeFunc func(ctx context.Context, id string, listener updatelistener.UpdateListener) (t tree.ObjectTree, err error) @@ -33,7 +34,8 @@ type Deps struct { TreeGetter treegetter.TreeGetter Store spacestorage.SpaceStorage RemoveFunc RemoveObjectsFunc - prov deletedIdsProvider + // prov exists mainly for the ease of testing + prov deletedIdsProvider } type settingsDocument struct { @@ -70,9 +72,10 @@ func NewSettingsDocument(ctx context.Context, deps Deps, spaceId string) (doc Se return } -func (s *settingsDocument) NotifyHeadsUpdate(id string) { +func (s *settingsDocument) NotifyObjectUpdate(id string) { s.deletionStateLock.Lock() - if _, exists := s.deletionState[id]; exists { + if state, exists := s.deletionState[id]; exists && state == DeletionStateDeleted { + // marking the document as queued, that means that document appeared later than we checked the storage for deletion s.deletionState[id] = DeletionStateQueued } s.deletionStateLock.Unlock() @@ -109,8 +112,7 @@ func (s *settingsDocument) toBeDeleted(ids []string) { s.deletionStateLock.Unlock() continue } - // if not already deleted - // TODO: here we can possibly have problems if the document is synced later, maybe we should block syncing with deleted documents + // if the document is not in storage it can happen that it will appear later, for that we have NotifyObjectUpdate method if _, err := s.store.TreeStorage(id); err == nil { s.deletionState[id] = DeletionStateQueued s.deletionStateLock.Unlock() @@ -121,14 +123,13 @@ func (s *settingsDocument) toBeDeleted(ids []string) { // TODO: add logging continue } - // TODO: add loop to double check that everything that should be deleted is actually deleted s.deletionStateLock.Lock() } - + s.deletionState[id] = DeletionStateDeleted s.deletionStateLock.Unlock() } - // notifying about removal + // notifying diff service that the ids should not be synced anymore s.removeNotifyFunc(ids) } diff --git a/common/commonspace/space.go b/common/commonspace/space.go index 55d9f338..f8502ff5 100644 --- a/common/commonspace/space.go +++ b/common/commonspace/space.go @@ -19,6 +19,7 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/util/keys/asymmetric/encryptionkey" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/util/keys/asymmetric/signingkey" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/util/periodicsync" "github.com/zeebo/errs" "go.uber.org/zap" "sync" @@ -41,7 +42,10 @@ type SpaceCreatePayload struct { ReplicationKey uint64 } -const SpaceTypeDerived = "derived.space" +const ( + SpaceTypeDerived = "derived.space" + SettingsSyncPeriodSeconds = 10 +) type SpaceDerivePayload struct { SigningKey signingkey.PrivKey @@ -92,6 +96,8 @@ type space struct { aclList *syncacl.SyncACL configuration nodeconf.Configuration settingsDocument settingsdocument.SettingsDocument + settingsSync periodicsync.PeriodicSync + headNotifiable diffservice.HeadNotifiable isClosed atomic.Bool } @@ -156,11 +162,19 @@ func (s *space) Init(ctx context.Context) (err error) { if err != nil { return } + s.headNotifiable = diffservice.HeadNotifiableFunc(func(id string, heads []string) { + s.diffService.UpdateHeads(id, heads) + s.settingsDocument.NotifyObjectUpdate(id) + }) s.settingsDocument.Refresh() s.aclList = syncacl.NewSyncACL(aclList, s.syncService.StreamPool()) objectGetter := newCommonSpaceGetter(s.id, s.aclList, s.cache, s.settingsDocument) s.syncService.Init(objectGetter) s.diffService.Init(initialIds) + s.settingsSync = periodicsync.NewPeriodicSync(SettingsSyncPeriodSeconds, func(ctx context.Context) error { + s.settingsDocument.Refresh() + return nil + }, log) return nil } @@ -190,7 +204,7 @@ func (s *space) DeriveTree(ctx context.Context, payload tree.ObjectTreeCreatePay Payload: payload, StreamPool: s.syncService.StreamPool(), Configuration: s.configuration, - HeadNotifiable: s.diffService, + HeadNotifiable: s.headNotifiable, Listener: listener, AclList: s.aclList, CreateStorage: s.storage.CreateTreeStorage, @@ -208,7 +222,7 @@ func (s *space) CreateTree(ctx context.Context, payload tree.ObjectTreeCreatePay Payload: payload, StreamPool: s.syncService.StreamPool(), Configuration: s.configuration, - HeadNotifiable: s.diffService, + HeadNotifiable: s.headNotifiable, Listener: listener, AclList: s.aclList, CreateStorage: s.storage.CreateTreeStorage, @@ -225,7 +239,7 @@ func (s *space) BuildTree(ctx context.Context, id string, listener updatelistene SpaceId: s.id, StreamPool: s.syncService.StreamPool(), Configuration: s.configuration, - HeadNotifiable: s.diffService, + HeadNotifiable: s.headNotifiable, Listener: listener, AclList: s.aclList, SpaceStorage: s.storage, @@ -250,6 +264,7 @@ func (s *space) Close() error { if err := s.syncService.Close(); err != nil { mError.Add(err) } + s.settingsSync.Close() if err := s.settingsDocument.Close(); err != nil { mError.Add(err) } diff --git a/common/commonspace/diffservice/periodicsync.go b/common/util/periodicsync/periodicsync.go similarity index 83% rename from common/commonspace/diffservice/periodicsync.go rename to common/util/periodicsync/periodicsync.go index a74b25cf..37647810 100644 --- a/common/commonspace/diffservice/periodicsync.go +++ b/common/util/periodicsync/periodicsync.go @@ -1,4 +1,4 @@ -package diffservice +package periodicsync import ( "context" @@ -11,7 +11,9 @@ type PeriodicSync interface { Close() } -func newPeriodicSync(periodSeconds int, syncer DiffSyncer, l *zap.Logger) *periodicSync { +type SyncerFunc func(ctx context.Context) error + +func NewPeriodicSync(periodSeconds int, syncer SyncerFunc, l *zap.Logger) PeriodicSync { ctx, cancel := context.WithCancel(context.Background()) return &periodicSync{ syncer: syncer, @@ -25,7 +27,7 @@ func newPeriodicSync(periodSeconds int, syncer DiffSyncer, l *zap.Logger) *perio type periodicSync struct { log *zap.Logger - syncer DiffSyncer + syncer SyncerFunc syncCtx context.Context syncCancel context.CancelFunc syncLoopDone chan struct{} @@ -42,7 +44,7 @@ func (p *periodicSync) syncLoop(periodSeconds int) { doSync := func() { ctx, cancel := context.WithTimeout(p.syncCtx, time.Minute) defer cancel() - if err := p.syncer.Sync(ctx); err != nil { + if err := p.syncer(ctx); err != nil { p.log.Warn("periodic sync error", zap.Error(err)) } } diff --git a/common/commonspace/diffservice/periodicsync_test.go b/common/util/periodicsync/periodicsync_test.go similarity index 55% rename from common/commonspace/diffservice/periodicsync_test.go rename to common/util/periodicsync/periodicsync_test.go index 068da366..c4463e41 100644 --- a/common/commonspace/diffservice/periodicsync_test.go +++ b/common/util/periodicsync/periodicsync_test.go @@ -1,9 +1,10 @@ -package diffservice +package periodicsync import ( + "context" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/logger" - "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/diffservice/mock_diffservice" "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" "testing" "time" ) @@ -14,25 +15,34 @@ func TestPeriodicSync_Run(t *testing.T) { defer ctrl.Finish() l := logger.NewNamed("sync") - diffSyncer := mock_diffservice.NewMockDiffSyncer(ctrl) + t.Run("diff syncer 1 time", func(t *testing.T) { secs := 0 - pSync := newPeriodicSync(secs, diffSyncer, l) - - diffSyncer.EXPECT().Sync(gomock.Any()).Times(1).Return(nil) + times := 0 + diffSyncer := func(ctx context.Context) (err error) { + times += 1 + return nil + } + pSync := NewPeriodicSync(secs, diffSyncer, l) pSync.Run() pSync.Close() + require.Equal(t, 1, times) }) t.Run("diff syncer 2 times", func(t *testing.T) { secs := 1 - pSync := newPeriodicSync(secs, diffSyncer, l) - diffSyncer.EXPECT().Sync(gomock.Any()).Times(2).Return(nil) + times := 0 + diffSyncer := func(ctx context.Context) (err error) { + times += 1 + return nil + } + pSync := NewPeriodicSync(secs, diffSyncer, l) pSync.Run() time.Sleep(time.Second * time.Duration(secs)) pSync.Close() + require.Equal(t, 2, times) }) } From 8854bbbe9575177adf576a056b1e56c65bd2175e Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Sat, 12 Nov 2022 22:55:12 +0100 Subject: [PATCH 08/27] Fix some document deletion bugs --- client/document/textdocument/textdocument.go | 4 ---- client/storage/spacestorage.go | 2 +- client/storage/treestorage.go | 8 ++++---- .../settingsdocument/settingsdocument.go | 15 ++++++++++----- common/commonspace/space.go | 10 +++++++--- common/commonspace/synctree/synctree.go | 4 ++-- common/pkg/acl/tree/changebuilder.go | 4 ++-- common/pkg/acl/tree/objecttree.go | 2 +- node/storage/keys.go | 2 +- node/storage/spacestorage.go | 2 +- 10 files changed, 29 insertions(+), 24 deletions(-) diff --git a/client/document/textdocument/textdocument.go b/client/document/textdocument/textdocument.go index 954160d9..4334afb9 100644 --- a/client/document/textdocument/textdocument.go +++ b/client/document/textdocument/textdocument.go @@ -106,7 +106,3 @@ func (t *textDocument) Text() (text string, err error) { func (t *textDocument) TreeDump() string { return t.TreeDump() } - -func (t *textDocument) Close() error { - return nil -} diff --git a/client/storage/spacestorage.go b/client/storage/spacestorage.go index 278abc0e..5c4f1866 100644 --- a/client/storage/spacestorage.go +++ b/client/storage/spacestorage.go @@ -78,7 +78,7 @@ func createSpaceStorage(db *badger.DB, payload spacestorage.SpaceStorageCreatePa _, err = spaceStore.CreateTreeStorage(storage.TreeStorageCreatePayload{ RootRawChange: payload.SpaceSettingsWithId, Changes: []*treechangeproto.RawTreeChangeWithId{payload.SpaceSettingsWithId}, - Heads: []string{payload.SpaceHeaderWithId.Id}, + Heads: []string{payload.SpaceSettingsWithId.Id}, }) if err != nil { return diff --git a/client/storage/treestorage.go b/client/storage/treestorage.go index d74d376c..a9403eed 100644 --- a/client/storage/treestorage.go +++ b/client/storage/treestorage.go @@ -166,15 +166,15 @@ func (t *treeStorage) storedKeys() (keys [][]byte, err error) { for it.Rewind(); it.Valid(); it.Next() { item := it.Item() key := item.Key() - // if it is a heads key - if len(key) <= len(t.keys.HeadsKey()) { - continue - } keyCopy := make([]byte, 0, len(key)) keyCopy = item.KeyCopy(key) keys = append(keys, keyCopy) } return nil }) + if err != nil { + return + } + keys = append(keys, t.keys.RootIdKey()) return } diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go index 5ba52732..54995af2 100644 --- a/common/commonspace/settingsdocument/settingsdocument.go +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -20,6 +20,7 @@ const ( type SettingsDocument interface { tree.ObjectTree + Init(ctx context.Context) (err error) Refresh() DeleteObject(id string) (err error) NotifyObjectUpdate(id string) @@ -48,10 +49,11 @@ type settingsDocument struct { lastChangeId string prov deletedIdsProvider removeNotifyFunc RemoveObjectsFunc + buildFunc BuildTreeFunc deletionStateLock sync.Mutex } -func NewSettingsDocument(ctx context.Context, deps Deps, spaceId string) (doc SettingsDocument, err error) { +func NewSettingsDocument(deps Deps, spaceId string) (doc SettingsDocument, err error) { s := &settingsDocument{ account: deps.Account, spaceId: spaceId, @@ -59,11 +61,9 @@ func NewSettingsDocument(ctx context.Context, deps Deps, spaceId string) (doc Se treeGetter: deps.TreeGetter, store: deps.Store, removeNotifyFunc: deps.RemoveFunc, + buildFunc: deps.BuildFunc, } - s.ObjectTree, err = deps.BuildFunc(ctx, deps.Store.SpaceSettingsId(), s) - if err != nil { - return - } + // this is needed mainly for testing if deps.prov == nil { s.prov = &provider{} @@ -99,6 +99,11 @@ func (s *settingsDocument) Rebuild(tr tree.ObjectTree) { s.toBeDeleted(ids) } +func (s *settingsDocument) Init(ctx context.Context) (err error) { + s.ObjectTree, err = s.buildFunc(ctx, s.store.SpaceSettingsId(), s) + return +} + func (s *settingsDocument) Refresh() { s.Lock() defer s.Unlock() diff --git a/common/commonspace/space.go b/common/commonspace/space.go index f8502ff5..8ee4ba5b 100644 --- a/common/commonspace/space.go +++ b/common/commonspace/space.go @@ -150,6 +150,7 @@ func (s *space) Init(ctx context.Context) (err error) { if err != nil { return } + s.aclList = syncacl.NewSyncACL(aclList, s.syncService.StreamPool()) deps := settingsdocument.Deps{ BuildFunc: s.BuildTree, @@ -158,7 +159,7 @@ func (s *space) Init(ctx context.Context) (err error) { Store: s.storage, RemoveFunc: s.diffService.RemoveObjects, } - s.settingsDocument, err = settingsdocument.NewSettingsDocument(context.Background(), deps, s.id) + s.settingsDocument, err = settingsdocument.NewSettingsDocument(deps, s.id) if err != nil { return } @@ -166,8 +167,10 @@ func (s *space) Init(ctx context.Context) (err error) { s.diffService.UpdateHeads(id, heads) s.settingsDocument.NotifyObjectUpdate(id) }) - s.settingsDocument.Refresh() - s.aclList = syncacl.NewSyncACL(aclList, s.syncService.StreamPool()) + err = s.settingsDocument.Init(ctx) + if err != nil { + return + } objectGetter := newCommonSpaceGetter(s.id, s.aclList, s.cache, s.settingsDocument) s.syncService.Init(objectGetter) s.diffService.Init(initialIds) @@ -175,6 +178,7 @@ func (s *space) Init(ctx context.Context) (err error) { s.settingsDocument.Refresh() return nil }, log) + s.settingsSync.Run() return nil } diff --git a/common/commonspace/synctree/synctree.go b/common/commonspace/synctree/synctree.go index 90a346ae..de1a06c5 100644 --- a/common/commonspace/synctree/synctree.go +++ b/common/commonspace/synctree/synctree.go @@ -277,8 +277,8 @@ func (s *SyncTree) Close() (err error) { log.With("id", s.ID()).Debug("closing sync tree") s.Lock() defer s.Unlock() - if err = s.checkAlive(); err != nil { - return + if s.isClosed { + return ErrSyncTreeClosed } s.isClosed = true return diff --git a/common/pkg/acl/tree/changebuilder.go b/common/pkg/acl/tree/changebuilder.go index ace60b23..5547b332 100644 --- a/common/pkg/acl/tree/changebuilder.go +++ b/common/pkg/acl/tree/changebuilder.go @@ -156,7 +156,8 @@ func (c *changeBuilder) BuildContent(payload BuilderContent) (ch *Change, rawIdC IsSnapshot: payload.IsSnapshot, } if payload.ReadKey != nil { - encrypted, err := payload.ReadKey.Encrypt(payload.Content) + var encrypted []byte + encrypted, err = payload.ReadKey.Encrypt(payload.Content) if err != nil { return } @@ -191,7 +192,6 @@ func (c *changeBuilder) BuildContent(payload BuilderContent) (ch *Change, rawIdC } ch = NewChange(id, change, signature) - ch.Model = payload.Content rawIdChange = &treechangeproto.RawTreeChangeWithId{ RawChange: marshalledRawChange, diff --git a/common/pkg/acl/tree/objecttree.go b/common/pkg/acl/tree/objecttree.go index 00b009c2..bb8dfc6a 100644 --- a/common/pkg/acl/tree/objecttree.go +++ b/common/pkg/acl/tree/objecttree.go @@ -528,7 +528,7 @@ func (ot *objectTree) Close() error { } func (ot *objectTree) Delete() error { - return nil + return ot.treeStorage.Delete() } func (ot *objectTree) SnapshotPath() []string { diff --git a/node/storage/keys.go b/node/storage/keys.go index 5dabc5da..ef728d74 100644 --- a/node/storage/keys.go +++ b/node/storage/keys.go @@ -47,7 +47,7 @@ func (t treeKeys) RawChangeKey(id string) []byte { } func (t treeKeys) isTreeRecordKey(key string) bool { - return strings.HasPrefix(key, t.prefix) && !strings.HasSuffix(key, "/heads") + return strings.HasPrefix(key, t.prefix) } type spaceKeys struct { diff --git a/node/storage/spacestorage.go b/node/storage/spacestorage.go index e700e0da..232dc838 100644 --- a/node/storage/spacestorage.go +++ b/node/storage/spacestorage.go @@ -137,7 +137,7 @@ func createSpaceStorage(rootPath string, payload spacestorage.SpaceStorageCreate _, err = store.CreateTreeStorage(storage.TreeStorageCreatePayload{ RootRawChange: payload.SpaceSettingsWithId, Changes: []*treechangeproto.RawTreeChangeWithId{payload.SpaceSettingsWithId}, - Heads: []string{payload.SpaceHeaderWithId.Id}, + Heads: []string{payload.SpaceSettingsWithId.Id}, }) if err != nil { return From 0d9360490db8f3fbc7ca1933e4f8d8a224b063f2 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Sun, 13 Nov 2022 09:35:47 +0100 Subject: [PATCH 09/27] Change deletion logic --- .../settingsdocument/settingsdocument.go | 77 ++++++++----------- .../settingsdocument/settingsqueue.go | 69 +++++++++++++++++ 2 files changed, 101 insertions(+), 45 deletions(-) create mode 100644 common/commonspace/settingsdocument/settingsqueue.go diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go index 54995af2..1971e00f 100644 --- a/common/commonspace/settingsdocument/settingsdocument.go +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -8,14 +8,6 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree/updatelistener" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" - "sync" -) - -type DeletionState int - -const ( - DeletionStateQueued DeletionState = iota - DeletionStateDeleted ) type SettingsDocument interface { @@ -41,23 +33,24 @@ type Deps struct { type settingsDocument struct { tree.ObjectTree - account account.Service - spaceId string - deletionState map[string]DeletionState - treeGetter treegetter.TreeGetter - store spacestorage.SpaceStorage - lastChangeId string - prov deletedIdsProvider - removeNotifyFunc RemoveObjectsFunc - buildFunc BuildTreeFunc - deletionStateLock sync.Mutex + account account.Service + spaceId string + treeGetter treegetter.TreeGetter + store spacestorage.SpaceStorage + prov deletedIdsProvider + removeNotifyFunc RemoveObjectsFunc + buildFunc BuildTreeFunc + + queue *settingsQueue + documentIds []string + lastChangeId string } func NewSettingsDocument(deps Deps, spaceId string) (doc SettingsDocument, err error) { s := &settingsDocument{ account: deps.Account, spaceId: spaceId, - deletionState: map[string]DeletionState{}, + queue: newSettingsQueue(), treeGetter: deps.TreeGetter, store: deps.Store, removeNotifyFunc: deps.RemoveFunc, @@ -73,12 +66,7 @@ func NewSettingsDocument(deps Deps, spaceId string) (doc SettingsDocument, err e } func (s *settingsDocument) NotifyObjectUpdate(id string) { - s.deletionStateLock.Lock() - if state, exists := s.deletionState[id]; exists && state == DeletionStateDeleted { - // marking the document as queued, that means that document appeared later than we checked the storage for deletion - s.deletionState[id] = DeletionStateQueued - } - s.deletionStateLock.Unlock() + s.queue.queueIfDeleted(id) } func (s *settingsDocument) Update(tr tree.ObjectTree) { @@ -86,8 +74,10 @@ func (s *settingsDocument) Update(tr tree.ObjectTree) { if err != nil { return } + s.documentIds = append(s.documentIds, ids...) s.lastChangeId = lastId - s.toBeDeleted(ids) + s.queue.add(ids) + s.deleteQueued() } func (s *settingsDocument) Rebuild(tr tree.ObjectTree) { @@ -95,8 +85,10 @@ func (s *settingsDocument) Rebuild(tr tree.ObjectTree) { if err != nil { return } + s.documentIds = ids s.lastChangeId = lastId - s.toBeDeleted(ids) + s.queue.add(ids) + s.deleteQueued() } func (s *settingsDocument) Init(ctx context.Context) (err error) { @@ -107,40 +99,35 @@ func (s *settingsDocument) Init(ctx context.Context) (err error) { func (s *settingsDocument) Refresh() { s.Lock() defer s.Unlock() - s.Rebuild(s) + if s.lastChangeId == "" { + s.Rebuild(s) + } else { + s.deleteQueued() + } } -func (s *settingsDocument) toBeDeleted(ids []string) { - for _, id := range ids { - s.deletionStateLock.Lock() - if state, exists := s.deletionState[id]; exists && state == DeletionStateDeleted { - s.deletionStateLock.Unlock() - continue - } - // if the document is not in storage it can happen that it will appear later, for that we have NotifyObjectUpdate method +func (s *settingsDocument) deleteQueued() { + allQueued := s.queue.getQueued() + for _, id := range allQueued { if _, err := s.store.TreeStorage(id); err == nil { - s.deletionState[id] = DeletionStateQueued - s.deletionStateLock.Unlock() - // doing this without lock err := s.treeGetter.DeleteTree(context.Background(), s.spaceId, id) if err != nil { // TODO: some errors may tell us that the tree is actually deleted, so we should have more checks here // TODO: add logging continue } - s.deletionStateLock.Lock() } - - s.deletionState[id] = DeletionStateDeleted - s.deletionStateLock.Unlock() + s.queue.delete(id) } - // notifying diff service that the ids should not be synced anymore - s.removeNotifyFunc(ids) } func (s *settingsDocument) DeleteObject(id string) (err error) { s.Lock() defer s.Unlock() + if s.queue.exists(id) { + return nil + } + content := &spacesyncproto.SpaceSettingsContent_ObjectDelete{ ObjectDelete: &spacesyncproto.ObjectDelete{Id: id}, } diff --git a/common/commonspace/settingsdocument/settingsqueue.go b/common/commonspace/settingsdocument/settingsqueue.go new file mode 100644 index 00000000..52a7329f --- /dev/null +++ b/common/commonspace/settingsdocument/settingsqueue.go @@ -0,0 +1,69 @@ +package settingsdocument + +import "sync" + +type settingsQueue struct { + sync.Mutex + queued map[string]struct{} + deleted map[string]struct{} +} + +func newSettingsQueue() *settingsQueue { + return &settingsQueue{ + Mutex: sync.Mutex{}, + queued: map[string]struct{}{}, + deleted: map[string]struct{}{}, + } +} + +func (q *settingsQueue) add(ids []string) { + q.Lock() + defer q.Unlock() + for _, id := range ids { + if _, exists := q.deleted[id]; exists { + continue + } + if _, exists := q.queued[id]; exists { + continue + } + q.queued[id] = struct{}{} + } +} + +func (q *settingsQueue) getQueued() (ids []string) { + q.Lock() + defer q.Unlock() + ids = make([]string, 0, len(q.queued)) + for id := range q.queued { + ids = append(ids, id) + } + return +} + +func (q *settingsQueue) delete(id string) { + q.Lock() + defer q.Unlock() + delete(q.queued, id) + q.deleted[id] = struct{}{} +} + +func (q *settingsQueue) queueIfDeleted(id string) { + q.Lock() + defer q.Unlock() + if _, exists := q.deleted[id]; exists { + delete(q.deleted, id) + q.queued[id] = struct{}{} + } +} + +func (q *settingsQueue) exists(id string) bool { + q.Lock() + defer q.Unlock() + if _, exists := q.deleted[id]; exists { + return true + } + if _, exists := q.queued[id]; exists { + return true + } + return false +} From b9f4ddd4807a0cbacee587aa972621a72cc8680f Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Sun, 13 Nov 2022 14:16:58 +0100 Subject: [PATCH 10/27] Add remove notifial --- common/commonspace/settingsdocument/settingsdocument.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go index 1971e00f..ca275663 100644 --- a/common/commonspace/settingsdocument/settingsdocument.go +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -77,6 +77,7 @@ func (s *settingsDocument) Update(tr tree.ObjectTree) { s.documentIds = append(s.documentIds, ids...) s.lastChangeId = lastId s.queue.add(ids) + s.removeNotifyFunc(ids) s.deleteQueued() } @@ -88,6 +89,7 @@ func (s *settingsDocument) Rebuild(tr tree.ObjectTree) { s.documentIds = ids s.lastChangeId = lastId s.queue.add(ids) + s.removeNotifyFunc(ids) s.deleteQueued() } From f6d818300116f497203bdad75181319e6be251f1 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 24 Nov 2022 01:17:56 +0100 Subject: [PATCH 11/27] Simplify deletion pipeline and change other stuff --- common/commonspace/commonstorage.go | 28 ++++ common/commonspace/diffservice/diffservice.go | 6 +- common/commonspace/diffservice/diffsyncer.go | 40 ++---- .../diffservice/diffsyncer_test.go | 2 +- .../settingsdocument/deleteloop.go | 53 ++++++++ .../commonspace/settingsdocument/deleter.go | 36 +++++ .../deletionstate/deletionstate.go | 126 ++++++++++++++++++ .../{provider.go => idprovider.go} | 0 .../settingsdocument/settingsdocument.go | 102 +++++++------- .../settingsdocument/settingsqueue.go | 69 ---------- common/commonspace/space.go | 45 +++---- common/commonspace/storage/storage.go | 15 ++- common/commonspace/synctree/synctree.go | 15 ++- common/pkg/acl/storage/provider.go | 8 +- node/storage/treestorage.go | 2 +- 15 files changed, 362 insertions(+), 185 deletions(-) create mode 100644 common/commonspace/commonstorage.go create mode 100644 common/commonspace/settingsdocument/deleteloop.go create mode 100644 common/commonspace/settingsdocument/deleter.go create mode 100644 common/commonspace/settingsdocument/deletionstate/deletionstate.go rename common/commonspace/settingsdocument/{provider.go => idprovider.go} (100%) delete mode 100644 common/commonspace/settingsdocument/settingsqueue.go diff --git a/common/commonspace/commonstorage.go b/common/commonspace/commonstorage.go new file mode 100644 index 00000000..ab6d5ca6 --- /dev/null +++ b/common/commonspace/commonstorage.go @@ -0,0 +1,28 @@ +package commonspace + +import ( + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" + treestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/storage" +) + +type commonStorage struct { + storage.SpaceStorage +} + +func newCommonStorage(spaceStorage storage.SpaceStorage) storage.SpaceStorage { + return &commonStorage{ + SpaceStorage: spaceStorage, + } +} + +func (c *commonStorage) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (store treestorage.TreeStorage, err error) { + status, err := c.TreeDeletedStatus(payload.RootRawChange.Id) + if err != nil { + return + } + if status == "" { + return c.CreateTreeStorage(payload) + } + err = storage.ErrTreeStorageAlreadyDeleted + return +} diff --git a/common/commonspace/diffservice/diffservice.go b/common/commonspace/diffservice/diffservice.go index 8888a111..1c5df8b3 100644 --- a/common/commonspace/diffservice/diffservice.go +++ b/common/commonspace/diffservice/diffservice.go @@ -4,6 +4,7 @@ package diffservice import ( "context" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/remotediff" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter" @@ -20,7 +21,7 @@ type DiffService interface { RemoveObjects(ids []string) AllIds() []string - Init(objectIds []string) + Init(objectIds []string, deletionState *deletionstate.DeletionState) Close() (err error) } @@ -60,8 +61,9 @@ func NewDiffService( } } -func (d *diffService) Init(objectIds []string) { +func (d *diffService) Init(objectIds []string, deletionState *deletionstate.DeletionState) { d.fillDiff(objectIds) + d.syncer.Init(deletionState) d.periodicSync.Run() } diff --git a/common/commonspace/diffservice/diffsyncer.go b/common/commonspace/diffservice/diffsyncer.go index 31c5071a..4f92d570 100644 --- a/common/commonspace/diffservice/diffsyncer.go +++ b/common/commonspace/diffservice/diffsyncer.go @@ -3,6 +3,7 @@ package diffservice import ( "context" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/remotediff" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter" @@ -11,7 +12,6 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/nodeconf" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/ldiff" "go.uber.org/zap" - "sync" "time" ) @@ -19,6 +19,7 @@ type DiffSyncer interface { Sync(ctx context.Context) error RemoveObjects(ids []string) UpdateHeads(id string, heads []string) + Init(deletionState *deletionstate.DeletionState) } func newDiffSyncer( @@ -37,12 +38,10 @@ func newDiffSyncer( confConnector: confConnector, clientFactory: clientFactory, log: log, - removedIds: map[string]struct{}{}, } } type diffSyncer struct { - sync.Mutex spaceId string diff ldiff.Diff confConnector nodeconf.ConfConnector @@ -50,22 +49,22 @@ type diffSyncer struct { storage storage.SpaceStorage clientFactory spacesyncproto.ClientFactory log *zap.Logger - removedIds map[string]struct{} + deletionState *deletionstate.DeletionState +} + +func (d *diffSyncer) Init(deletionState *deletionstate.DeletionState) { + d.deletionState = deletionState + d.deletionState.AddObserver(d.RemoveObjects) } func (d *diffSyncer) RemoveObjects(ids []string) { - d.Lock() - defer d.Unlock() for _, id := range ids { d.diff.RemoveId(id) - d.removedIds[id] = struct{}{} } } func (d *diffSyncer) UpdateHeads(id string, heads []string) { - d.Lock() - defer d.Unlock() - if _, exists := d.removedIds[id]; exists { + if d.deletionState.Exists(id) { return } d.diff.Set(ldiff.Element{ @@ -101,30 +100,17 @@ func (d *diffSyncer) syncWithPeer(ctx context.Context, p peer.Peer) (err error) if err == spacesyncproto.ErrSpaceMissing { return d.sendPushSpaceRequest(ctx, cl) } - var afterFilterIds []string - filter := func(ids []string) { - for _, id := range ids { - if _, exists := d.removedIds[id]; !exists { - afterFilterIds = append(afterFilterIds, id) - } - } - } - d.Lock() - totalLen := 0 + totalLen := len(newIds) + len(changedIds) + len(removedIds) // not syncing ids which were removed through settings document - for _, ids := range [][]string{newIds, changedIds, removedIds} { - totalLen += len(ids) - filter(ids) - } - d.Unlock() + filteredIds := d.deletionState.FilterJoin(newIds, changedIds, removedIds) ctx = peer.CtxWithPeerId(ctx, p.Id()) - d.pingTreesInCache(ctx, afterFilterIds) + d.pingTreesInCache(ctx, filteredIds) d.log.Info("sync done:", zap.Int("newIds", len(newIds)), zap.Int("changedIds", len(changedIds)), zap.Int("removedIds", len(removedIds)), - zap.Int("filteredIds", totalLen-len(afterFilterIds))) + zap.Int("already deleted ids", totalLen-len(filteredIds))) return } diff --git a/common/commonspace/diffservice/diffsyncer_test.go b/common/commonspace/diffservice/diffsyncer_test.go index 13001df8..e6074a71 100644 --- a/common/commonspace/diffservice/diffsyncer_test.go +++ b/common/commonspace/diffservice/diffsyncer_test.go @@ -34,7 +34,7 @@ func (p pushSpaceRequestMatcher) Matches(x interface{}) bool { return false } - return res.AclPayloadId == p.aclRootId && res.SpaceHeader == p.spaceHeader + return res.Payload.AclPayloadId == p.aclRootId && res.Payload.SpaceHeader == p.spaceHeader } func (p pushSpaceRequestMatcher) String() string { diff --git a/common/commonspace/settingsdocument/deleteloop.go b/common/commonspace/settingsdocument/deleteloop.go new file mode 100644 index 00000000..f122fd83 --- /dev/null +++ b/common/commonspace/settingsdocument/deleteloop.go @@ -0,0 +1,53 @@ +package settingsdocument + +import ( + "context" +) + +type deleteLoop struct { + deleteCtx context.Context + deleteCancel context.CancelFunc + deleteChan chan struct{} + deleteFunc func() + loopDone chan struct{} +} + +func newDeleteLoop(deleteFunc func()) *deleteLoop { + ctx, cancel := context.WithCancel(context.Background()) + return &deleteLoop{ + deleteCtx: ctx, + deleteCancel: cancel, + deleteChan: make(chan struct{}, 1), + deleteFunc: deleteFunc, + loopDone: make(chan struct{}), + } +} + +func (dl *deleteLoop) Run() { + go dl.loop() +} + +func (dl *deleteLoop) loop() { + defer close(dl.loopDone) + dl.deleteFunc() + for { + select { + case <-dl.deleteCtx.Done(): + return + case <-dl.deleteChan: + dl.deleteFunc() + } + } +} + +func (dl *deleteLoop) notify() { + select { + case dl.deleteChan <- struct{}{}: + default: + } +} + +func (dl *deleteLoop) Close() { + dl.deleteCancel() + <-dl.loopDone +} diff --git a/common/commonspace/settingsdocument/deleter.go b/common/commonspace/settingsdocument/deleter.go new file mode 100644 index 00000000..48b6c871 --- /dev/null +++ b/common/commonspace/settingsdocument/deleter.go @@ -0,0 +1,36 @@ +package settingsdocument + +import ( + "context" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter" + "go.uber.org/zap" +) + +type deleter struct { + st storage.SpaceStorage + state *deletionstate.DeletionState + getter treegetter.TreeGetter +} + +func newDeleter(st storage.SpaceStorage, state *deletionstate.DeletionState, getter treegetter.TreeGetter) *deleter { + return &deleter{st, state, getter} +} + +func (d *deleter) delete() { + allQueued := d.state.GetQueued() + for _, id := range allQueued { + if _, err := d.st.TreeStorage(id); err == nil { + err := d.getter.DeleteTree(context.Background(), d.st.Id(), id) + if err != nil { + log.With(zap.String("id", id), zap.Error(err)).Error("failed to delete object") + continue + } + } + err := d.state.Delete(id) + if err != nil { + log.With(zap.String("id", id), zap.Error(err)).Error("failed to mark object as deleted") + } + } +} diff --git a/common/commonspace/settingsdocument/deletionstate/deletionstate.go b/common/commonspace/settingsdocument/deletionstate/deletionstate.go new file mode 100644 index 00000000..3387dd9d --- /dev/null +++ b/common/commonspace/settingsdocument/deletionstate/deletionstate.go @@ -0,0 +1,126 @@ +package deletionstate + +import ( + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" + "sync" +) + +type StateUpdateObserver func(ids []string) + +type DeletionState struct { + sync.RWMutex + queued map[string]struct{} + deleted map[string]struct{} + stateUpdateObservers []StateUpdateObserver + storage storage.SpaceStorage +} + +func NewDeletionState(storage storage.SpaceStorage) *DeletionState { + return &DeletionState{ + queued: map[string]struct{}{}, + deleted: map[string]struct{}{}, + storage: storage, + } +} + +func (st *DeletionState) AddObserver(observer StateUpdateObserver) { + st.Lock() + defer st.Unlock() + st.stateUpdateObservers = append(st.stateUpdateObservers, observer) +} + +func (st *DeletionState) Add(ids []string) (err error) { + st.Lock() + defer func() { + st.Unlock() + if err != nil { + return + } + for _, ob := range st.stateUpdateObservers { + ob(ids) + } + }() + + for _, id := range ids { + if _, exists := st.deleted[id]; exists { + continue + } + if _, exists := st.queued[id]; exists { + continue + } + + var status string + status, err = st.storage.TreeDeletedStatus(id) + if err != nil { + return + } + + switch status { + case storage.TreeDeletedStatusQueued: + st.queued[id] = struct{}{} + case storage.TreeDeletedStatusDeleted: + st.deleted[id] = struct{}{} + default: + st.queued[id] = struct{}{} + err = st.storage.SetTreeDeletedStatus(id, storage.TreeDeletedStatusQueued) + if err != nil { + return + } + } + } + return +} + +func (st *DeletionState) GetQueued() (ids []string) { + st.RLock() + defer st.RUnlock() + ids = make([]string, 0, len(st.queued)) + for id := range st.queued { + ids = append(ids, id) + } + return +} + +func (st *DeletionState) Delete(id string) (err error) { + st.Lock() + defer st.Unlock() + delete(st.queued, id) + st.deleted[id] = struct{}{} + err = st.storage.SetTreeDeletedStatus(id, storage.TreeDeletedStatusQueued) + if err != nil { + return + } + return +} + +func (st *DeletionState) Exists(id string) bool { + st.RLock() + defer st.RUnlock() + return st.exists(id) +} + +func (st *DeletionState) FilterJoin(ids ...[]string) (filtered []string) { + st.RLock() + defer st.RUnlock() + filter := func(ids []string) { + for _, id := range ids { + if !st.exists(id) { + filtered = append(filtered, id) + } + } + } + for _, arr := range ids { + filter(arr) + } + return +} + +func (st *DeletionState) exists(id string) bool { + if _, exists := st.deleted[id]; exists { + return true + } + if _, exists := st.queued[id]; exists { + return true + } + return false +} diff --git a/common/commonspace/settingsdocument/provider.go b/common/commonspace/settingsdocument/idprovider.go similarity index 100% rename from common/commonspace/settingsdocument/provider.go rename to common/commonspace/settingsdocument/idprovider.go diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go index ca275663..3060cb20 100644 --- a/common/commonspace/settingsdocument/settingsdocument.go +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -3,98 +3,108 @@ package settingsdocument import ( "context" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/account" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/logger" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" spacestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree/updatelistener" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" + "go.uber.org/zap" ) +var log = logger.NewNamed("commonspace.settingsdocument") + type SettingsDocument interface { tree.ObjectTree Init(ctx context.Context) (err error) Refresh() DeleteObject(id string) (err error) - NotifyObjectUpdate(id string) } type BuildTreeFunc func(ctx context.Context, id string, listener updatelistener.UpdateListener) (t tree.ObjectTree, err error) -type RemoveObjectsFunc func([]string) type Deps struct { - BuildFunc BuildTreeFunc - Account account.Service - TreeGetter treegetter.TreeGetter - Store spacestorage.SpaceStorage - RemoveFunc RemoveObjectsFunc + BuildFunc BuildTreeFunc + Account account.Service + TreeGetter treegetter.TreeGetter + Store spacestorage.SpaceStorage + DeletionState *deletionstate.DeletionState // prov exists mainly for the ease of testing prov deletedIdsProvider } type settingsDocument struct { tree.ObjectTree - account account.Service - spaceId string - treeGetter treegetter.TreeGetter - store spacestorage.SpaceStorage - prov deletedIdsProvider - removeNotifyFunc RemoveObjectsFunc - buildFunc BuildTreeFunc + account account.Service + spaceId string + treeGetter treegetter.TreeGetter + store spacestorage.SpaceStorage + prov deletedIdsProvider + buildFunc BuildTreeFunc + loop deleteLoop - queue *settingsQueue - documentIds []string - lastChangeId string + deletionState *deletionstate.DeletionState + lastChangeId string } func NewSettingsDocument(deps Deps, spaceId string) (doc SettingsDocument, err error) { + deleter := newDeleter(deps.Store, deps.DeletionState, deps.TreeGetter) + loop := newDeleteLoop(func() { + deleter.delete() + }) + deps.DeletionState.AddObserver(func(ids []string) { + loop.notify() + }) + s := &settingsDocument{ - account: deps.Account, - spaceId: spaceId, - queue: newSettingsQueue(), - treeGetter: deps.TreeGetter, - store: deps.Store, - removeNotifyFunc: deps.RemoveFunc, - buildFunc: deps.BuildFunc, + spaceId: spaceId, + account: deps.Account, + deletionState: deps.DeletionState, + treeGetter: deps.TreeGetter, + store: deps.Store, + buildFunc: deps.BuildFunc, } // this is needed mainly for testing if deps.prov == nil { s.prov = &provider{} } + doc = s return } -func (s *settingsDocument) NotifyObjectUpdate(id string) { - s.queue.queueIfDeleted(id) -} - func (s *settingsDocument) Update(tr tree.ObjectTree) { ids, lastId, err := s.prov.ProvideIds(tr, s.lastChangeId) if err != nil { + log.With(zap.Strings("ids", ids), zap.Error(err)).Error("failed to update state") return } - s.documentIds = append(s.documentIds, ids...) s.lastChangeId = lastId - s.queue.add(ids) - s.removeNotifyFunc(ids) - s.deleteQueued() + if err = s.deletionState.Add(ids); err != nil { + log.With(zap.Strings("ids", ids), zap.Error(err)).Error("failed to queue ids to delete") + } } func (s *settingsDocument) Rebuild(tr tree.ObjectTree) { ids, lastId, err := s.prov.ProvideIds(tr, "") if err != nil { + log.With(zap.Strings("ids", ids), zap.Error(err)).Error("failed to rebuild state") return } - s.documentIds = ids s.lastChangeId = lastId - s.queue.add(ids) - s.removeNotifyFunc(ids) - s.deleteQueued() + if err = s.deletionState.Add(ids); err != nil { + log.With(zap.Strings("ids", ids), zap.Error(err)).Error("failed to queue ids to delete") + } } func (s *settingsDocument) Init(ctx context.Context) (err error) { s.ObjectTree, err = s.buildFunc(ctx, s.store.SpaceSettingsId(), s) + if err != nil { + return + } + s.loop.Run() return } @@ -104,29 +114,19 @@ func (s *settingsDocument) Refresh() { if s.lastChangeId == "" { s.Rebuild(s) } else { - s.deleteQueued() + s.Update(s) } } -func (s *settingsDocument) deleteQueued() { - allQueued := s.queue.getQueued() - for _, id := range allQueued { - if _, err := s.store.TreeStorage(id); err == nil { - err := s.treeGetter.DeleteTree(context.Background(), s.spaceId, id) - if err != nil { - // TODO: some errors may tell us that the tree is actually deleted, so we should have more checks here - // TODO: add logging - continue - } - } - s.queue.delete(id) - } +func (s *settingsDocument) Close() error { + s.loop.Close() + return s.ObjectTree.Close() } func (s *settingsDocument) DeleteObject(id string) (err error) { s.Lock() defer s.Unlock() - if s.queue.exists(id) { + if s.deletionState.Exists(id) { return nil } diff --git a/common/commonspace/settingsdocument/settingsqueue.go b/common/commonspace/settingsdocument/settingsqueue.go deleted file mode 100644 index 52a7329f..00000000 --- a/common/commonspace/settingsdocument/settingsqueue.go +++ /dev/null @@ -1,69 +0,0 @@ -package settingsdocument - -import "sync" - -type settingsQueue struct { - sync.Mutex - queued map[string]struct{} - deleted map[string]struct{} -} - -func newSettingsQueue() *settingsQueue { - return &settingsQueue{ - Mutex: sync.Mutex{}, - queued: map[string]struct{}{}, - deleted: map[string]struct{}{}, - } -} - -func (q *settingsQueue) add(ids []string) { - q.Lock() - defer q.Unlock() - for _, id := range ids { - if _, exists := q.deleted[id]; exists { - continue - } - if _, exists := q.queued[id]; exists { - continue - } - q.queued[id] = struct{}{} - } -} - -func (q *settingsQueue) getQueued() (ids []string) { - q.Lock() - defer q.Unlock() - ids = make([]string, 0, len(q.queued)) - for id := range q.queued { - ids = append(ids, id) - } - return -} - -func (q *settingsQueue) delete(id string) { - q.Lock() - defer q.Unlock() - delete(q.queued, id) - q.deleted[id] = struct{}{} -} - -func (q *settingsQueue) queueIfDeleted(id string) { - q.Lock() - defer q.Unlock() - if _, exists := q.deleted[id]; exists { - delete(q.deleted, id) - q.queued[id] = struct{}{} - } -} - -func (q *settingsQueue) exists(id string) bool { - q.Lock() - defer q.Unlock() - if _, exists := q.deleted[id]; exists { - return true - } - if _, exists := q.queued[id]; exists { - return true - } - return false -} diff --git a/common/commonspace/space.go b/common/commonspace/space.go index 8ee4ba5b..9a1f39ff 100644 --- a/common/commonspace/space.go +++ b/common/commonspace/space.go @@ -7,6 +7,7 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/account" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/diffservice" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/syncacl" @@ -19,7 +20,6 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/util/keys/asymmetric/encryptionkey" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/util/keys/asymmetric/signingkey" - "github.com/anytypeio/go-anytype-infrastructure-experiments/common/util/periodicsync" "github.com/zeebo/errs" "go.uber.org/zap" "sync" @@ -96,8 +96,6 @@ type space struct { aclList *syncacl.SyncACL configuration nodeconf.Configuration settingsDocument settingsdocument.SettingsDocument - settingsSync periodicsync.PeriodicSync - headNotifiable diffservice.HeadNotifiable isClosed atomic.Bool } @@ -132,6 +130,8 @@ func (s *space) Description() (desc SpaceDescription, err error) { } func (s *space) Init(ctx context.Context) (err error) { + s.storage = newCommonStorage(s.storage) + header, err := s.storage.SpaceHeader() if err != nil { return @@ -152,33 +152,27 @@ func (s *space) Init(ctx context.Context) (err error) { } s.aclList = syncacl.NewSyncACL(aclList, s.syncService.StreamPool()) + deletionState := deletionstate.NewDeletionState(s.storage) deps := settingsdocument.Deps{ - BuildFunc: s.BuildTree, - Account: s.account, - TreeGetter: s.cache, - Store: s.storage, - RemoveFunc: s.diffService.RemoveObjects, + BuildFunc: s.BuildTree, + Account: s.account, + TreeGetter: s.cache, + Store: s.storage, + DeletionState: deletionState, } s.settingsDocument, err = settingsdocument.NewSettingsDocument(deps, s.id) if err != nil { return } - s.headNotifiable = diffservice.HeadNotifiableFunc(func(id string, heads []string) { - s.diffService.UpdateHeads(id, heads) - s.settingsDocument.NotifyObjectUpdate(id) - }) + + objectGetter := newCommonSpaceGetter(s.id, s.aclList, s.cache, s.settingsDocument) + s.syncService.Init(objectGetter) + s.diffService.Init(initialIds, deletionState) err = s.settingsDocument.Init(ctx) if err != nil { return } - objectGetter := newCommonSpaceGetter(s.id, s.aclList, s.cache, s.settingsDocument) - s.syncService.Init(objectGetter) - s.diffService.Init(initialIds) - s.settingsSync = periodicsync.NewPeriodicSync(SettingsSyncPeriodSeconds, func(ctx context.Context) error { - s.settingsDocument.Refresh() - return nil - }, log) - s.settingsSync.Run() + return nil } @@ -208,10 +202,10 @@ func (s *space) DeriveTree(ctx context.Context, payload tree.ObjectTreeCreatePay Payload: payload, StreamPool: s.syncService.StreamPool(), Configuration: s.configuration, - HeadNotifiable: s.headNotifiable, + HeadNotifiable: s.diffService, Listener: listener, AclList: s.aclList, - CreateStorage: s.storage.CreateTreeStorage, + SpaceStorage: s.storage, } return synctree.DeriveSyncTree(ctx, deps) } @@ -226,10 +220,10 @@ func (s *space) CreateTree(ctx context.Context, payload tree.ObjectTreeCreatePay Payload: payload, StreamPool: s.syncService.StreamPool(), Configuration: s.configuration, - HeadNotifiable: s.headNotifiable, + HeadNotifiable: s.diffService, Listener: listener, AclList: s.aclList, - CreateStorage: s.storage.CreateTreeStorage, + SpaceStorage: s.storage, } return synctree.CreateSyncTree(ctx, deps) } @@ -243,7 +237,7 @@ func (s *space) BuildTree(ctx context.Context, id string, listener updatelistene SpaceId: s.id, StreamPool: s.syncService.StreamPool(), Configuration: s.configuration, - HeadNotifiable: s.headNotifiable, + HeadNotifiable: s.diffService, Listener: listener, AclList: s.aclList, SpaceStorage: s.storage, @@ -268,7 +262,6 @@ func (s *space) Close() error { if err := s.syncService.Close(); err != nil { mError.Add(err) } - s.settingsSync.Close() if err := s.settingsDocument.Close(); err != nil { mError.Add(err) } diff --git a/common/commonspace/storage/storage.go b/common/commonspace/storage/storage.go index ba9d0e26..bbca9c55 100644 --- a/common/commonspace/storage/storage.go +++ b/common/commonspace/storage/storage.go @@ -12,12 +12,23 @@ import ( const CName = "commonspace.storage" -var ErrSpaceStorageExists = errors.New("space storage exists") -var ErrSpaceStorageMissing = errors.New("space storage missing") +var ( + ErrSpaceStorageExists = errors.New("space storage exists") + ErrSpaceStorageMissing = errors.New("space storage missing") + + ErrTreeStorageAlreadyDeleted = errors.New("tree storage already deleted") +) + +const ( + TreeDeletedStatusQueued = "queued" + TreeDeletedStatusDeleted = "deleted" +) type SpaceStorage interface { storage.Provider Id() string + SetTreeDeletedStatus(id, state string) error + TreeDeletedStatus(id string) (string, error) SpaceSettingsId() string ACLStorage() (storage.ListStorage, error) SpaceHeader() (*spacesyncproto.RawSpaceHeaderWithId, error) diff --git a/common/commonspace/synctree/synctree.go b/common/commonspace/synctree/synctree.go index de1a06c5..3647ecc7 100644 --- a/common/commonspace/synctree/synctree.go +++ b/common/commonspace/synctree/synctree.go @@ -50,7 +50,7 @@ type CreateDeps struct { StreamPool syncservice.StreamPool Listener updatelistener.UpdateListener AclList list.ACLList - CreateStorage storage.TreeStorageCreatorFunc + SpaceStorage spacestorage.SpaceStorage } type BuildDeps struct { @@ -65,7 +65,7 @@ type BuildDeps struct { } func DeriveSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, err error) { - t, err = createDerivedObjectTree(deps.Payload, deps.AclList, deps.CreateStorage) + t, err = createDerivedObjectTree(deps.Payload, deps.AclList, deps.SpaceStorage.CreateTreeStorage) if err != nil { return } @@ -83,6 +83,9 @@ func DeriveSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, er syncHandler := newSyncTreeHandler(syncTree, syncClient) syncTree.SyncHandler = syncHandler t = syncTree + syncTree.Lock() + defer syncTree.Unlock() + syncTree.listener.Rebuild(syncTree) headUpdate := syncClient.CreateHeadUpdate(t, nil) err = syncClient.BroadcastAsync(headUpdate) @@ -90,7 +93,7 @@ func DeriveSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, er } func CreateSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, err error) { - t, err = createObjectTree(deps.Payload, deps.AclList, deps.CreateStorage) + t, err = createObjectTree(deps.Payload, deps.AclList, deps.SpaceStorage.CreateTreeStorage) if err != nil { return } @@ -108,6 +111,9 @@ func CreateSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, er syncHandler := newSyncTreeHandler(syncTree, syncClient) syncTree.SyncHandler = syncHandler t = syncTree + syncTree.Lock() + defer syncTree.Unlock() + syncTree.listener.Rebuild(syncTree) headUpdate := syncClient.CreateHeadUpdate(t, nil) err = syncClient.BroadcastAsync(headUpdate) @@ -193,6 +199,9 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t tr syncHandler := newSyncTreeHandler(syncTree, syncClient) syncTree.SyncHandler = syncHandler t = syncTree + syncTree.Lock() + defer syncTree.Unlock() + syncTree.listener.Rebuild(syncTree) headUpdate := syncTree.syncClient.CreateHeadUpdate(t, nil) // here we will have different behaviour based on who is sending this update diff --git a/common/pkg/acl/storage/provider.go b/common/pkg/acl/storage/provider.go index 51b53b4f..fd065f78 100644 --- a/common/pkg/acl/storage/provider.go +++ b/common/pkg/acl/storage/provider.go @@ -5,9 +5,11 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/treechangeproto" ) -var ErrUnknownTreeId = errors.New("tree does not exist") -var ErrTreeExists = errors.New("tree already exists") -var ErrUnkownChange = errors.New("change doesn't exist") +var ( + ErrUnknownTreeId = errors.New("tree does not exist") + ErrTreeExists = errors.New("tree already exists") + ErrUnknownChange = errors.New("change doesn't exist") +) type TreeStorageCreatePayload struct { RootRawChange *treechangeproto.RawTreeChangeWithId diff --git a/node/storage/treestorage.go b/node/storage/treestorage.go index 5ab47451..2abcedc1 100644 --- a/node/storage/treestorage.go +++ b/node/storage/treestorage.go @@ -122,7 +122,7 @@ func (t *treeStorage) GetRawChange(ctx context.Context, id string) (raw *treecha return } if res == nil { - err = storage.ErrUnkownChange + err = storage.ErrUnknownChange } raw = &treechangeproto.RawTreeChangeWithId{ From 55e04cd2cd3f4603220890e56aea54f7ba9ac3e5 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 24 Nov 2022 20:24:59 +0100 Subject: [PATCH 12/27] Change push space and some tree deletion logic --- client/clientspace/rpchandler.go | 3 +- client/clientspace/service.go | 5 -- client/storage/keys.go | 6 ++ client/storage/spacestorage.go | 27 +++++-- common/commonspace/commonstorage.go | 2 +- common/commonspace/service.go | 81 ++++++++++--------- .../commonspace/settingsdocument/deleter.go | 12 ++- .../settingsdocument/settingsdocument.go | 30 ++----- common/commonspace/synctree/synctree.go | 9 +++ node/nodespace/rpchandler.go | 3 +- node/nodespace/service.go | 5 -- node/storage/keys.go | 6 +- node/storage/spacestorage.go | 19 +++-- node/storage/treestorage.go | 2 +- 14 files changed, 118 insertions(+), 92 deletions(-) diff --git a/client/clientspace/rpchandler.go b/client/clientspace/rpchandler.go index bef22138..223efb84 100644 --- a/client/clientspace/rpchandler.go +++ b/client/clientspace/rpchandler.go @@ -45,7 +45,8 @@ func (r *rpcHandler) PushSpace(ctx context.Context, req *spacesyncproto.PushSpac SpaceSettingsPayload: req.Payload.SpaceSettingsPayload, SpaceSettingsId: req.Payload.SpaceSettingsPayloadId, } - err = r.s.AddSpace(ctx, description) + ctx = context.WithValue(ctx, commonspace.AddSpaceCtxKey, description) + _, err = r.s.GetSpace(ctx, description.SpaceHeader.GetId()) if err != nil { return } diff --git a/client/clientspace/service.go b/client/clientspace/service.go index d22ab135..fc43d7b8 100644 --- a/client/clientspace/service.go +++ b/client/clientspace/service.go @@ -23,7 +23,6 @@ func New() Service { type Service interface { GetSpace(ctx context.Context, id string) (commonspace.Space, error) - AddSpace(ctx context.Context, description commonspace.SpaceDescription) (err error) CreateSpace(ctx context.Context, payload commonspace.SpaceCreatePayload) (commonspace.Space, error) DeriveSpace(ctx context.Context, payload commonspace.SpaceDerivePayload) (commonspace.Space, error) app.ComponentRunnable @@ -91,10 +90,6 @@ func (s *service) GetSpace(ctx context.Context, id string) (container commonspac return v.(commonspace.Space), nil } -func (s *service) AddSpace(ctx context.Context, description commonspace.SpaceDescription) (err error) { - return s.commonSpace.AddSpace(ctx, description) -} - func (s *service) loadSpace(ctx context.Context, id string) (value ocache.Object, err error) { cc, err := s.commonSpace.NewSpace(ctx, id) if err != nil { diff --git a/client/storage/keys.go b/client/storage/keys.go index 45eb8cb0..cd19c07a 100644 --- a/client/storage/keys.go +++ b/client/storage/keys.go @@ -65,6 +65,7 @@ func (t treeKeys) RawChangePrefix() []byte { } type spaceKeys struct { + spaceId string headerKey []byte treePrefixKey []byte spaceSettingsIdKey []byte @@ -72,6 +73,7 @@ type spaceKeys struct { func newSpaceKeys(spaceId string) spaceKeys { return spaceKeys{ + spaceId: spaceId, headerKey: storage.JoinStringsToBytes("space", "header", spaceId), treePrefixKey: storage.JoinStringsToBytes("space", spaceId, "t", "rootId"), spaceSettingsIdKey: storage.JoinStringsToBytes("space", spaceId, "spaceSettingsId"), @@ -90,6 +92,10 @@ func (s spaceKeys) SpaceSettingsId() []byte { return s.spaceSettingsIdKey } +func (s spaceKeys) TreeDeletedKey(id string) []byte { + return storage.JoinStringsToBytes("space", s.spaceId, "deleted", id) +} + type storageServiceKeys struct { spacePrefix []byte } diff --git a/client/storage/spacestorage.go b/client/storage/spacestorage.go index 5c4f1866..7479808c 100644 --- a/client/storage/spacestorage.go +++ b/client/storage/spacestorage.go @@ -6,7 +6,6 @@ import ( storage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/treechangeproto" "github.com/dgraph-io/badger/v3" - "sync" ) type spaceStorage struct { @@ -16,7 +15,6 @@ type spaceStorage struct { keys spaceKeys aclStorage storage.ListStorage header *spacesyncproto.RawSpaceHeaderWithId - mx sync.Mutex } var spaceValidationFunc = spacestorage.ValidateSpaceStorageCreatePayload @@ -118,10 +116,6 @@ func (s *spaceStorage) TreeStorage(id string) (storage.TreeStorage, error) { } func (s *spaceStorage) CreateTreeStorage(payload storage.TreeStorageCreatePayload) (ts storage.TreeStorage, err error) { - // we have mutex here, so we prevent overwriting the heads of a tree on concurrent creation - s.mx.Lock() - defer s.mx.Unlock() - return createTreeStorage(s.objDb, s.spaceId, payload) } @@ -156,6 +150,27 @@ func (s *spaceStorage) StoredIds() (ids []string, err error) { return } +func (s *spaceStorage) SetTreeDeletedStatus(id, status string) (err error) { + return s.objDb.Update(func(txn *badger.Txn) error { + return txn.Set(s.keys.TreeDeletedKey(id), []byte(status)) + }) +} + +func (s *spaceStorage) TreeDeletedStatus(id string) (status string, err error) { + err = s.objDb.View(func(txn *badger.Txn) error { + res, err := getTxn(txn, s.keys.TreeDeletedKey(id)) + if err != nil { + return err + } + status = string(res) + return nil + }) + if err == badger.ErrKeyNotFound { + err = nil + } + return +} + func (s *spaceStorage) Close() (err error) { return nil } diff --git a/common/commonspace/commonstorage.go b/common/commonspace/commonstorage.go index ab6d5ca6..3299f54e 100644 --- a/common/commonspace/commonstorage.go +++ b/common/commonspace/commonstorage.go @@ -21,7 +21,7 @@ func (c *commonStorage) CreateTreeStorage(payload treestorage.TreeStorageCreateP return } if status == "" { - return c.CreateTreeStorage(payload) + return c.SpaceStorage.CreateTreeStorage(payload) } err = storage.ErrTreeStorageAlreadyDeleted return diff --git a/common/commonspace/service.go b/common/commonspace/service.go index dc4d4b83..a5d45a75 100644 --- a/common/commonspace/service.go +++ b/common/commonspace/service.go @@ -26,11 +26,14 @@ func New() Service { return &service{} } +type ctxKey int + +const AddSpaceCtxKey ctxKey = 0 + type Service interface { DeriveSpace(ctx context.Context, payload SpaceDerivePayload) (string, error) CreateSpace(ctx context.Context, payload SpaceCreatePayload) (string, error) NewSpace(ctx context.Context, id string) (sp Space, err error) - AddSpace(ctx context.Context, spaceDescription SpaceDescription) (err error) app.Component } @@ -83,36 +86,6 @@ func (s *service) DeriveSpace(ctx context.Context, payload SpaceDerivePayload) ( return store.Id(), nil } -func (s *service) AddSpace(ctx context.Context, spaceDescription SpaceDescription) (err error) { - _, err = s.storageProvider.SpaceStorage(spaceDescription.SpaceHeader.Id) - if err == nil { - err = spacesyncproto.ErrSpaceExists - return - } - if err != storage.ErrSpaceStorageMissing { - err = spacesyncproto.ErrUnexpected - return - } - - payload := storage.SpaceStorageCreatePayload{ - AclWithId: &aclrecordproto.RawACLRecordWithId{ - Payload: spaceDescription.AclPayload, - Id: spaceDescription.AclId, - }, - SpaceHeaderWithId: spaceDescription.SpaceHeader, - } - st, err := s.storageProvider.CreateSpaceStorage(payload) - if err != nil { - err = spacesyncproto.ErrUnexpected - if err == storage.ErrSpaceStorageExists { - err = spacesyncproto.ErrSpaceExists - } - return - } - err = st.Close() - return -} - func (s *service) NewSpace(ctx context.Context, id string) (Space, error) { st, err := s.storageProvider.SpaceStorage(id) if err != nil { @@ -120,10 +93,17 @@ func (s *service) NewSpace(ctx context.Context, id string) (Space, error) { return nil, err } - st, err = s.getSpaceStorageFromRemote(ctx, id) - if err != nil { - err = storage.ErrSpaceStorageMissing - return nil, err + if description, ok := ctx.Value(AddSpaceCtxKey).(SpaceDescription); ok { + st, err = s.addSpaceStorage(ctx, description) + if err != nil { + return nil, err + } + } else { + st, err = s.getSpaceStorageFromRemote(ctx, id) + if err != nil { + err = storage.ErrSpaceStorageMissing + return nil, err + } } } @@ -143,10 +123,39 @@ func (s *service) NewSpace(ctx context.Context, id string) (Space, error) { return sp, nil } +func (s *service) addSpaceStorage(ctx context.Context, spaceDescription SpaceDescription) (st storage.SpaceStorage, err error) { + _, err = s.storageProvider.SpaceStorage(spaceDescription.SpaceHeader.Id) + if err == nil { + err = spacesyncproto.ErrSpaceExists + return + } + if err != storage.ErrSpaceStorageMissing { + err = spacesyncproto.ErrUnexpected + return + } + + payload := storage.SpaceStorageCreatePayload{ + AclWithId: &aclrecordproto.RawACLRecordWithId{ + Payload: spaceDescription.AclPayload, + Id: spaceDescription.AclId, + }, + SpaceHeaderWithId: spaceDescription.SpaceHeader, + } + st, err = s.storageProvider.CreateSpaceStorage(payload) + if err != nil { + err = spacesyncproto.ErrUnexpected + if err == storage.ErrSpaceStorageExists { + err = spacesyncproto.ErrSpaceExists + } + return + } + return +} + func (s *service) getSpaceStorageFromRemote(ctx context.Context, id string) (st storage.SpaceStorage, err error) { var p peer.Peer lastConfiguration := s.configurationService.GetLast() - // for nodes we always get remote space only if we have id in the context + // we can't connect to client if it is a node if lastConfiguration.IsResponsible(id) { err = spacesyncproto.ErrSpaceMissing return diff --git a/common/commonspace/settingsdocument/deleter.go b/common/commonspace/settingsdocument/deleter.go index 48b6c871..c259e0d3 100644 --- a/common/commonspace/settingsdocument/deleter.go +++ b/common/commonspace/settingsdocument/deleter.go @@ -21,14 +21,12 @@ func newDeleter(st storage.SpaceStorage, state *deletionstate.DeletionState, get func (d *deleter) delete() { allQueued := d.state.GetQueued() for _, id := range allQueued { - if _, err := d.st.TreeStorage(id); err == nil { - err := d.getter.DeleteTree(context.Background(), d.st.Id(), id) - if err != nil { - log.With(zap.String("id", id), zap.Error(err)).Error("failed to delete object") - continue - } + err := d.getter.DeleteTree(context.Background(), d.st.Id(), id) + if err != nil && err != storage.ErrTreeStorageAlreadyDeleted { + log.With(zap.String("id", id), zap.Error(err)).Error("failed to delete object") + continue } - err := d.state.Delete(id) + err = d.state.Delete(id) if err != nil { log.With(zap.String("id", id), zap.Error(err)).Error("failed to mark object as deleted") } diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go index 3060cb20..0dea6b8e 100644 --- a/common/commonspace/settingsdocument/settingsdocument.go +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -18,7 +18,6 @@ var log = logger.NewNamed("commonspace.settingsdocument") type SettingsDocument interface { tree.ObjectTree Init(ctx context.Context) (err error) - Refresh() DeleteObject(id string) (err error) } @@ -75,8 +74,9 @@ func NewSettingsDocument(deps Deps, spaceId string) (doc SettingsDocument, err e return } -func (s *settingsDocument) Update(tr tree.ObjectTree) { - ids, lastId, err := s.prov.ProvideIds(tr, s.lastChangeId) +func (s *settingsDocument) updateIds(lastChangeId string) { + s.lastChangeId = lastChangeId + ids, lastId, err := s.prov.ProvideIds(s, s.lastChangeId) if err != nil { log.With(zap.Strings("ids", ids), zap.Error(err)).Error("failed to update state") return @@ -87,16 +87,12 @@ func (s *settingsDocument) Update(tr tree.ObjectTree) { } } +func (s *settingsDocument) Update(tr tree.ObjectTree) { + s.updateIds(s.lastChangeId) +} + func (s *settingsDocument) Rebuild(tr tree.ObjectTree) { - ids, lastId, err := s.prov.ProvideIds(tr, "") - if err != nil { - log.With(zap.Strings("ids", ids), zap.Error(err)).Error("failed to rebuild state") - return - } - s.lastChangeId = lastId - if err = s.deletionState.Add(ids); err != nil { - log.With(zap.Strings("ids", ids), zap.Error(err)).Error("failed to queue ids to delete") - } + s.updateIds("") } func (s *settingsDocument) Init(ctx context.Context) (err error) { @@ -108,16 +104,6 @@ func (s *settingsDocument) Init(ctx context.Context) (err error) { return } -func (s *settingsDocument) Refresh() { - s.Lock() - defer s.Unlock() - if s.lastChangeId == "" { - s.Rebuild(s) - } else { - s.Update(s) - } -} - func (s *settingsDocument) Close() error { s.loop.Close() return s.ObjectTree.Close() diff --git a/common/commonspace/synctree/synctree.go b/common/commonspace/synctree/synctree.go index 3647ecc7..6a56593d 100644 --- a/common/commonspace/synctree/synctree.go +++ b/common/commonspace/synctree/synctree.go @@ -150,6 +150,15 @@ func BuildSyncTreeOrGetRemote(ctx context.Context, id string, deps BuildDeps) (t return } + status, err := deps.SpaceStorage.TreeDeletedStatus(id) + if err != nil { + return + } + if status != "" { + err = spacestorage.ErrTreeStorageAlreadyDeleted + return + } + resp, err := getTreeRemote() if err != nil { return diff --git a/node/nodespace/rpchandler.go b/node/nodespace/rpchandler.go index 901f0a3e..f7b3f6a1 100644 --- a/node/nodespace/rpchandler.go +++ b/node/nodespace/rpchandler.go @@ -45,7 +45,8 @@ func (r *rpcHandler) PushSpace(ctx context.Context, req *spacesyncproto.PushSpac SpaceSettingsPayload: req.Payload.SpaceSettingsPayload, SpaceSettingsId: req.Payload.SpaceSettingsPayloadId, } - err = r.s.AddSpace(ctx, description) + ctx = context.WithValue(ctx, commonspace.AddSpaceCtxKey, description) + _, err = r.s.GetSpace(ctx, description.SpaceHeader.GetId()) if err != nil { return } diff --git a/node/nodespace/service.go b/node/nodespace/service.go index a1ad1f0a..82faa71f 100644 --- a/node/nodespace/service.go +++ b/node/nodespace/service.go @@ -22,7 +22,6 @@ func New() Service { } type Service interface { - AddSpace(ctx context.Context, description commonspace.SpaceDescription) (err error) GetSpace(ctx context.Context, id string) (commonspace.Space, error) app.ComponentRunnable } @@ -63,10 +62,6 @@ func (s *service) GetSpace(ctx context.Context, id string) (commonspace.Space, e return v.(commonspace.Space), nil } -func (s *service) AddSpace(ctx context.Context, description commonspace.SpaceDescription) (err error) { - return s.commonSpace.AddSpace(ctx, description) -} - func (s *service) loadSpace(ctx context.Context, id string) (value ocache.Object, err error) { cc, err := s.commonSpace.NewSpace(ctx, id) if err != nil { diff --git a/node/storage/keys.go b/node/storage/keys.go index ef728d74..650d392c 100644 --- a/node/storage/keys.go +++ b/node/storage/keys.go @@ -46,7 +46,7 @@ func (t treeKeys) RawChangeKey(id string) []byte { return storage.JoinStringsToBytes("t", t.id, id) } -func (t treeKeys) isTreeRecordKey(key string) bool { +func (t treeKeys) isTreeRelatedKey(key string) bool { return strings.HasPrefix(key, t.prefix) } @@ -75,6 +75,10 @@ func (s spaceKeys) SpaceSettingsIdKey() []byte { return spaceSettingsIdKey } +func (s spaceKeys) TreeDeletedKey(id string) []byte { + return storage.JoinStringsToBytes("del", id) +} + func isTreeHeadsKey(key string) bool { return strings.HasPrefix(key, "t/") && strings.HasSuffix(key, "/heads") } diff --git a/node/storage/spacestorage.go b/node/storage/spacestorage.go index 232dc838..b89c346b 100644 --- a/node/storage/spacestorage.go +++ b/node/storage/spacestorage.go @@ -9,7 +9,6 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/treechangeproto" "go.uber.org/zap" "path" - "sync" "time" ) @@ -26,7 +25,6 @@ type spaceStorage struct { keys spaceKeys aclStorage storage.ListStorage header *spacesyncproto.RawSpaceHeaderWithId - mx sync.Mutex } func newSpaceStorage(rootPath string, spaceId string) (store spacestorage.SpaceStorage, err error) { @@ -174,10 +172,6 @@ func (s *spaceStorage) TreeStorage(id string) (storage.TreeStorage, error) { } func (s *spaceStorage) CreateTreeStorage(payload storage.TreeStorageCreatePayload) (ts storage.TreeStorage, err error) { - // we have mutex here, so we prevent overwriting the heads of a tree on concurrent creation - s.mx.Lock() - defer s.mx.Unlock() - return createTreeStorage(s.objDb, payload) } @@ -189,6 +183,19 @@ func (s *spaceStorage) SpaceHeader() (header *spacesyncproto.RawSpaceHeaderWithI return s.header, nil } +func (s *spaceStorage) SetTreeDeletedStatus(id, state string) (err error) { + return s.objDb.Put(s.keys.TreeDeletedKey(id), []byte(state)) +} + +func (s *spaceStorage) TreeDeletedStatus(id string) (status string, err error) { + res, err := s.objDb.Get(s.keys.TreeDeletedKey(id)) + if err != nil { + return + } + status = string(res) + return +} + func (s *spaceStorage) StoredIds() (ids []string, err error) { index := s.objDb.Items() diff --git a/node/storage/treestorage.go b/node/storage/treestorage.go index 2abcedc1..d1ab6fd6 100644 --- a/node/storage/treestorage.go +++ b/node/storage/treestorage.go @@ -156,7 +156,7 @@ func (t *treeStorage) storedKeys() (keys [][]byte, err error) { key, _, err := index.Next() for err == nil { strKey := string(key) - if t.keys.isTreeRecordKey(strKey) { + if t.keys.isTreeRelatedKey(strKey) { keys = append(keys, key) } key, _, err = index.Next() From f98d629a787d0f4b8b13bf4272f27484f4264aa1 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 24 Nov 2022 21:03:43 +0100 Subject: [PATCH 13/27] Add deletion test --- client/storage/treestorage_test.go | 65 ++++++++++++++++++++++++++++++ node/storage/treestorage_test.go | 48 ++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/client/storage/treestorage_test.go b/client/storage/treestorage_test.go index 84b422d5..43ff6b27 100644 --- a/client/storage/treestorage_test.go +++ b/client/storage/treestorage_test.go @@ -61,6 +61,42 @@ func (fx *fixture) stop(t *testing.T) { require.NoError(t, fx.db.Close()) } +func (fx *fixture) testNoKeysExist(t *testing.T, spaceId, treeId string) { + treeKeys := newTreeKeys(spaceId, treeId) + + var keys [][]byte + err := fx.db.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.PrefetchValues = false + opts.Prefix = treeKeys.RawChangePrefix() + + it := txn.NewIterator(opts) + defer it.Close() + + for it.Rewind(); it.Valid(); it.Next() { + item := it.Item() + key := item.Key() + keyCopy := make([]byte, 0, len(key)) + keyCopy = item.KeyCopy(key) + keys = append(keys, keyCopy) + } + return nil + }) + + require.NoError(t, err) + require.Equal(t, 0, len(keys)) + + err = fx.db.View(func(txn *badger.Txn) error { + _, err = getTxn(txn, treeKeys.RootIdKey()) + require.Equal(t, err, badger.ErrKeyNotFound) + + _, err = getTxn(txn, treeKeys.HeadsKey()) + require.Equal(t, err, badger.ErrKeyNotFound) + + return nil + }) +} + func TestTreeStorage_Create(t *testing.T) { fx := newFixture(t) fx.open(t) @@ -121,3 +157,32 @@ func TestTreeStorage_Methods(t *testing.T) { require.False(t, has) }) } + +func TestTreeStorage_Delete(t *testing.T) { + fx := newFixture(t) + fx.open(t) + payload := treeTestPayload() + spaceId := "spaceId" + _, err := createTreeStorage(fx.db, spaceId, payload) + require.NoError(t, err) + fx.stop(t) + + fx.open(t) + defer fx.stop(t) + store, err := newTreeStorage(fx.db, spaceId, payload.RootRawChange.Id) + require.NoError(t, err) + testTreePayload(t, store, payload) + + t.Run("add raw change, get change and has change", func(t *testing.T) { + newChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("ab"), Id: "newId"} + require.NoError(t, store.AddRawChange(newChange)) + + err = store.Delete() + require.NoError(t, err) + + _, err = newTreeStorage(fx.db, spaceId, payload.RootRawChange.Id) + require.Equal(t, err, storage.ErrUnknownTreeId) + + fx.testNoKeysExist(t, spaceId, payload.RootRawChange.Id) + }) +} diff --git a/node/storage/treestorage_test.go b/node/storage/treestorage_test.go index 00cce688..85ba1e3d 100644 --- a/node/storage/treestorage_test.go +++ b/node/storage/treestorage_test.go @@ -42,6 +42,26 @@ func (fx *fixture) stop(t *testing.T) { require.NoError(t, fx.db.Close()) } +func (fx *fixture) testNoKeysExist(t *testing.T, treeId string) { + index := fx.db.Items() + treeKeys := newTreeKeys(treeId) + var keys [][]byte + key, _, err := index.Next() + for err == nil { + strKey := string(key) + if treeKeys.isTreeRelatedKey(strKey) { + keys = append(keys, key) + } + key, _, err = index.Next() + } + require.Equal(t, pogreb.ErrIterationDone, err) + require.Equal(t, 0, len(keys)) + + res, err := fx.db.Has(treeKeys.HeadsKey()) + require.NoError(t, err) + require.False(t, res) +} + func testTreePayload(t *testing.T, store storage.TreeStorage, payload storage.TreeStorageCreatePayload) { require.Equal(t, payload.RootRawChange.Id, store.Id()) @@ -119,3 +139,31 @@ func TestTreeStorage_Methods(t *testing.T) { require.False(t, has) }) } + +func TestTreeStorage_Delete(t *testing.T) { + fx := newFixture(t) + fx.open(t) + payload := treeTestPayload() + _, err := createTreeStorage(fx.db, payload) + require.NoError(t, err) + fx.stop(t) + + fx.open(t) + defer fx.stop(t) + store, err := newTreeStorage(fx.db, payload.RootRawChange.Id) + require.NoError(t, err) + testTreePayload(t, store, payload) + + t.Run("add raw change, get change and has change", func(t *testing.T) { + newChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("ab"), Id: "newId"} + require.NoError(t, store.AddRawChange(newChange)) + + err = store.Delete() + require.NoError(t, err) + + _, err = newTreeStorage(fx.db, payload.RootRawChange.Id) + require.Equal(t, err, storage.ErrUnknownTreeId) + + fx.testNoKeysExist(t, payload.RootRawChange.Id) + }) +} From 9155a3cbf231127722a6cc6d81ac415bbe91b183 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 24 Nov 2022 22:20:01 +0100 Subject: [PATCH 14/27] Fix tests --- client/storage/spacestorage_test.go | 24 +++++++++++++++++++++--- node/storage/spacestorage_test.go | 24 +++++++++++++++++++++--- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/client/storage/spacestorage_test.go b/client/storage/spacestorage_test.go index 84665039..10b29f6d 100644 --- a/client/storage/spacestorage_test.go +++ b/client/storage/spacestorage_test.go @@ -4,7 +4,9 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" spacestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/aclrecordproto" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/treechangeproto" "github.com/stretchr/testify/require" + "sort" "strconv" "testing" ) @@ -18,9 +20,14 @@ func spaceTestPayload() spacestorage.SpaceStorageCreatePayload { Payload: []byte("aclRoot"), Id: "aclRootId", } + settings := &treechangeproto.RawTreeChangeWithId{ + RawChange: []byte("settings"), + Id: "settingsId", + } return spacestorage.SpaceStorageCreatePayload{ - AclWithId: aclRoot, - SpaceHeaderWithId: header, + AclWithId: aclRoot, + SpaceHeaderWithId: header, + SpaceSettingsWithId: settings, } } @@ -68,7 +75,7 @@ func TestSpaceStorage_NewAndCreateTree(t *testing.T) { require.NoError(t, err) testSpace(t, store, payload) - t.Run("create tree and get tree", func(t *testing.T) { + t.Run("create tree, get tree and mark deleted", func(t *testing.T) { payload := treeTestPayload() treeStore, err := store.CreateTreeStorage(payload) require.NoError(t, err) @@ -77,6 +84,14 @@ func TestSpaceStorage_NewAndCreateTree(t *testing.T) { otherStore, err := store.TreeStorage(payload.RootRawChange.Id) require.NoError(t, err) testTreePayload(t, otherStore, payload) + + initialStatus := "deleted" + err = store.SetTreeDeletedStatus(otherStore.Id(), initialStatus) + require.NoError(t, err) + + status, err := store.TreeDeletedStatus(otherStore.Id()) + require.NoError(t, err) + require.Equal(t, initialStatus, status) }) } @@ -101,8 +116,11 @@ func TestSpaceStorage_StoredIds(t *testing.T) { _, err := store.CreateTreeStorage(treePayload) require.NoError(t, err) } + ids = append(ids, payload.SpaceSettingsWithId.Id) + sort.Strings(ids) storedIds, err := store.StoredIds() require.NoError(t, err) + sort.Strings(storedIds) require.Equal(t, ids, storedIds) } diff --git a/node/storage/spacestorage_test.go b/node/storage/spacestorage_test.go index 4009bd1f..8b4460e0 100644 --- a/node/storage/spacestorage_test.go +++ b/node/storage/spacestorage_test.go @@ -4,8 +4,10 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" spacestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/aclrecordproto" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/treechangeproto" "github.com/stretchr/testify/require" "os" + "sort" "strconv" "testing" ) @@ -19,9 +21,14 @@ func spaceTestPayload() spacestorage.SpaceStorageCreatePayload { Payload: []byte("aclRoot"), Id: "aclRootId", } + settings := &treechangeproto.RawTreeChangeWithId{ + RawChange: []byte("settings"), + Id: "settingsId", + } return spacestorage.SpaceStorageCreatePayload{ - AclWithId: aclRoot, - SpaceHeaderWithId: header, + AclWithId: aclRoot, + SpaceHeaderWithId: header, + SpaceSettingsWithId: settings, } } @@ -68,7 +75,7 @@ func TestSpaceStorage_NewAndCreateTree(t *testing.T) { }() testSpace(t, store, payload) - t.Run("create tree and get tree", func(t *testing.T) { + t.Run("create tree, get tree and mark deleted", func(t *testing.T) { payload := treeTestPayload() treeStore, err := store.CreateTreeStorage(payload) require.NoError(t, err) @@ -77,6 +84,14 @@ func TestSpaceStorage_NewAndCreateTree(t *testing.T) { otherStore, err := store.TreeStorage(payload.RootRawChange.Id) require.NoError(t, err) testTreePayload(t, otherStore, payload) + + initialStatus := "deleted" + err = store.SetTreeDeletedStatus(otherStore.Id(), initialStatus) + require.NoError(t, err) + + status, err := store.TreeDeletedStatus(otherStore.Id()) + require.NoError(t, err) + require.Equal(t, initialStatus, status) }) } @@ -100,8 +115,11 @@ func TestSpaceStorage_StoredIds(t *testing.T) { _, err := store.CreateTreeStorage(treePayload) require.NoError(t, err) } + ids = append(ids, payload.SpaceSettingsWithId.Id) + sort.Strings(ids) storedIds, err := store.StoredIds() + sort.Strings(storedIds) require.NoError(t, err) require.Equal(t, ids, storedIds) } From 71717a5dbb4129fafdd0e77c9c312f94968c7c2a Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 24 Nov 2022 22:41:34 +0100 Subject: [PATCH 15/27] Fix settings document --- common/commonspace/service.go | 10 ---------- .../commonspace/settingsdocument/settingsdocument.go | 12 +++++++----- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/common/commonspace/service.go b/common/commonspace/service.go index a5d45a75..6b101b31 100644 --- a/common/commonspace/service.go +++ b/common/commonspace/service.go @@ -124,16 +124,6 @@ func (s *service) NewSpace(ctx context.Context, id string) (Space, error) { } func (s *service) addSpaceStorage(ctx context.Context, spaceDescription SpaceDescription) (st storage.SpaceStorage, err error) { - _, err = s.storageProvider.SpaceStorage(spaceDescription.SpaceHeader.Id) - if err == nil { - err = spacesyncproto.ErrSpaceExists - return - } - if err != storage.ErrSpaceStorageMissing { - err = spacesyncproto.ErrUnexpected - return - } - payload := storage.SpaceStorageCreatePayload{ AclWithId: &aclrecordproto.RawACLRecordWithId{ Payload: spaceDescription.AclPayload, diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go index 0dea6b8e..f2d055a4 100644 --- a/common/commonspace/settingsdocument/settingsdocument.go +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -41,7 +41,7 @@ type settingsDocument struct { store spacestorage.SpaceStorage prov deletedIdsProvider buildFunc BuildTreeFunc - loop deleteLoop + loop *deleteLoop deletionState *deletionstate.DeletionState lastChangeId string @@ -57,6 +57,7 @@ func NewSettingsDocument(deps Deps, spaceId string) (doc SettingsDocument, err e }) s := &settingsDocument{ + loop: loop, spaceId: spaceId, account: deps.Account, deletionState: deps.DeletionState, @@ -74,9 +75,9 @@ func NewSettingsDocument(deps Deps, spaceId string) (doc SettingsDocument, err e return } -func (s *settingsDocument) updateIds(lastChangeId string) { +func (s *settingsDocument) updateIds(tr tree.ObjectTree, lastChangeId string) { s.lastChangeId = lastChangeId - ids, lastId, err := s.prov.ProvideIds(s, s.lastChangeId) + ids, lastId, err := s.prov.ProvideIds(tr, s.lastChangeId) if err != nil { log.With(zap.Strings("ids", ids), zap.Error(err)).Error("failed to update state") return @@ -88,11 +89,12 @@ func (s *settingsDocument) updateIds(lastChangeId string) { } func (s *settingsDocument) Update(tr tree.ObjectTree) { - s.updateIds(s.lastChangeId) + s.updateIds(tr, s.lastChangeId) } func (s *settingsDocument) Rebuild(tr tree.ObjectTree) { - s.updateIds("") + // at initial build "s" may not contain the object tree, so it is safer to provide it from the function parameter + s.updateIds(tr, "") } func (s *settingsDocument) Init(ctx context.Context) (err error) { From ff81bf1fac3dd5bfe23d31581455bc7c7c9c9712 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 25 Nov 2022 09:46:01 +0100 Subject: [PATCH 16/27] Add sync listener nil check --- common/commonspace/synctree/synctree.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/common/commonspace/synctree/synctree.go b/common/commonspace/synctree/synctree.go index 6a56593d..e471a961 100644 --- a/common/commonspace/synctree/synctree.go +++ b/common/commonspace/synctree/synctree.go @@ -85,7 +85,9 @@ func DeriveSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, er t = syncTree syncTree.Lock() defer syncTree.Unlock() - syncTree.listener.Rebuild(syncTree) + if syncTree.listener != nil { + syncTree.listener.Rebuild(syncTree) + } headUpdate := syncClient.CreateHeadUpdate(t, nil) err = syncClient.BroadcastAsync(headUpdate) @@ -113,7 +115,10 @@ func CreateSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, er t = syncTree syncTree.Lock() defer syncTree.Unlock() - syncTree.listener.Rebuild(syncTree) + // TODO: refactor here because the code is duplicated, when we create a tree we should only create a storage and then build a tree + if syncTree.listener != nil { + syncTree.listener.Rebuild(syncTree) + } headUpdate := syncClient.CreateHeadUpdate(t, nil) err = syncClient.BroadcastAsync(headUpdate) @@ -210,7 +215,9 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t tr t = syncTree syncTree.Lock() defer syncTree.Unlock() - syncTree.listener.Rebuild(syncTree) + if syncTree.listener != nil { + syncTree.listener.Rebuild(syncTree) + } headUpdate := syncTree.syncClient.CreateHeadUpdate(t, nil) // here we will have different behaviour based on who is sending this update From 558cecea76bc8f733ca8b05e85c46fb95cf51c6d Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 25 Nov 2022 11:40:36 +0100 Subject: [PATCH 17/27] Fix more bugs --- client/clientspace/clientcache/treecache.go | 8 +++++++- client/document/textdocument/textdocument.go | 5 +++++ common/commonspace/service.go | 7 +++++-- common/commonspace/settingsdocument/deleter.go | 1 + common/commonspace/settingsdocument/settingsdocument.go | 6 ++++-- 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/client/clientspace/clientcache/treecache.go b/client/clientspace/clientcache/treecache.go index abdd931a..e33a1235 100644 --- a/client/clientspace/clientcache/treecache.go +++ b/client/clientspace/clientcache/treecache.go @@ -99,7 +99,13 @@ func (c *treeCache) GetDocument(ctx context.Context, spaceId, id string) (doc te } func (c *treeCache) GetTree(ctx context.Context, spaceId, id string) (tr tree.ObjectTree, err error) { - return c.GetDocument(ctx, spaceId, id) + doc, err := c.GetDocument(ctx, spaceId, id) + if err != nil { + return + } + // we have to do this trick, otherwise the compiler won't understand that TextDocument conforms to SyncHandler interface + tr = doc.InnerTree() + return } func (c *treeCache) DeleteTree(ctx context.Context, spaceId, treeId string) (err error) { diff --git a/client/document/textdocument/textdocument.go b/client/document/textdocument/textdocument.go index 4334afb9..4c4abb44 100644 --- a/client/document/textdocument/textdocument.go +++ b/client/document/textdocument/textdocument.go @@ -12,6 +12,7 @@ import ( type TextDocument interface { tree.ObjectTree + InnerTree() tree.ObjectTree AddText(text string) error Text() (string, error) TreeDump() string @@ -55,6 +56,10 @@ func NewTextDocument(ctx context.Context, space commonspace.Space, id string, li }, nil } +func (t *textDocument) InnerTree() tree.ObjectTree { + return t.ObjectTree +} + func (t *textDocument) AddText(text string) (err error) { content := &testchanges.TextContent_TextAppend{ TextAppend: &testchanges.TextAppend{Text: text}, diff --git a/common/commonspace/service.go b/common/commonspace/service.go index 6b101b31..ba3dabd2 100644 --- a/common/commonspace/service.go +++ b/common/commonspace/service.go @@ -89,7 +89,7 @@ func (s *service) DeriveSpace(ctx context.Context, payload SpaceDerivePayload) ( func (s *service) NewSpace(ctx context.Context, id string) (Space, error) { st, err := s.storageProvider.SpaceStorage(id) if err != nil { - if err != spacesyncproto.ErrSpaceMissing { + if err != storage.ErrSpaceStorageMissing { return nil, err } @@ -101,7 +101,6 @@ func (s *service) NewSpace(ctx context.Context, id string) (Space, error) { } else { st, err = s.getSpaceStorageFromRemote(ctx, id) if err != nil { - err = storage.ErrSpaceStorageMissing return nil, err } } @@ -130,6 +129,10 @@ func (s *service) addSpaceStorage(ctx context.Context, spaceDescription SpaceDes Id: spaceDescription.AclId, }, SpaceHeaderWithId: spaceDescription.SpaceHeader, + SpaceSettingsWithId: &treechangeproto.RawTreeChangeWithId{ + RawChange: spaceDescription.SpaceSettingsPayload, + Id: spaceDescription.SpaceSettingsId, + }, } st, err = s.storageProvider.CreateSpaceStorage(payload) if err != nil { diff --git a/common/commonspace/settingsdocument/deleter.go b/common/commonspace/settingsdocument/deleter.go index c259e0d3..51a8c32b 100644 --- a/common/commonspace/settingsdocument/deleter.go +++ b/common/commonspace/settingsdocument/deleter.go @@ -30,5 +30,6 @@ func (d *deleter) delete() { if err != nil { log.With(zap.String("id", id), zap.Error(err)).Error("failed to mark object as deleted") } + log.With(zap.String("id", id), zap.Error(err)).Debug("object successfully deleted") } } diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go index f2d055a4..8c603a5e 100644 --- a/common/commonspace/settingsdocument/settingsdocument.go +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -7,6 +7,7 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" spacestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree/updatelistener" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" @@ -34,7 +35,7 @@ type Deps struct { } type settingsDocument struct { - tree.ObjectTree + *synctree.SyncTree account account.Service spaceId string treeGetter treegetter.TreeGetter @@ -98,10 +99,11 @@ func (s *settingsDocument) Rebuild(tr tree.ObjectTree) { } func (s *settingsDocument) Init(ctx context.Context) (err error) { - s.ObjectTree, err = s.buildFunc(ctx, s.store.SpaceSettingsId(), s) + syncTree, err := s.buildFunc(ctx, s.store.SpaceSettingsId(), s) if err != nil { return } + s.SyncTree = syncTree.(*synctree.SyncTree) s.loop.Run() return } From 620731b464633adc5da7e3b785eaa985da23350e Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 25 Nov 2022 14:37:00 +0100 Subject: [PATCH 18/27] Re-generate mocks and fix sync tree tests --- common/commonspace/diffservice/diffservice.go | 2 +- .../mock_diffservice/mock_diffservice.go | 74 ++++++++----------- .../storage/mock_storage/mock_storage.go | 43 +++++++++++ common/commonspace/synctree/synctree_test.go | 27 +++++-- .../mock_treegetter/mock_treegetter.go | 14 ++++ .../acl/storage/mock_storage/mock_storage.go | 14 ++++ .../tree/mock_objecttree/mock_objecttree.go | 14 ++++ .../mock_periodicsync/mock_periodicsync.go | 58 +++++++++++++++ common/util/periodicsync/periodicsync.go | 1 + 9 files changed, 199 insertions(+), 48 deletions(-) create mode 100644 common/util/periodicsync/mock_periodicsync/mock_periodicsync.go diff --git a/common/commonspace/diffservice/diffservice.go b/common/commonspace/diffservice/diffservice.go index 1c5df8b3..113e4e5c 100644 --- a/common/commonspace/diffservice/diffservice.go +++ b/common/commonspace/diffservice/diffservice.go @@ -1,4 +1,4 @@ -//go:generate mockgen -destination mock_diffservice/mock_diffservice.go github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/diffservice DiffSyncer,PeriodicSync +//go:generate mockgen -destination mock_diffservice/mock_diffservice.go github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/diffservice DiffSyncer package diffservice import ( diff --git a/common/commonspace/diffservice/mock_diffservice/mock_diffservice.go b/common/commonspace/diffservice/mock_diffservice/mock_diffservice.go index 966a7cc5..f54342d9 100644 --- a/common/commonspace/diffservice/mock_diffservice/mock_diffservice.go +++ b/common/commonspace/diffservice/mock_diffservice/mock_diffservice.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/diffservice (interfaces: DiffSyncer,PeriodicSync) +// Source: github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/diffservice (interfaces: DiffSyncer) // Package mock_diffservice is a generated GoMock package. package mock_diffservice @@ -8,6 +8,7 @@ import ( context "context" reflect "reflect" + deletionstate "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" gomock "github.com/golang/mock/gomock" ) @@ -34,6 +35,30 @@ func (m *MockDiffSyncer) EXPECT() *MockDiffSyncerMockRecorder { return m.recorder } +// Init mocks base method. +func (m *MockDiffSyncer) Init(arg0 *deletionstate.DeletionState) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Init", arg0) +} + +// Init indicates an expected call of Init. +func (mr *MockDiffSyncerMockRecorder) Init(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockDiffSyncer)(nil).Init), arg0) +} + +// RemoveObjects mocks base method. +func (m *MockDiffSyncer) RemoveObjects(arg0 []string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RemoveObjects", arg0) +} + +// RemoveObjects indicates an expected call of RemoveObjects. +func (mr *MockDiffSyncerMockRecorder) RemoveObjects(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveObjects", reflect.TypeOf((*MockDiffSyncer)(nil).RemoveObjects), arg0) +} + // Sync mocks base method. func (m *MockDiffSyncer) Sync(arg0 context.Context) error { m.ctrl.T.Helper() @@ -48,49 +73,14 @@ func (mr *MockDiffSyncerMockRecorder) Sync(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Sync", reflect.TypeOf((*MockDiffSyncer)(nil).Sync), arg0) } -// MockPeriodicSync is a mock of PeriodicSync interface. -type MockPeriodicSync struct { - ctrl *gomock.Controller - recorder *MockPeriodicSyncMockRecorder -} - -// MockPeriodicSyncMockRecorder is the mock recorder for MockPeriodicSync. -type MockPeriodicSyncMockRecorder struct { - mock *MockPeriodicSync -} - -// NewMockPeriodicSync creates a new mock instance. -func NewMockPeriodicSync(ctrl *gomock.Controller) *MockPeriodicSync { - mock := &MockPeriodicSync{ctrl: ctrl} - mock.recorder = &MockPeriodicSyncMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockPeriodicSync) EXPECT() *MockPeriodicSyncMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MockPeriodicSync) Close() { +// UpdateHeads mocks base method. +func (m *MockDiffSyncer) UpdateHeads(arg0 string, arg1 []string) { m.ctrl.T.Helper() - m.ctrl.Call(m, "Close") + m.ctrl.Call(m, "UpdateHeads", arg0, arg1) } -// Close indicates an expected call of Close. -func (mr *MockPeriodicSyncMockRecorder) Close() *gomock.Call { +// UpdateHeads indicates an expected call of UpdateHeads. +func (mr *MockDiffSyncerMockRecorder) UpdateHeads(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockPeriodicSync)(nil).Close)) -} - -// Run mocks base method. -func (m *MockPeriodicSync) Run() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Run") -} - -// Run indicates an expected call of Run. -func (mr *MockPeriodicSyncMockRecorder) Run() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockPeriodicSync)(nil).Run)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateHeads", reflect.TypeOf((*MockDiffSyncer)(nil).UpdateHeads), arg0, arg1) } diff --git a/common/commonspace/storage/mock_storage/mock_storage.go b/common/commonspace/storage/mock_storage/mock_storage.go index 419c61fa..c8ab901e 100644 --- a/common/commonspace/storage/mock_storage/mock_storage.go +++ b/common/commonspace/storage/mock_storage/mock_storage.go @@ -176,6 +176,20 @@ func (mr *MockSpaceStorageMockRecorder) Id() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Id", reflect.TypeOf((*MockSpaceStorage)(nil).Id)) } +// SetTreeDeletedStatus mocks base method. +func (m *MockSpaceStorage) SetTreeDeletedStatus(arg0, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetTreeDeletedStatus", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetTreeDeletedStatus indicates an expected call of SetTreeDeletedStatus. +func (mr *MockSpaceStorageMockRecorder) SetTreeDeletedStatus(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetTreeDeletedStatus", reflect.TypeOf((*MockSpaceStorage)(nil).SetTreeDeletedStatus), arg0, arg1) +} + // SpaceHeader mocks base method. func (m *MockSpaceStorage) SpaceHeader() (*spacesyncproto.RawSpaceHeaderWithId, error) { m.ctrl.T.Helper() @@ -191,6 +205,20 @@ func (mr *MockSpaceStorageMockRecorder) SpaceHeader() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SpaceHeader", reflect.TypeOf((*MockSpaceStorage)(nil).SpaceHeader)) } +// SpaceSettingsId mocks base method. +func (m *MockSpaceStorage) SpaceSettingsId() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SpaceSettingsId") + ret0, _ := ret[0].(string) + return ret0 +} + +// SpaceSettingsId indicates an expected call of SpaceSettingsId. +func (mr *MockSpaceStorageMockRecorder) SpaceSettingsId() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SpaceSettingsId", reflect.TypeOf((*MockSpaceStorage)(nil).SpaceSettingsId)) +} + // StoredIds mocks base method. func (m *MockSpaceStorage) StoredIds() ([]string, error) { m.ctrl.T.Helper() @@ -206,6 +234,21 @@ func (mr *MockSpaceStorageMockRecorder) StoredIds() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StoredIds", reflect.TypeOf((*MockSpaceStorage)(nil).StoredIds)) } +// TreeDeletedStatus mocks base method. +func (m *MockSpaceStorage) TreeDeletedStatus(arg0 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TreeDeletedStatus", arg0) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TreeDeletedStatus indicates an expected call of TreeDeletedStatus. +func (mr *MockSpaceStorageMockRecorder) TreeDeletedStatus(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TreeDeletedStatus", reflect.TypeOf((*MockSpaceStorage)(nil).TreeDeletedStatus), arg0) +} + // TreeStorage mocks base method. func (m *MockSpaceStorage) TreeStorage(arg0 string) (storage0.TreeStorage, error) { m.ctrl.T.Helper() diff --git a/common/commonspace/synctree/synctree_test.go b/common/commonspace/synctree/synctree_test.go index 95a3436a..e14433cb 100644 --- a/common/commonspace/synctree/synctree_test.go +++ b/common/commonspace/synctree/synctree_test.go @@ -3,6 +3,7 @@ package synctree import ( "context" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/diffservice" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage/mock_storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/syncservice" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree/mock_synctree" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree/updatelistener" @@ -45,7 +46,8 @@ func Test_DeriveSyncTree(t *testing.T) { updateListenerMock := mock_updatelistener.NewMockUpdateListener(ctrl) syncClientMock := mock_synctree.NewMockSyncClient(ctrl) aclListMock := mock_list.NewMockACLList(ctrl) - objTreeMock := mock_tree.NewMockObjectTree(ctrl) + objTreeMock := newTestObjMock(mock_tree.NewMockObjectTree(ctrl)) + spaceStorageMock := mock_storage.NewMockSpaceStorage(ctrl) spaceId := "spaceId" expectedPayload := tree.ObjectTreeCreatePayload{SpaceId: spaceId} createDerivedObjectTree = func(payload tree.ObjectTreeCreatePayload, l list.ACLList, create storage2.TreeStorageCreatorFunc) (objTree tree.ObjectTree, err error) { @@ -59,7 +61,14 @@ func Test_DeriveSyncTree(t *testing.T) { headUpdate := &treechangeproto.TreeSyncMessage{} syncClientMock.EXPECT().CreateHeadUpdate(syncTreeMatcher{objTreeMock, syncClientMock, updateListenerMock}, gomock.Nil()).Return(headUpdate) syncClientMock.EXPECT().BroadcastAsync(gomock.Eq(headUpdate)).Return(nil) - deps := CreateDeps{AclList: aclListMock, SpaceId: spaceId, Payload: expectedPayload, Listener: updateListenerMock} + updateListenerMock.EXPECT().Rebuild(gomock.Any()) + deps := CreateDeps{ + AclList: aclListMock, + SpaceId: spaceId, + Payload: expectedPayload, + Listener: updateListenerMock, + SpaceStorage: spaceStorageMock, + } _, err := DeriveSyncTree(ctx, deps) require.NoError(t, err) @@ -73,7 +82,8 @@ func Test_CreateSyncTree(t *testing.T) { updateListenerMock := mock_updatelistener.NewMockUpdateListener(ctrl) syncClientMock := mock_synctree.NewMockSyncClient(ctrl) aclListMock := mock_list.NewMockACLList(ctrl) - objTreeMock := mock_tree.NewMockObjectTree(ctrl) + objTreeMock := newTestObjMock(mock_tree.NewMockObjectTree(ctrl)) + spaceStorageMock := mock_storage.NewMockSpaceStorage(ctrl) spaceId := "spaceId" expectedPayload := tree.ObjectTreeCreatePayload{SpaceId: spaceId} createObjectTree = func(payload tree.ObjectTreeCreatePayload, l list.ACLList, create storage2.TreeStorageCreatorFunc) (objTree tree.ObjectTree, err error) { @@ -87,7 +97,14 @@ func Test_CreateSyncTree(t *testing.T) { headUpdate := &treechangeproto.TreeSyncMessage{} syncClientMock.EXPECT().CreateHeadUpdate(syncTreeMatcher{objTreeMock, syncClientMock, updateListenerMock}, gomock.Nil()).Return(headUpdate) syncClientMock.EXPECT().BroadcastAsync(gomock.Eq(headUpdate)).Return(nil) - deps := CreateDeps{AclList: aclListMock, SpaceId: spaceId, Payload: expectedPayload, Listener: updateListenerMock} + updateListenerMock.EXPECT().Rebuild(gomock.Any()) + deps := CreateDeps{ + AclList: aclListMock, + SpaceId: spaceId, + Payload: expectedPayload, + Listener: updateListenerMock, + SpaceStorage: spaceStorageMock, + } _, err := CreateSyncTree(ctx, deps) require.NoError(t, err) @@ -100,7 +117,7 @@ func Test_BuildSyncTree(t *testing.T) { updateListenerMock := mock_updatelistener.NewMockUpdateListener(ctrl) syncClientMock := mock_synctree.NewMockSyncClient(ctrl) - objTreeMock := mock_tree.NewMockObjectTree(ctrl) + objTreeMock := newTestObjMock(mock_tree.NewMockObjectTree(ctrl)) tr := &SyncTree{ ObjectTree: objTreeMock, SyncHandler: nil, diff --git a/common/commonspace/treegetter/mock_treegetter/mock_treegetter.go b/common/commonspace/treegetter/mock_treegetter/mock_treegetter.go index a07949bc..4706000b 100644 --- a/common/commonspace/treegetter/mock_treegetter/mock_treegetter.go +++ b/common/commonspace/treegetter/mock_treegetter/mock_treegetter.go @@ -50,6 +50,20 @@ func (mr *MockTreeGetterMockRecorder) Close(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockTreeGetter)(nil).Close), arg0) } +// DeleteTree mocks base method. +func (m *MockTreeGetter) DeleteTree(arg0 context.Context, arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteTree", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteTree indicates an expected call of DeleteTree. +func (mr *MockTreeGetterMockRecorder) DeleteTree(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteTree", reflect.TypeOf((*MockTreeGetter)(nil).DeleteTree), arg0, arg1, arg2) +} + // GetTree mocks base method. func (m *MockTreeGetter) GetTree(arg0 context.Context, arg1, arg2 string) (tree.ObjectTree, error) { m.ctrl.T.Helper() diff --git a/common/pkg/acl/storage/mock_storage/mock_storage.go b/common/pkg/acl/storage/mock_storage/mock_storage.go index 751184a7..d6361f69 100644 --- a/common/pkg/acl/storage/mock_storage/mock_storage.go +++ b/common/pkg/acl/storage/mock_storage/mock_storage.go @@ -160,6 +160,20 @@ func (mr *MockTreeStorageMockRecorder) AddRawChange(arg0 interface{}) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRawChange", reflect.TypeOf((*MockTreeStorage)(nil).AddRawChange), arg0) } +// Delete mocks base method. +func (m *MockTreeStorage) Delete() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete") + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockTreeStorageMockRecorder) Delete() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockTreeStorage)(nil).Delete)) +} + // GetRawChange mocks base method. func (m *MockTreeStorage) GetRawChange(arg0 context.Context, arg1 string) (*treechangeproto.RawTreeChangeWithId, error) { m.ctrl.T.Helper() diff --git a/common/pkg/acl/tree/mock_objecttree/mock_objecttree.go b/common/pkg/acl/tree/mock_objecttree/mock_objecttree.go index 205fb8ce..4cfca949 100644 --- a/common/pkg/acl/tree/mock_objecttree/mock_objecttree.go +++ b/common/pkg/acl/tree/mock_objecttree/mock_objecttree.go @@ -116,6 +116,20 @@ func (mr *MockObjectTreeMockRecorder) DebugDump() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DebugDump", reflect.TypeOf((*MockObjectTree)(nil).DebugDump)) } +// Delete mocks base method. +func (m *MockObjectTree) Delete() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete") + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockObjectTreeMockRecorder) Delete() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockObjectTree)(nil).Delete)) +} + // HasChanges mocks base method. func (m *MockObjectTree) HasChanges(arg0 ...string) bool { m.ctrl.T.Helper() diff --git a/common/util/periodicsync/mock_periodicsync/mock_periodicsync.go b/common/util/periodicsync/mock_periodicsync/mock_periodicsync.go new file mode 100644 index 00000000..b032a401 --- /dev/null +++ b/common/util/periodicsync/mock_periodicsync/mock_periodicsync.go @@ -0,0 +1,58 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/anytypeio/go-anytype-infrastructure-experiments/common/util/periodicsync (interfaces: PeriodicSync) + +// Package mock_periodicsync is a generated GoMock package. +package mock_periodicsync + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockPeriodicSync is a mock of PeriodicSync interface. +type MockPeriodicSync struct { + ctrl *gomock.Controller + recorder *MockPeriodicSyncMockRecorder +} + +// MockPeriodicSyncMockRecorder is the mock recorder for MockPeriodicSync. +type MockPeriodicSyncMockRecorder struct { + mock *MockPeriodicSync +} + +// NewMockPeriodicSync creates a new mock instance. +func NewMockPeriodicSync(ctrl *gomock.Controller) *MockPeriodicSync { + mock := &MockPeriodicSync{ctrl: ctrl} + mock.recorder = &MockPeriodicSyncMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPeriodicSync) EXPECT() *MockPeriodicSyncMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockPeriodicSync) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockPeriodicSyncMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockPeriodicSync)(nil).Close)) +} + +// Run mocks base method. +func (m *MockPeriodicSync) Run() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Run") +} + +// Run indicates an expected call of Run. +func (mr *MockPeriodicSyncMockRecorder) Run() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockPeriodicSync)(nil).Run)) +} diff --git a/common/util/periodicsync/periodicsync.go b/common/util/periodicsync/periodicsync.go index 37647810..52db0935 100644 --- a/common/util/periodicsync/periodicsync.go +++ b/common/util/periodicsync/periodicsync.go @@ -1,3 +1,4 @@ +//go:generate mockgen -destination mock_periodicsync/mock_periodicsync.go github.com/anytypeio/go-anytype-infrastructure-experiments/common/util/periodicsync PeriodicSync package periodicsync import ( From 41fde6f1adda94581d2ad747e11cdd8adbc82f93 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 25 Nov 2022 15:09:44 +0100 Subject: [PATCH 19/27] Fix diffservice tests --- .../diffservice/diffservice_test.go | 20 ++++++++---- common/commonspace/diffservice/diffsyncer.go | 4 +-- .../diffservice/diffsyncer_test.go | 31 ++++++++++++++----- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/common/commonspace/diffservice/diffservice_test.go b/common/commonspace/diffservice/diffservice_test.go index 9df40e46..ce3600c1 100644 --- a/common/commonspace/diffservice/diffservice_test.go +++ b/common/commonspace/diffservice/diffservice_test.go @@ -3,10 +3,12 @@ package diffservice import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/logger" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/diffservice/mock_diffservice" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage/mock_storage" mock_storage2 "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/storage/mock_storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/ldiff" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/ldiff/mock_ldiff" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/util/periodicsync/mock_periodicsync" "github.com/golang/mock/gomock" "testing" ) @@ -17,10 +19,12 @@ func TestDiffService(t *testing.T) { spaceId := "spaceId" l := logger.NewNamed("sync") - pSyncMock := mock_diffservice.NewMockPeriodicSync(ctrl) + pSyncMock := mock_periodicsync.NewMockPeriodicSync(ctrl) storageMock := mock_storage.NewMockSpaceStorage(ctrl) treeStorageMock := mock_storage2.NewMockTreeStorage(ctrl) diffMock := mock_ldiff.NewMockDiff(ctrl) + syncer := mock_diffservice.NewMockDiffSyncer(ctrl) + delState := deletionstate.NewDeletionState(storageMock) syncPeriod := 1 initId := "initId" @@ -28,6 +32,7 @@ func TestDiffService(t *testing.T) { spaceId: spaceId, storage: storageMock, periodicSync: pSyncMock, + syncer: syncer, diff: diffMock, log: l, syncPeriod: syncPeriod, @@ -36,22 +41,25 @@ func TestDiffService(t *testing.T) { t.Run("init", func(t *testing.T) { storageMock.EXPECT().TreeStorage(initId).Return(treeStorageMock, nil) treeStorageMock.EXPECT().Heads().Return([]string{"h1", "h2"}, nil) + syncer.EXPECT().Init(delState) diffMock.EXPECT().Set(ldiff.Element{ Id: initId, Head: "h1h2", }) pSyncMock.EXPECT().Run() - service.Init([]string{initId}) + service.Init([]string{initId}, delState) }) t.Run("update heads", func(t *testing.T) { - diffMock.EXPECT().Set(ldiff.Element{ - Id: initId, - Head: "h1h2", - }) + syncer.EXPECT().UpdateHeads(initId, []string{"h1", "h2"}) service.UpdateHeads(initId, []string{"h1", "h2"}) }) + t.Run("remove objects", func(t *testing.T) { + syncer.EXPECT().RemoveObjects([]string{"h1", "h2"}) + service.RemoveObjects([]string{"h1", "h2"}) + }) + t.Run("close", func(t *testing.T) { pSyncMock.EXPECT().Close() service.Close() diff --git a/common/commonspace/diffservice/diffsyncer.go b/common/commonspace/diffservice/diffsyncer.go index 4f92d570..723cd50d 100644 --- a/common/commonspace/diffservice/diffsyncer.go +++ b/common/commonspace/diffservice/diffsyncer.go @@ -136,11 +136,11 @@ func (d *diffSyncer) sendPushSpaceRequest(ctx context.Context, cl spacesyncproto return } - spaceSettingsTreeStorage, err := d.storage.TreeStorage(d.storage.SpaceSettingsId()) + settingsStorage, err := d.storage.TreeStorage(d.storage.SpaceSettingsId()) if err != nil { return } - spaceSettingsRoot, err := spaceSettingsTreeStorage.Root() + spaceSettingsRoot, err := settingsStorage.Root() if err != nil { return } diff --git a/common/commonspace/diffservice/diffsyncer_test.go b/common/commonspace/diffservice/diffsyncer_test.go index e6074a71..182eab2c 100644 --- a/common/commonspace/diffservice/diffsyncer_test.go +++ b/common/commonspace/diffservice/diffsyncer_test.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/logger" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/remotediff" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto/mock_spacesyncproto" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage/mock_storage" @@ -13,6 +14,8 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/nodeconf/mock_nodeconf" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/aclrecordproto" mock_aclstorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/storage/mock_storage" + mock_treestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/storage/mock_storage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/treechangeproto" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/ldiff/mock_ldiff" "github.com/golang/mock/gomock" "github.com/libp2p/go-libp2p/core/sec" @@ -25,6 +28,7 @@ import ( type pushSpaceRequestMatcher struct { spaceId string aclRootId string + settingsId string spaceHeader *spacesyncproto.RawSpaceHeaderWithId } @@ -34,7 +38,7 @@ func (p pushSpaceRequestMatcher) Matches(x interface{}) bool { return false } - return res.Payload.AclPayloadId == p.aclRootId && res.Payload.SpaceHeader == p.spaceHeader + return res.Payload.AclPayloadId == p.aclRootId && res.Payload.SpaceHeader == p.spaceHeader && res.Payload.SpaceSettingsPayloadId == p.settingsId } func (p pushSpaceRequestMatcher) String() string { @@ -77,10 +81,12 @@ func (m mockPeer) NewStream(ctx context.Context, rpc string, enc drpc.Encoding) func newPushSpaceRequestMatcher( spaceId string, aclRootId string, + settingsId string, spaceHeader *spacesyncproto.RawSpaceHeaderWithId) *pushSpaceRequestMatcher { return &pushSpaceRequestMatcher{ spaceId: spaceId, aclRootId: aclRootId, + settingsId: settingsId, spaceHeader: spaceHeader, } } @@ -99,10 +105,12 @@ func TestDiffSyncer_Sync(t *testing.T) { factory := spacesyncproto.ClientFactoryFunc(func(cc drpc.Conn) spacesyncproto.DRPCSpaceClient { return clientMock }) + delState := deletionstate.NewDeletionState(stMock) spaceId := "spaceId" aclRootId := "aclRootId" l := logger.NewNamed(spaceId) diffSyncer := newDiffSyncer(spaceId, diffMock, connectorMock, cacheMock, stMock, factory, l) + diffSyncer.Init(delState) t.Run("diff syncer sync simple", func(t *testing.T) { connectorMock.EXPECT(). @@ -129,10 +137,16 @@ func TestDiffSyncer_Sync(t *testing.T) { t.Run("diff syncer sync space missing", func(t *testing.T) { aclStorageMock := mock_aclstorage.NewMockListStorage(ctrl) + settingsStorage := mock_treestorage.NewMockTreeStorage(ctrl) + settingsId := "settingsId" aclRoot := &aclrecordproto.RawACLRecordWithId{ Id: aclRootId, } + settingsRoot := &treechangeproto.RawTreeChangeWithId{ + Id: settingsId, + } spaceHeader := &spacesyncproto.RawSpaceHeaderWithId{} + spaceSettingsId := "spaceSettingsId" connectorMock.EXPECT(). GetResponsiblePeers(gomock.Any(), spaceId). @@ -140,17 +154,18 @@ func TestDiffSyncer_Sync(t *testing.T) { diffMock.EXPECT(). Diff(gomock.Any(), gomock.Eq(remotediff.NewRemoteDiff(spaceId, clientMock))). Return(nil, nil, nil, spacesyncproto.ErrSpaceMissing) - stMock.EXPECT(). - ACLStorage(). - Return(aclStorageMock, nil) - stMock.EXPECT(). - SpaceHeader(). - Return(spaceHeader, nil) + + stMock.EXPECT().ACLStorage().Return(aclStorageMock, nil) + stMock.EXPECT().SpaceHeader().Return(spaceHeader, nil) + stMock.EXPECT().SpaceSettingsId().Return(spaceSettingsId) + stMock.EXPECT().TreeStorage(spaceSettingsId).Return(settingsStorage, nil) + + settingsStorage.EXPECT().Root().Return(settingsRoot, nil) aclStorageMock.EXPECT(). Root(). Return(aclRoot, nil) clientMock.EXPECT(). - PushSpace(gomock.Any(), newPushSpaceRequestMatcher(spaceId, aclRootId, spaceHeader)). + PushSpace(gomock.Any(), newPushSpaceRequestMatcher(spaceId, aclRootId, settingsId, spaceHeader)). Return(nil, nil) require.NoError(t, diffSyncer.Sync(ctx)) From 3c1072db4af29e0e50071e91b65ee7fe58a472a1 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 25 Nov 2022 15:24:25 +0100 Subject: [PATCH 20/27] Update diffsyncer --- .../diffservice/diffsyncer_test.go | 43 +++++++++++++++++++ node/go.mod | 7 ++- node/go.sum | 7 ++- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/common/commonspace/diffservice/diffsyncer_test.go b/common/commonspace/diffservice/diffsyncer_test.go index 182eab2c..fd74b232 100644 --- a/common/commonspace/diffservice/diffsyncer_test.go +++ b/common/commonspace/diffservice/diffsyncer_test.go @@ -8,6 +8,7 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto/mock_spacesyncproto" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage/mock_storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter/mock_treegetter" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/net/peer" @@ -16,6 +17,7 @@ import ( mock_aclstorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/storage/mock_storage" mock_treestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/storage/mock_storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/treechangeproto" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/ldiff" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/ldiff/mock_ldiff" "github.com/golang/mock/gomock" "github.com/libp2p/go-libp2p/core/sec" @@ -112,6 +114,13 @@ func TestDiffSyncer_Sync(t *testing.T) { diffSyncer := newDiffSyncer(spaceId, diffMock, connectorMock, cacheMock, stMock, factory, l) diffSyncer.Init(delState) + delStateAdd := func(deletedId string) { + stMock.EXPECT().TreeDeletedStatus(deletedId).Return("", nil) + stMock.EXPECT().SetTreeDeletedStatus(deletedId, storage.TreeDeletedStatusQueued) + diffMock.EXPECT().RemoveId(deletedId) + require.NoError(t, delState.Add([]string{deletedId})) + } + t.Run("diff syncer sync simple", func(t *testing.T) { connectorMock.EXPECT(). GetResponsiblePeers(gomock.Any(), spaceId). @@ -127,6 +136,22 @@ func TestDiffSyncer_Sync(t *testing.T) { require.NoError(t, diffSyncer.Sync(ctx)) }) + t.Run("diff syncer sync filtered", func(t *testing.T) { + delStateAdd("changed") + connectorMock.EXPECT(). + GetResponsiblePeers(gomock.Any(), spaceId). + Return([]peer.Peer{mockPeer{}}, nil) + diffMock.EXPECT(). + Diff(gomock.Any(), gomock.Eq(remotediff.NewRemoteDiff(spaceId, clientMock))). + Return([]string{"new1", "new2"}, []string{"changed"}, nil, nil) + for _, arg := range []string{"new1", "new2"} { + cacheMock.EXPECT(). + GetTree(gomock.Any(), spaceId, arg). + Return(nil, nil) + } + require.NoError(t, diffSyncer.Sync(ctx)) + }) + t.Run("diff syncer sync conf error", func(t *testing.T) { connectorMock.EXPECT(). GetResponsiblePeers(gomock.Any(), spaceId). @@ -135,6 +160,24 @@ func TestDiffSyncer_Sync(t *testing.T) { require.Error(t, diffSyncer.Sync(ctx)) }) + t.Run("deletion state remove objects", func(t *testing.T) { + deletedId := "id" + delStateAdd(deletedId) + + // this should not result in any mock being called + diffSyncer.UpdateHeads(deletedId, []string{"someHead"}) + }) + + t.Run("update heads updates diff", func(t *testing.T) { + newId := "newId" + newHeads := []string{"h1", "h2"} + diffMock.EXPECT().Set(ldiff.Element{ + Id: newId, + Head: concatStrings(newHeads), + }) + diffSyncer.UpdateHeads(newId, newHeads) + }) + t.Run("diff syncer sync space missing", func(t *testing.T) { aclStorageMock := mock_aclstorage.NewMockListStorage(ctrl) settingsStorage := mock_treestorage.NewMockTreeStorage(ctrl) diff --git a/node/go.mod b/node/go.mod index a65cf8f2..ca016399 100644 --- a/node/go.mod +++ b/node/go.mod @@ -10,6 +10,9 @@ require ( github.com/akrylysov/pogreb v0.10.1 github.com/anytypeio/go-anytype-infrastructure-experiments/common v0.0.0-00010101000000-000000000000 github.com/anytypeio/go-anytype-infrastructure-experiments/consensus v0.0.0-00010101000000-000000000000 + github.com/golang/mock v1.6.0 + github.com/ipfs/go-cid v0.3.2 + github.com/stretchr/testify v1.8.0 go.uber.org/zap v1.23.0 ) @@ -19,6 +22,7 @@ require ( github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cheggaaa/mb/v2 v2.0.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect github.com/goccy/go-graphviz v0.0.9 // indirect @@ -26,12 +30,10 @@ require ( github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/huandu/skiplist v1.2.0 // indirect - github.com/ipfs/go-cid v0.3.2 // indirect github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/klauspost/cpuid/v2 v2.1.1 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-libp2p v0.23.2 // indirect - github.com/libp2p/go-libp2p-core v0.20.1 // indirect github.com/libp2p/go-openssl v0.1.0 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-pointer v0.0.1 // indirect @@ -46,6 +48,7 @@ require ( github.com/multiformats/go-multihash v0.2.1 // indirect github.com/multiformats/go-varint v0.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.13.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect diff --git a/node/go.sum b/node/go.sum index 55067483..78a23ca2 100644 --- a/node/go.sum +++ b/node/go.sum @@ -109,6 +109,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -191,8 +192,6 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-libp2p v0.23.2 h1:yqyTeKQJyofWXxEv/eEVUvOrGdt/9x+0PIQ4N1kaxmE= github.com/libp2p/go-libp2p v0.23.2/go.mod h1:s9DEa5NLR4g+LZS+md5uGU4emjMWFiqkZr6hBTY8UxI= -github.com/libp2p/go-libp2p-core v0.20.1 h1:fQz4BJyIFmSZAiTbKV8qoYhEH5Dtv/cVhZbG3Ib/+Cw= -github.com/libp2p/go-libp2p-core v0.20.1/go.mod h1:6zR8H7CvQWgYLsbG4on6oLNSGcyKaYFSEYyDt51+bIY= github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -271,11 +270,14 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -497,6 +499,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From d49384b98bd993088b342e9a46ed8d9935f6899e Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 28 Nov 2022 16:15:25 +0100 Subject: [PATCH 21/27] Changes for tests and database fixes --- client/storage/spacestorage.go | 3 ++- client/storage/treestorage.go | 3 ++- .../settingsdocument/deletionstate/deletionstate.go | 2 +- .../commonspace/settingsdocument/settingsdocument.go | 10 ++++++---- .../settingsdocument/settingsdocument_test.go | 7 +++++++ 5 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 common/commonspace/settingsdocument/settingsdocument_test.go diff --git a/client/storage/spacestorage.go b/client/storage/spacestorage.go index 7479808c..a6cee186 100644 --- a/client/storage/spacestorage.go +++ b/client/storage/spacestorage.go @@ -138,7 +138,8 @@ func (s *spaceStorage) StoredIds() (ids []string, err error) { for it.Rewind(); it.Valid(); it.Next() { item := it.Item() - id := item.Key() + id := make([]byte, 0, len(item.Key())) + id = item.KeyCopy(id) if len(id) <= len(s.keys.TreeRootPrefix())+1 { continue } diff --git a/client/storage/treestorage.go b/client/storage/treestorage.go index a9403eed..fe96516d 100644 --- a/client/storage/treestorage.go +++ b/client/storage/treestorage.go @@ -158,6 +158,7 @@ func (t *treeStorage) storedKeys() (keys [][]byte, err error) { err = t.db.View(func(txn *badger.Txn) error { opts := badger.DefaultIteratorOptions opts.PrefetchValues = false + // this will get all raw changes and also "heads" opts.Prefix = t.keys.RawChangePrefix() it := txn.NewIterator(opts) @@ -167,7 +168,7 @@ func (t *treeStorage) storedKeys() (keys [][]byte, err error) { item := it.Item() key := item.Key() keyCopy := make([]byte, 0, len(key)) - keyCopy = item.KeyCopy(key) + keyCopy = item.KeyCopy(keyCopy) keys = append(keys, keyCopy) } return nil diff --git a/common/commonspace/settingsdocument/deletionstate/deletionstate.go b/common/commonspace/settingsdocument/deletionstate/deletionstate.go index 3387dd9d..8a38528c 100644 --- a/common/commonspace/settingsdocument/deletionstate/deletionstate.go +++ b/common/commonspace/settingsdocument/deletionstate/deletionstate.go @@ -86,7 +86,7 @@ func (st *DeletionState) Delete(id string) (err error) { defer st.Unlock() delete(st.queued, id) st.deleted[id] = struct{}{} - err = st.storage.SetTreeDeletedStatus(id, storage.TreeDeletedStatusQueued) + err = st.storage.SetTreeDeletedStatus(id, storage.TreeDeletedStatusDeleted) if err != nil { return } diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go index 8c603a5e..c9359321 100644 --- a/common/commonspace/settingsdocument/settingsdocument.go +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -7,7 +7,7 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" spacestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" - "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/syncservice/synchandler" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree/updatelistener" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" @@ -35,7 +35,8 @@ type Deps struct { } type settingsDocument struct { - *synctree.SyncTree + tree.ObjectTree + synchandler.SyncHandler account account.Service spaceId string treeGetter treegetter.TreeGetter @@ -99,11 +100,12 @@ func (s *settingsDocument) Rebuild(tr tree.ObjectTree) { } func (s *settingsDocument) Init(ctx context.Context) (err error) { - syncTree, err := s.buildFunc(ctx, s.store.SpaceSettingsId(), s) + s.ObjectTree, err = s.buildFunc(ctx, s.store.SpaceSettingsId(), s) if err != nil { return } - s.SyncTree = syncTree.(*synctree.SyncTree) + // this is needed, so we would easily convert the object to synchandler interface in objectgetter + s.SyncHandler = s.ObjectTree.(synchandler.SyncHandler) s.loop.Run() return } diff --git a/common/commonspace/settingsdocument/settingsdocument_test.go b/common/commonspace/settingsdocument/settingsdocument_test.go new file mode 100644 index 00000000..43d929dc --- /dev/null +++ b/common/commonspace/settingsdocument/settingsdocument_test.go @@ -0,0 +1,7 @@ +package settingsdocument + +import "testing" + +func TestSettingsDocument_Init(t *testing.T) { + +} From 82400407b9fcf40dcd4ac845a5225651ffdcf4a2 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 28 Nov 2022 16:54:31 +0100 Subject: [PATCH 22/27] Improve interfaces for synctree, generate mocks --- common/account/mock_account/mock_account.go | 78 +++++ common/account/service.go | 1 + .../settingsdocument/idprovider.go | 3 +- .../mock_settingsdocument.go | 51 +++ .../settingsdocument/settingsdocument.go | 19 +- common/commonspace/space.go | 9 +- .../synctree/mock_synctree/mock_synctree.go | 313 +++++++++++++++++- common/commonspace/synctree/syncclient.go | 2 +- common/commonspace/synctree/synctree.go | 47 +-- common/commonspace/synctree/synctree_test.go | 4 +- 10 files changed, 489 insertions(+), 38 deletions(-) create mode 100644 common/account/mock_account/mock_account.go create mode 100644 common/commonspace/settingsdocument/mock_settingsdocument/mock_settingsdocument.go diff --git a/common/account/mock_account/mock_account.go b/common/account/mock_account/mock_account.go new file mode 100644 index 00000000..6f6d3b54 --- /dev/null +++ b/common/account/mock_account/mock_account.go @@ -0,0 +1,78 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/anytypeio/go-anytype-infrastructure-experiments/common/account (interfaces: Service) + +// Package mock_account is a generated GoMock package. +package mock_account + +import ( + reflect "reflect" + + app "github.com/anytypeio/go-anytype-infrastructure-experiments/common/app" + account "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/account" + gomock "github.com/golang/mock/gomock" +) + +// MockService is a mock of Service interface. +type MockService struct { + ctrl *gomock.Controller + recorder *MockServiceMockRecorder +} + +// MockServiceMockRecorder is the mock recorder for MockService. +type MockServiceMockRecorder struct { + mock *MockService +} + +// NewMockService creates a new mock instance. +func NewMockService(ctrl *gomock.Controller) *MockService { + mock := &MockService{ctrl: ctrl} + mock.recorder = &MockServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockService) EXPECT() *MockServiceMockRecorder { + return m.recorder +} + +// Account mocks base method. +func (m *MockService) Account() *account.AccountData { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Account") + ret0, _ := ret[0].(*account.AccountData) + return ret0 +} + +// Account indicates an expected call of Account. +func (mr *MockServiceMockRecorder) Account() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Account", reflect.TypeOf((*MockService)(nil).Account)) +} + +// Init mocks base method. +func (m *MockService) Init(arg0 *app.App) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Init", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Init indicates an expected call of Init. +func (mr *MockServiceMockRecorder) Init(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockService)(nil).Init), arg0) +} + +// Name mocks base method. +func (m *MockService) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockServiceMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockService)(nil).Name)) +} diff --git a/common/account/service.go b/common/account/service.go index ab009a51..23b279b2 100644 --- a/common/account/service.go +++ b/common/account/service.go @@ -1,3 +1,4 @@ +//go:generate mockgen -destination mock_account/mock_account.go github.com/anytypeio/go-anytype-infrastructure-experiments/common/account Service package account import ( diff --git a/common/commonspace/settingsdocument/idprovider.go b/common/commonspace/settingsdocument/idprovider.go index bc64da57..56fccab6 100644 --- a/common/commonspace/settingsdocument/idprovider.go +++ b/common/commonspace/settingsdocument/idprovider.go @@ -1,3 +1,4 @@ +//go:generate mockgen -destination mock_settingsdocument/mock_settingsdocument.go github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument DeletedIdsProvider package settingsdocument import ( @@ -6,7 +7,7 @@ import ( "github.com/gogo/protobuf/proto" ) -type deletedIdsProvider interface { +type DeletedIdsProvider interface { ProvideIds(tr tree.ObjectTree, startId string) (ids []string, lastId string, err error) } diff --git a/common/commonspace/settingsdocument/mock_settingsdocument/mock_settingsdocument.go b/common/commonspace/settingsdocument/mock_settingsdocument/mock_settingsdocument.go new file mode 100644 index 00000000..4a8a58d8 --- /dev/null +++ b/common/commonspace/settingsdocument/mock_settingsdocument/mock_settingsdocument.go @@ -0,0 +1,51 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument (interfaces: DeletedIdsProvider) + +// Package mock_settingsdocument is a generated GoMock package. +package mock_settingsdocument + +import ( + reflect "reflect" + + tree "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" + gomock "github.com/golang/mock/gomock" +) + +// MockDeletedIdsProvider is a mock of DeletedIdsProvider interface. +type MockDeletedIdsProvider struct { + ctrl *gomock.Controller + recorder *MockDeletedIdsProviderMockRecorder +} + +// MockDeletedIdsProviderMockRecorder is the mock recorder for MockDeletedIdsProvider. +type MockDeletedIdsProviderMockRecorder struct { + mock *MockDeletedIdsProvider +} + +// NewMockDeletedIdsProvider creates a new mock instance. +func NewMockDeletedIdsProvider(ctrl *gomock.Controller) *MockDeletedIdsProvider { + mock := &MockDeletedIdsProvider{ctrl: ctrl} + mock.recorder = &MockDeletedIdsProviderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDeletedIdsProvider) EXPECT() *MockDeletedIdsProviderMockRecorder { + return m.recorder +} + +// ProvideIds mocks base method. +func (m *MockDeletedIdsProvider) ProvideIds(arg0 tree.ObjectTree, arg1 string) ([]string, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ProvideIds", arg0, arg1) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ProvideIds indicates an expected call of ProvideIds. +func (mr *MockDeletedIdsProviderMockRecorder) ProvideIds(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProvideIds", reflect.TypeOf((*MockDeletedIdsProvider)(nil).ProvideIds), arg0, arg1) +} diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go index c9359321..64d07987 100644 --- a/common/commonspace/settingsdocument/settingsdocument.go +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -7,7 +7,7 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" spacestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" - "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/syncservice/synchandler" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree/updatelistener" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" @@ -17,12 +17,12 @@ import ( var log = logger.NewNamed("commonspace.settingsdocument") type SettingsDocument interface { - tree.ObjectTree + synctree.SyncTree Init(ctx context.Context) (err error) DeleteObject(id string) (err error) } -type BuildTreeFunc func(ctx context.Context, id string, listener updatelistener.UpdateListener) (t tree.ObjectTree, err error) +type BuildTreeFunc func(ctx context.Context, id string, listener updatelistener.UpdateListener) (t synctree.SyncTree, err error) type Deps struct { BuildFunc BuildTreeFunc @@ -31,17 +31,16 @@ type Deps struct { Store spacestorage.SpaceStorage DeletionState *deletionstate.DeletionState // prov exists mainly for the ease of testing - prov deletedIdsProvider + prov DeletedIdsProvider } type settingsDocument struct { - tree.ObjectTree - synchandler.SyncHandler + synctree.SyncTree account account.Service spaceId string treeGetter treegetter.TreeGetter store spacestorage.SpaceStorage - prov deletedIdsProvider + prov DeletedIdsProvider buildFunc BuildTreeFunc loop *deleteLoop @@ -100,19 +99,17 @@ func (s *settingsDocument) Rebuild(tr tree.ObjectTree) { } func (s *settingsDocument) Init(ctx context.Context) (err error) { - s.ObjectTree, err = s.buildFunc(ctx, s.store.SpaceSettingsId(), s) + s.SyncTree, err = s.buildFunc(ctx, s.store.SpaceSettingsId(), s) if err != nil { return } - // this is needed, so we would easily convert the object to synchandler interface in objectgetter - s.SyncHandler = s.ObjectTree.(synchandler.SyncHandler) s.loop.Run() return } func (s *settingsDocument) Close() error { s.loop.Close() - return s.ObjectTree.Close() + return s.SyncTree.Close() } func (s *settingsDocument) DeleteObject(id string) (err error) { diff --git a/common/commonspace/space.go b/common/commonspace/space.go index 9a1f39ff..78085be4 100644 --- a/common/commonspace/space.go +++ b/common/commonspace/space.go @@ -154,7 +154,14 @@ func (s *space) Init(ctx context.Context) (err error) { deletionState := deletionstate.NewDeletionState(s.storage) deps := settingsdocument.Deps{ - BuildFunc: s.BuildTree, + BuildFunc: func(ctx context.Context, id string, listener updatelistener.UpdateListener) (t synctree.SyncTree, err error) { + res, err := s.BuildTree(ctx, id, listener) + if err != nil { + return + } + t = res.(synctree.SyncTree) + return + }, Account: s.account, TreeGetter: s.cache, Store: s.storage, diff --git a/common/commonspace/synctree/mock_synctree/mock_synctree.go b/common/commonspace/synctree/mock_synctree/mock_synctree.go index 914a6ec1..8f726e61 100644 --- a/common/commonspace/synctree/mock_synctree/mock_synctree.go +++ b/common/commonspace/synctree/mock_synctree/mock_synctree.go @@ -1,12 +1,15 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree (interfaces: SyncClient) +// Source: github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree (interfaces: SyncClient,SyncTree) // Package mock_synctree is a generated GoMock package. package mock_synctree import ( + context "context" reflect "reflect" + spacesyncproto "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" + storage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/storage" tree "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" treechangeproto "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/treechangeproto" gomock "github.com/golang/mock/gomock" @@ -134,3 +137,311 @@ func (mr *MockSyncClientMockRecorder) SendAsync(arg0, arg1, arg2 interface{}) *g mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendAsync", reflect.TypeOf((*MockSyncClient)(nil).SendAsync), arg0, arg1, arg2) } + +// MockSyncTree is a mock of SyncTree interface. +type MockSyncTree struct { + ctrl *gomock.Controller + recorder *MockSyncTreeMockRecorder +} + +// MockSyncTreeMockRecorder is the mock recorder for MockSyncTree. +type MockSyncTreeMockRecorder struct { + mock *MockSyncTree +} + +// NewMockSyncTree creates a new mock instance. +func NewMockSyncTree(ctrl *gomock.Controller) *MockSyncTree { + mock := &MockSyncTree{ctrl: ctrl} + mock.recorder = &MockSyncTreeMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSyncTree) EXPECT() *MockSyncTreeMockRecorder { + return m.recorder +} + +// AddContent mocks base method. +func (m *MockSyncTree) AddContent(arg0 context.Context, arg1 tree.SignableChangeContent) (tree.AddResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddContent", arg0, arg1) + ret0, _ := ret[0].(tree.AddResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AddContent indicates an expected call of AddContent. +func (mr *MockSyncTreeMockRecorder) AddContent(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddContent", reflect.TypeOf((*MockSyncTree)(nil).AddContent), arg0, arg1) +} + +// AddRawChanges mocks base method. +func (m *MockSyncTree) AddRawChanges(arg0 context.Context, arg1 ...*treechangeproto.RawTreeChangeWithId) (tree.AddResult, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0} + for _, a := range arg1 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "AddRawChanges", varargs...) + ret0, _ := ret[0].(tree.AddResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AddRawChanges indicates an expected call of AddRawChanges. +func (mr *MockSyncTreeMockRecorder) AddRawChanges(arg0 interface{}, arg1 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0}, arg1...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRawChanges", reflect.TypeOf((*MockSyncTree)(nil).AddRawChanges), varargs...) +} + +// ChangesAfterCommonSnapshot mocks base method. +func (m *MockSyncTree) ChangesAfterCommonSnapshot(arg0, arg1 []string) ([]*treechangeproto.RawTreeChangeWithId, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ChangesAfterCommonSnapshot", arg0, arg1) + ret0, _ := ret[0].([]*treechangeproto.RawTreeChangeWithId) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ChangesAfterCommonSnapshot indicates an expected call of ChangesAfterCommonSnapshot. +func (mr *MockSyncTreeMockRecorder) ChangesAfterCommonSnapshot(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChangesAfterCommonSnapshot", reflect.TypeOf((*MockSyncTree)(nil).ChangesAfterCommonSnapshot), arg0, arg1) +} + +// Close mocks base method. +func (m *MockSyncTree) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockSyncTreeMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockSyncTree)(nil).Close)) +} + +// DebugDump mocks base method. +func (m *MockSyncTree) DebugDump() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DebugDump") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DebugDump indicates an expected call of DebugDump. +func (mr *MockSyncTreeMockRecorder) DebugDump() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DebugDump", reflect.TypeOf((*MockSyncTree)(nil).DebugDump)) +} + +// Delete mocks base method. +func (m *MockSyncTree) Delete() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete") + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockSyncTreeMockRecorder) Delete() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockSyncTree)(nil).Delete)) +} + +// HandleMessage mocks base method. +func (m *MockSyncTree) HandleMessage(arg0 context.Context, arg1 string, arg2 *spacesyncproto.ObjectSyncMessage) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HandleMessage", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// HandleMessage indicates an expected call of HandleMessage. +func (mr *MockSyncTreeMockRecorder) HandleMessage(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandleMessage", reflect.TypeOf((*MockSyncTree)(nil).HandleMessage), arg0, arg1, arg2) +} + +// HasChanges mocks base method. +func (m *MockSyncTree) HasChanges(arg0 ...string) bool { + m.ctrl.T.Helper() + varargs := []interface{}{} + for _, a := range arg0 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "HasChanges", varargs...) + ret0, _ := ret[0].(bool) + return ret0 +} + +// HasChanges indicates an expected call of HasChanges. +func (mr *MockSyncTreeMockRecorder) HasChanges(arg0 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasChanges", reflect.TypeOf((*MockSyncTree)(nil).HasChanges), arg0...) +} + +// Header mocks base method. +func (m *MockSyncTree) Header() *treechangeproto.RawTreeChangeWithId { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Header") + ret0, _ := ret[0].(*treechangeproto.RawTreeChangeWithId) + return ret0 +} + +// Header indicates an expected call of Header. +func (mr *MockSyncTreeMockRecorder) Header() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Header", reflect.TypeOf((*MockSyncTree)(nil).Header)) +} + +// Heads mocks base method. +func (m *MockSyncTree) Heads() []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Heads") + ret0, _ := ret[0].([]string) + return ret0 +} + +// Heads indicates an expected call of Heads. +func (mr *MockSyncTreeMockRecorder) Heads() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Heads", reflect.TypeOf((*MockSyncTree)(nil).Heads)) +} + +// ID mocks base method. +func (m *MockSyncTree) ID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ID") + ret0, _ := ret[0].(string) + return ret0 +} + +// ID indicates an expected call of ID. +func (mr *MockSyncTreeMockRecorder) ID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ID", reflect.TypeOf((*MockSyncTree)(nil).ID)) +} + +// Iterate mocks base method. +func (m *MockSyncTree) Iterate(arg0 func([]byte) (interface{}, error), arg1 func(*tree.Change) bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Iterate", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Iterate indicates an expected call of Iterate. +func (mr *MockSyncTreeMockRecorder) Iterate(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Iterate", reflect.TypeOf((*MockSyncTree)(nil).Iterate), arg0, arg1) +} + +// IterateFrom mocks base method. +func (m *MockSyncTree) IterateFrom(arg0 string, arg1 func([]byte) (interface{}, error), arg2 func(*tree.Change) bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IterateFrom", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// IterateFrom indicates an expected call of IterateFrom. +func (mr *MockSyncTreeMockRecorder) IterateFrom(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IterateFrom", reflect.TypeOf((*MockSyncTree)(nil).IterateFrom), arg0, arg1, arg2) +} + +// Lock mocks base method. +func (m *MockSyncTree) Lock() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Lock") +} + +// Lock indicates an expected call of Lock. +func (mr *MockSyncTreeMockRecorder) Lock() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Lock", reflect.TypeOf((*MockSyncTree)(nil).Lock)) +} + +// RLock mocks base method. +func (m *MockSyncTree) RLock() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RLock") +} + +// RLock indicates an expected call of RLock. +func (mr *MockSyncTreeMockRecorder) RLock() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RLock", reflect.TypeOf((*MockSyncTree)(nil).RLock)) +} + +// RUnlock mocks base method. +func (m *MockSyncTree) RUnlock() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RUnlock") +} + +// RUnlock indicates an expected call of RUnlock. +func (mr *MockSyncTreeMockRecorder) RUnlock() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RUnlock", reflect.TypeOf((*MockSyncTree)(nil).RUnlock)) +} + +// Root mocks base method. +func (m *MockSyncTree) Root() *tree.Change { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Root") + ret0, _ := ret[0].(*tree.Change) + return ret0 +} + +// Root indicates an expected call of Root. +func (mr *MockSyncTreeMockRecorder) Root() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Root", reflect.TypeOf((*MockSyncTree)(nil).Root)) +} + +// SnapshotPath mocks base method. +func (m *MockSyncTree) SnapshotPath() []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SnapshotPath") + ret0, _ := ret[0].([]string) + return ret0 +} + +// SnapshotPath indicates an expected call of SnapshotPath. +func (mr *MockSyncTreeMockRecorder) SnapshotPath() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SnapshotPath", reflect.TypeOf((*MockSyncTree)(nil).SnapshotPath)) +} + +// Storage mocks base method. +func (m *MockSyncTree) Storage() storage.TreeStorage { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Storage") + ret0, _ := ret[0].(storage.TreeStorage) + return ret0 +} + +// Storage indicates an expected call of Storage. +func (mr *MockSyncTreeMockRecorder) Storage() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Storage", reflect.TypeOf((*MockSyncTree)(nil).Storage)) +} + +// Unlock mocks base method. +func (m *MockSyncTree) Unlock() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Unlock") +} + +// Unlock indicates an expected call of Unlock. +func (mr *MockSyncTreeMockRecorder) Unlock() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unlock", reflect.TypeOf((*MockSyncTree)(nil).Unlock)) +} diff --git a/common/commonspace/synctree/syncclient.go b/common/commonspace/synctree/syncclient.go index d8a721ec..506447ee 100644 --- a/common/commonspace/synctree/syncclient.go +++ b/common/commonspace/synctree/syncclient.go @@ -1,4 +1,4 @@ -//go:generate mockgen -destination mock_synctree/mock_synctree.go github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree SyncClient +//go:generate mockgen -destination mock_synctree/mock_synctree.go github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree SyncClient,SyncTree package synctree import ( diff --git a/common/commonspace/synctree/synctree.go b/common/commonspace/synctree/synctree.go index e471a961..8aa94c03 100644 --- a/common/commonspace/synctree/synctree.go +++ b/common/commonspace/synctree/synctree.go @@ -25,8 +25,13 @@ var ( ErrSyncTreeDeleted = errors.New("sync tree is deleted") ) +type SyncTree interface { + tree.ObjectTree + synchandler.SyncHandler +} + // SyncTree sends head updates to sync service and also sends new changes to update listener -type SyncTree struct { +type syncTree struct { tree.ObjectTree synchandler.SyncHandler syncClient SyncClient @@ -64,8 +69,8 @@ type BuildDeps struct { TreeStorage storage.TreeStorage } -func DeriveSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, err error) { - t, err = createDerivedObjectTree(deps.Payload, deps.AclList, deps.SpaceStorage.CreateTreeStorage) +func DeriveSyncTree(ctx context.Context, deps CreateDeps) (t SyncTree, err error) { + objTree, err := createDerivedObjectTree(deps.Payload, deps.AclList, deps.SpaceStorage.CreateTreeStorage) if err != nil { return } @@ -75,8 +80,8 @@ func DeriveSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, er deps.HeadNotifiable, sharedFactory, deps.Configuration) - syncTree := &SyncTree{ - ObjectTree: t, + syncTree := &syncTree{ + ObjectTree: objTree, syncClient: syncClient, listener: deps.Listener, } @@ -94,8 +99,8 @@ func DeriveSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, er return } -func CreateSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, err error) { - t, err = createObjectTree(deps.Payload, deps.AclList, deps.SpaceStorage.CreateTreeStorage) +func CreateSyncTree(ctx context.Context, deps CreateDeps) (t SyncTree, err error) { + objTree, err := createObjectTree(deps.Payload, deps.AclList, deps.SpaceStorage.CreateTreeStorage) if err != nil { return } @@ -105,8 +110,8 @@ func CreateSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, er deps.HeadNotifiable, GetRequestFactory(), deps.Configuration) - syncTree := &SyncTree{ - ObjectTree: t, + syncTree := &syncTree{ + ObjectTree: objTree, syncClient: syncClient, listener: deps.Listener, } @@ -125,7 +130,7 @@ func CreateSyncTree(ctx context.Context, deps CreateDeps) (t tree.ObjectTree, er return } -func BuildSyncTreeOrGetRemote(ctx context.Context, id string, deps BuildDeps) (t tree.ObjectTree, err error) { +func BuildSyncTreeOrGetRemote(ctx context.Context, id string, deps BuildDeps) (t SyncTree, err error) { getTreeRemote := func() (msg *treechangeproto.TreeSyncMessage, err error) { peerId, err := peer.CtxPeerId(ctx) if err != nil { @@ -194,8 +199,8 @@ func BuildSyncTreeOrGetRemote(ctx context.Context, id string, deps BuildDeps) (t return buildSyncTree(ctx, true, deps) } -func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t tree.ObjectTree, err error) { - t, err = buildObjectTree(deps.TreeStorage, deps.AclList) +func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t SyncTree, err error) { + objTree, err := buildObjectTree(deps.TreeStorage, deps.AclList) if err != nil { return } @@ -205,8 +210,8 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t tr deps.HeadNotifiable, GetRequestFactory(), deps.Configuration) - syncTree := &SyncTree{ - ObjectTree: t, + syncTree := &syncTree{ + ObjectTree: objTree, syncClient: syncClient, listener: deps.Listener, } @@ -231,21 +236,21 @@ func buildSyncTree(ctx context.Context, isFirstBuild bool, deps BuildDeps) (t tr return } -func (s *SyncTree) IterateFrom(id string, convert tree.ChangeConvertFunc, iterate tree.ChangeIterateFunc) (err error) { +func (s *syncTree) IterateFrom(id string, convert tree.ChangeConvertFunc, iterate tree.ChangeIterateFunc) (err error) { if err = s.checkAlive(); err != nil { return } return s.ObjectTree.IterateFrom(id, convert, iterate) } -func (s *SyncTree) Iterate(convert tree.ChangeConvertFunc, iterate tree.ChangeIterateFunc) (err error) { +func (s *syncTree) Iterate(convert tree.ChangeConvertFunc, iterate tree.ChangeIterateFunc) (err error) { if err = s.checkAlive(); err != nil { return } return s.ObjectTree.Iterate(convert, iterate) } -func (s *SyncTree) AddContent(ctx context.Context, content tree.SignableChangeContent) (res tree.AddResult, err error) { +func (s *syncTree) AddContent(ctx context.Context, content tree.SignableChangeContent) (res tree.AddResult, err error) { if err = s.checkAlive(); err != nil { return } @@ -258,7 +263,7 @@ func (s *SyncTree) AddContent(ctx context.Context, content tree.SignableChangeCo return } -func (s *SyncTree) AddRawChanges(ctx context.Context, changes ...*treechangeproto.RawTreeChangeWithId) (res tree.AddResult, err error) { +func (s *syncTree) AddRawChanges(ctx context.Context, changes ...*treechangeproto.RawTreeChangeWithId) (res tree.AddResult, err error) { if err = s.checkAlive(); err != nil { return } @@ -283,7 +288,7 @@ func (s *SyncTree) AddRawChanges(ctx context.Context, changes ...*treechangeprot return } -func (s *SyncTree) Delete() (err error) { +func (s *syncTree) Delete() (err error) { log.With("id", s.ID()).Debug("deleting sync tree") s.Lock() defer s.Unlock() @@ -298,7 +303,7 @@ func (s *SyncTree) Delete() (err error) { return } -func (s *SyncTree) Close() (err error) { +func (s *syncTree) Close() (err error) { log.With("id", s.ID()).Debug("closing sync tree") s.Lock() defer s.Unlock() @@ -309,7 +314,7 @@ func (s *SyncTree) Close() (err error) { return } -func (s *SyncTree) checkAlive() (err error) { +func (s *syncTree) checkAlive() (err error) { if s.isClosed { err = ErrSyncTreeClosed } diff --git a/common/commonspace/synctree/synctree_test.go b/common/commonspace/synctree/synctree_test.go index e14433cb..df852e07 100644 --- a/common/commonspace/synctree/synctree_test.go +++ b/common/commonspace/synctree/synctree_test.go @@ -27,7 +27,7 @@ type syncTreeMatcher struct { } func (s syncTreeMatcher) Matches(x interface{}) bool { - t, ok := x.(*SyncTree) + t, ok := x.(*syncTree) if !ok { return false } @@ -118,7 +118,7 @@ func Test_BuildSyncTree(t *testing.T) { updateListenerMock := mock_updatelistener.NewMockUpdateListener(ctrl) syncClientMock := mock_synctree.NewMockSyncClient(ctrl) objTreeMock := newTestObjMock(mock_tree.NewMockObjectTree(ctrl)) - tr := &SyncTree{ + tr := &syncTree{ ObjectTree: objTreeMock, SyncHandler: nil, syncClient: syncClientMock, From 4a3f20b236facb73b069ed3db7a3eccf7563d5d6 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 29 Nov 2022 16:54:04 +0100 Subject: [PATCH 23/27] Tests and refactoring --- common/commonspace/diffservice/diffservice.go | 4 +- common/commonspace/diffservice/diffsyncer.go | 6 +- .../commonspace/settingsdocument/deleter.go | 10 +- .../deletionstate/deletionstate.go | 29 +++-- .../settingsdocument/idprovider.go | 1 - .../mock_settingsdocument.go | 37 ++++++- .../settingsdocument/settingsdocument.go | 23 +++- .../settingsdocument/settingsdocument_test.go | 103 +++++++++++++++++- common/commonspace/space.go | 5 +- 9 files changed, 187 insertions(+), 31 deletions(-) diff --git a/common/commonspace/diffservice/diffservice.go b/common/commonspace/diffservice/diffservice.go index 113e4e5c..ae3b5cb4 100644 --- a/common/commonspace/diffservice/diffservice.go +++ b/common/commonspace/diffservice/diffservice.go @@ -21,7 +21,7 @@ type DiffService interface { RemoveObjects(ids []string) AllIds() []string - Init(objectIds []string, deletionState *deletionstate.DeletionState) + Init(objectIds []string, deletionState deletionstate.DeletionState) Close() (err error) } @@ -61,7 +61,7 @@ func NewDiffService( } } -func (d *diffService) Init(objectIds []string, deletionState *deletionstate.DeletionState) { +func (d *diffService) Init(objectIds []string, deletionState deletionstate.DeletionState) { d.fillDiff(objectIds) d.syncer.Init(deletionState) d.periodicSync.Run() diff --git a/common/commonspace/diffservice/diffsyncer.go b/common/commonspace/diffservice/diffsyncer.go index 723cd50d..1618ee71 100644 --- a/common/commonspace/diffservice/diffsyncer.go +++ b/common/commonspace/diffservice/diffsyncer.go @@ -19,7 +19,7 @@ type DiffSyncer interface { Sync(ctx context.Context) error RemoveObjects(ids []string) UpdateHeads(id string, heads []string) - Init(deletionState *deletionstate.DeletionState) + Init(deletionState deletionstate.DeletionState) } func newDiffSyncer( @@ -49,10 +49,10 @@ type diffSyncer struct { storage storage.SpaceStorage clientFactory spacesyncproto.ClientFactory log *zap.Logger - deletionState *deletionstate.DeletionState + deletionState deletionstate.DeletionState } -func (d *diffSyncer) Init(deletionState *deletionstate.DeletionState) { +func (d *diffSyncer) Init(deletionState deletionstate.DeletionState) { d.deletionState = deletionState d.deletionState.AddObserver(d.RemoveObjects) } diff --git a/common/commonspace/settingsdocument/deleter.go b/common/commonspace/settingsdocument/deleter.go index 51a8c32b..f1b5ca45 100644 --- a/common/commonspace/settingsdocument/deleter.go +++ b/common/commonspace/settingsdocument/deleter.go @@ -8,17 +8,21 @@ import ( "go.uber.org/zap" ) +type Deleter interface { + Delete() +} + type deleter struct { st storage.SpaceStorage - state *deletionstate.DeletionState + state deletionstate.DeletionState getter treegetter.TreeGetter } -func newDeleter(st storage.SpaceStorage, state *deletionstate.DeletionState, getter treegetter.TreeGetter) *deleter { +func newDeleter(st storage.SpaceStorage, state deletionstate.DeletionState, getter treegetter.TreeGetter) Deleter { return &deleter{st, state, getter} } -func (d *deleter) delete() { +func (d *deleter) Delete() { allQueued := d.state.GetQueued() for _, id := range allQueued { err := d.getter.DeleteTree(context.Background(), d.st.Id(), id) diff --git a/common/commonspace/settingsdocument/deletionstate/deletionstate.go b/common/commonspace/settingsdocument/deletionstate/deletionstate.go index 8a38528c..fe56f0b2 100644 --- a/common/commonspace/settingsdocument/deletionstate/deletionstate.go +++ b/common/commonspace/settingsdocument/deletionstate/deletionstate.go @@ -7,7 +7,16 @@ import ( type StateUpdateObserver func(ids []string) -type DeletionState struct { +type DeletionState interface { + AddObserver(observer StateUpdateObserver) + Add(ids []string) (err error) + GetQueued() (ids []string) + Delete(id string) (err error) + Exists(id string) bool + FilterJoin(ids ...[]string) (filtered []string) +} + +type deletionState struct { sync.RWMutex queued map[string]struct{} deleted map[string]struct{} @@ -15,21 +24,21 @@ type DeletionState struct { storage storage.SpaceStorage } -func NewDeletionState(storage storage.SpaceStorage) *DeletionState { - return &DeletionState{ +func NewDeletionState(storage storage.SpaceStorage) DeletionState { + return &deletionState{ queued: map[string]struct{}{}, deleted: map[string]struct{}{}, storage: storage, } } -func (st *DeletionState) AddObserver(observer StateUpdateObserver) { +func (st *deletionState) AddObserver(observer StateUpdateObserver) { st.Lock() defer st.Unlock() st.stateUpdateObservers = append(st.stateUpdateObservers, observer) } -func (st *DeletionState) Add(ids []string) (err error) { +func (st *deletionState) Add(ids []string) (err error) { st.Lock() defer func() { st.Unlock() @@ -71,7 +80,7 @@ func (st *DeletionState) Add(ids []string) (err error) { return } -func (st *DeletionState) GetQueued() (ids []string) { +func (st *deletionState) GetQueued() (ids []string) { st.RLock() defer st.RUnlock() ids = make([]string, 0, len(st.queued)) @@ -81,7 +90,7 @@ func (st *DeletionState) GetQueued() (ids []string) { return } -func (st *DeletionState) Delete(id string) (err error) { +func (st *deletionState) Delete(id string) (err error) { st.Lock() defer st.Unlock() delete(st.queued, id) @@ -93,13 +102,13 @@ func (st *DeletionState) Delete(id string) (err error) { return } -func (st *DeletionState) Exists(id string) bool { +func (st *deletionState) Exists(id string) bool { st.RLock() defer st.RUnlock() return st.exists(id) } -func (st *DeletionState) FilterJoin(ids ...[]string) (filtered []string) { +func (st *deletionState) FilterJoin(ids ...[]string) (filtered []string) { st.RLock() defer st.RUnlock() filter := func(ids []string) { @@ -115,7 +124,7 @@ func (st *DeletionState) FilterJoin(ids ...[]string) (filtered []string) { return } -func (st *DeletionState) exists(id string) bool { +func (st *deletionState) exists(id string) bool { if _, exists := st.deleted[id]; exists { return true } diff --git a/common/commonspace/settingsdocument/idprovider.go b/common/commonspace/settingsdocument/idprovider.go index 56fccab6..9ced17b3 100644 --- a/common/commonspace/settingsdocument/idprovider.go +++ b/common/commonspace/settingsdocument/idprovider.go @@ -1,4 +1,3 @@ -//go:generate mockgen -destination mock_settingsdocument/mock_settingsdocument.go github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument DeletedIdsProvider package settingsdocument import ( diff --git a/common/commonspace/settingsdocument/mock_settingsdocument/mock_settingsdocument.go b/common/commonspace/settingsdocument/mock_settingsdocument/mock_settingsdocument.go index 4a8a58d8..7d950895 100644 --- a/common/commonspace/settingsdocument/mock_settingsdocument/mock_settingsdocument.go +++ b/common/commonspace/settingsdocument/mock_settingsdocument/mock_settingsdocument.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument (interfaces: DeletedIdsProvider) +// Source: github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument (interfaces: DeletedIdsProvider,Deleter) // Package mock_settingsdocument is a generated GoMock package. package mock_settingsdocument @@ -49,3 +49,38 @@ func (mr *MockDeletedIdsProviderMockRecorder) ProvideIds(arg0, arg1 interface{}) mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProvideIds", reflect.TypeOf((*MockDeletedIdsProvider)(nil).ProvideIds), arg0, arg1) } + +// MockDeleter is a mock of Deleter interface. +type MockDeleter struct { + ctrl *gomock.Controller + recorder *MockDeleterMockRecorder +} + +// MockDeleterMockRecorder is the mock recorder for MockDeleter. +type MockDeleterMockRecorder struct { + mock *MockDeleter +} + +// NewMockDeleter creates a new mock instance. +func NewMockDeleter(ctrl *gomock.Controller) *MockDeleter { + mock := &MockDeleter{ctrl: ctrl} + mock.recorder = &MockDeleterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDeleter) EXPECT() *MockDeleterMockRecorder { + return m.recorder +} + +// Delete mocks base method. +func (m *MockDeleter) Delete() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Delete") +} + +// Delete indicates an expected call of Delete. +func (mr *MockDeleterMockRecorder) Delete() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockDeleter)(nil).Delete)) +} diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go index 64d07987..7bb034a6 100644 --- a/common/commonspace/settingsdocument/settingsdocument.go +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -1,3 +1,4 @@ +//go:generate mockgen -destination mock_settingsdocument/mock_settingsdocument.go github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument DeletedIdsProvider,Deleter package settingsdocument import ( @@ -29,9 +30,10 @@ type Deps struct { Account account.Service TreeGetter treegetter.TreeGetter Store spacestorage.SpaceStorage - DeletionState *deletionstate.DeletionState - // prov exists mainly for the ease of testing + DeletionState deletionstate.DeletionState + // testing dependencies prov DeletedIdsProvider + del Deleter } type settingsDocument struct { @@ -44,14 +46,20 @@ type settingsDocument struct { buildFunc BuildTreeFunc loop *deleteLoop - deletionState *deletionstate.DeletionState + deletionState deletionstate.DeletionState lastChangeId string } -func NewSettingsDocument(deps Deps, spaceId string) (doc SettingsDocument, err error) { - deleter := newDeleter(deps.Store, deps.DeletionState, deps.TreeGetter) +func NewSettingsDocument(deps Deps, spaceId string) (doc SettingsDocument) { + var deleter Deleter + if deps.del == nil { + deleter = newDeleter(deps.Store, deps.DeletionState, deps.TreeGetter) + } else { + deleter = deps.del + } + loop := newDeleteLoop(func() { - deleter.delete() + deleter.Delete() }) deps.DeletionState.AddObserver(func(ids []string) { loop.notify() @@ -89,10 +97,12 @@ func (s *settingsDocument) updateIds(tr tree.ObjectTree, lastChangeId string) { } } +// Update is called as part of UpdateListener interface func (s *settingsDocument) Update(tr tree.ObjectTree) { s.updateIds(tr, s.lastChangeId) } +// Rebuild is called as part of UpdateListener interface (including when the object is built for the first time, e.g. on Init call) func (s *settingsDocument) Rebuild(tr tree.ObjectTree) { // at initial build "s" may not contain the object tree, so it is safer to provide it from the function parameter s.updateIds(tr, "") @@ -128,6 +138,7 @@ func (s *settingsDocument) DeleteObject(id string) (err error) { }, Snapshot: nil, } + // TODO: add snapshot logic res, err := change.Marshal() if err != nil { return diff --git a/common/commonspace/settingsdocument/settingsdocument_test.go b/common/commonspace/settingsdocument/settingsdocument_test.go index 43d929dc..9ee5a9e4 100644 --- a/common/commonspace/settingsdocument/settingsdocument_test.go +++ b/common/commonspace/settingsdocument/settingsdocument_test.go @@ -1,7 +1,108 @@ package settingsdocument -import "testing" +import ( + "context" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/account/mock_account" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/mock_settingsdocument" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage/mock_storage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree/mock_synctree" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree/updatelistener" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter/mock_treegetter" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +type settingsFixture struct { + spaceId string + docId string + doc SettingsDocument + ctrl *gomock.Controller + treeGetter *mock_treegetter.MockTreeGetter + spaceStorage *mock_storage.MockSpaceStorage + provider *mock_settingsdocument.MockDeletedIdsProvider + deleter *mock_settingsdocument.MockDeleter + syncTree *mock_synctree.MockSyncTree + delState *deletionstate.DeletionState + account *mock_account.MockService +} + +func newSettingsFixture(t *testing.T) *settingsFixture { + spaceId := "spaceId" + docId := "documentId" + + ctrl := gomock.NewController(t) + acc := mock_account.NewMockService(ctrl) + treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) + st := mock_storage.NewMockSpaceStorage(ctrl) + delState := deletionstate.NewDeletionState(st) + prov := mock_settingsdocument.NewMockDeletedIdsProvider(ctrl) + syncTree := mock_synctree.NewMockSyncTree(ctrl) + del := mock_settingsdocument.NewMockDeleter(ctrl) + + buildFunc := BuildTreeFunc(func(ctx context.Context, id string, listener updatelistener.UpdateListener) (synctree.SyncTree, error) { + require.Equal(t, docId, id) + return syncTree, nil + }) + + deps := Deps{ + BuildFunc: buildFunc, + Account: acc, + TreeGetter: treeGetter, + Store: st, + DeletionState: delState, + prov: prov, + del: del, + } + doc := NewSettingsDocument(deps, spaceId) + return &settingsFixture{ + spaceId: spaceId, + docId: docId, + doc: doc, + ctrl: ctrl, + treeGetter: treeGetter, + spaceStorage: st, + provider: prov, + deleter: del, + syncTree: syncTree, + account: acc, + delState: delState, + } +} + +func (fx *settingsFixture) stop() { + fx.ctrl.Finish() +} func TestSettingsDocument_Init(t *testing.T) { + fx := newSettingsFixture(t) + defer fx.stop() + fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId) + fx.deleter.EXPECT().Delete() + fx.syncTree.EXPECT().Close().Return(nil) + + err := fx.doc.Init(context.Background()) + require.NoError(t, err) + err = fx.doc.Close() + require.NoError(t, err) +} + +func TestSettingsDocument_DeleteObject(t *testing.T) { + fx := newSettingsFixture(t) + defer fx.stop() + + fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId) + fx.deleter.EXPECT().Delete() + fx.syncTree.EXPECT().Close().Return(nil) + + err := fx.doc.Init(context.Background()) + require.NoError(t, err) + time.Sleep(10 * time.Millisecond) + + err = fx.doc.Close() + require.NoError(t, err) } diff --git a/common/commonspace/space.go b/common/commonspace/space.go index 78085be4..7401f30c 100644 --- a/common/commonspace/space.go +++ b/common/commonspace/space.go @@ -167,10 +167,7 @@ func (s *space) Init(ctx context.Context) (err error) { Store: s.storage, DeletionState: deletionState, } - s.settingsDocument, err = settingsdocument.NewSettingsDocument(deps, s.id) - if err != nil { - return - } + s.settingsDocument = settingsdocument.NewSettingsDocument(deps, s.id) objectGetter := newCommonSpaceGetter(s.id, s.aclList, s.cache, s.settingsDocument) s.syncService.Init(objectGetter) From 19bc9a0c932ae30ebc3d6eb8008c3fa46859fe9b Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 29 Nov 2022 18:53:04 +0100 Subject: [PATCH 24/27] Add settingsdocument tests --- .../diffservice/diffservice_test.go | 4 +- .../diffservice/diffsyncer_test.go | 35 +---- .../mock_diffservice/mock_diffservice.go | 2 +- .../deletionstate/deletionstate.go | 18 +++ .../mock_deletionstate/mock_deletionstate.go | 136 ++++++++++++++++++ .../settingsdocument/idprovider.go | 47 +++--- .../settingsdocument/settingsdocument.go | 27 ++-- .../settingsdocument/settingsdocument_test.go | 96 +++++++++++-- 8 files changed, 290 insertions(+), 75 deletions(-) create mode 100644 common/commonspace/settingsdocument/deletionstate/mock_deletionstate/mock_deletionstate.go diff --git a/common/commonspace/diffservice/diffservice_test.go b/common/commonspace/diffservice/diffservice_test.go index ce3600c1..6cc21d90 100644 --- a/common/commonspace/diffservice/diffservice_test.go +++ b/common/commonspace/diffservice/diffservice_test.go @@ -3,7 +3,7 @@ package diffservice import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/logger" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/diffservice/mock_diffservice" - "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate/mock_deletionstate" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage/mock_storage" mock_storage2 "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/storage/mock_storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/ldiff" @@ -24,7 +24,7 @@ func TestDiffService(t *testing.T) { treeStorageMock := mock_storage2.NewMockTreeStorage(ctrl) diffMock := mock_ldiff.NewMockDiff(ctrl) syncer := mock_diffservice.NewMockDiffSyncer(ctrl) - delState := deletionstate.NewDeletionState(storageMock) + delState := mock_deletionstate.NewMockDeletionState(ctrl) syncPeriod := 1 initId := "initId" diff --git a/common/commonspace/diffservice/diffsyncer_test.go b/common/commonspace/diffservice/diffsyncer_test.go index fd74b232..88238b9a 100644 --- a/common/commonspace/diffservice/diffsyncer_test.go +++ b/common/commonspace/diffservice/diffsyncer_test.go @@ -5,10 +5,9 @@ import ( "fmt" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/logger" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/remotediff" - "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate/mock_deletionstate" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto/mock_spacesyncproto" - "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage/mock_storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter/mock_treegetter" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/net/peer" @@ -107,27 +106,22 @@ func TestDiffSyncer_Sync(t *testing.T) { factory := spacesyncproto.ClientFactoryFunc(func(cc drpc.Conn) spacesyncproto.DRPCSpaceClient { return clientMock }) - delState := deletionstate.NewDeletionState(stMock) + delState := mock_deletionstate.NewMockDeletionState(ctrl) spaceId := "spaceId" aclRootId := "aclRootId" l := logger.NewNamed(spaceId) diffSyncer := newDiffSyncer(spaceId, diffMock, connectorMock, cacheMock, stMock, factory, l) + delState.EXPECT().AddObserver(gomock.Any()) diffSyncer.Init(delState) - delStateAdd := func(deletedId string) { - stMock.EXPECT().TreeDeletedStatus(deletedId).Return("", nil) - stMock.EXPECT().SetTreeDeletedStatus(deletedId, storage.TreeDeletedStatusQueued) - diffMock.EXPECT().RemoveId(deletedId) - require.NoError(t, delState.Add([]string{deletedId})) - } - - t.Run("diff syncer sync simple", func(t *testing.T) { + t.Run("diff syncer sync", func(t *testing.T) { connectorMock.EXPECT(). GetResponsiblePeers(gomock.Any(), spaceId). Return([]peer.Peer{mockPeer{}}, nil) diffMock.EXPECT(). Diff(gomock.Any(), gomock.Eq(remotediff.NewRemoteDiff(spaceId, clientMock))). Return([]string{"new"}, []string{"changed"}, nil, nil) + delState.EXPECT().FilterJoin(gomock.Any()).Return([]string{"new", "changed"}) for _, arg := range []string{"new", "changed"} { cacheMock.EXPECT(). GetTree(gomock.Any(), spaceId, arg). @@ -136,22 +130,6 @@ func TestDiffSyncer_Sync(t *testing.T) { require.NoError(t, diffSyncer.Sync(ctx)) }) - t.Run("diff syncer sync filtered", func(t *testing.T) { - delStateAdd("changed") - connectorMock.EXPECT(). - GetResponsiblePeers(gomock.Any(), spaceId). - Return([]peer.Peer{mockPeer{}}, nil) - diffMock.EXPECT(). - Diff(gomock.Any(), gomock.Eq(remotediff.NewRemoteDiff(spaceId, clientMock))). - Return([]string{"new1", "new2"}, []string{"changed"}, nil, nil) - for _, arg := range []string{"new1", "new2"} { - cacheMock.EXPECT(). - GetTree(gomock.Any(), spaceId, arg). - Return(nil, nil) - } - require.NoError(t, diffSyncer.Sync(ctx)) - }) - t.Run("diff syncer sync conf error", func(t *testing.T) { connectorMock.EXPECT(). GetResponsiblePeers(gomock.Any(), spaceId). @@ -162,7 +140,7 @@ func TestDiffSyncer_Sync(t *testing.T) { t.Run("deletion state remove objects", func(t *testing.T) { deletedId := "id" - delStateAdd(deletedId) + delState.EXPECT().Exists(deletedId).Return(true) // this should not result in any mock being called diffSyncer.UpdateHeads(deletedId, []string{"someHead"}) @@ -175,6 +153,7 @@ func TestDiffSyncer_Sync(t *testing.T) { Id: newId, Head: concatStrings(newHeads), }) + delState.EXPECT().Exists(newId).Return(false) diffSyncer.UpdateHeads(newId, newHeads) }) diff --git a/common/commonspace/diffservice/mock_diffservice/mock_diffservice.go b/common/commonspace/diffservice/mock_diffservice/mock_diffservice.go index f54342d9..1bca9970 100644 --- a/common/commonspace/diffservice/mock_diffservice/mock_diffservice.go +++ b/common/commonspace/diffservice/mock_diffservice/mock_diffservice.go @@ -36,7 +36,7 @@ func (m *MockDiffSyncer) EXPECT() *MockDiffSyncerMockRecorder { } // Init mocks base method. -func (m *MockDiffSyncer) Init(arg0 *deletionstate.DeletionState) { +func (m *MockDiffSyncer) Init(arg0 deletionstate.DeletionState) { m.ctrl.T.Helper() m.ctrl.Call(m, "Init", arg0) } diff --git a/common/commonspace/settingsdocument/deletionstate/deletionstate.go b/common/commonspace/settingsdocument/deletionstate/deletionstate.go index fe56f0b2..92f60cc3 100644 --- a/common/commonspace/settingsdocument/deletionstate/deletionstate.go +++ b/common/commonspace/settingsdocument/deletionstate/deletionstate.go @@ -1,6 +1,8 @@ +//go:generate mockgen -destination mock_deletionstate/mock_deletionstate.go github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate DeletionState package deletionstate import ( + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" "sync" ) @@ -14,6 +16,7 @@ type DeletionState interface { Delete(id string) (err error) Exists(id string) bool FilterJoin(ids ...[]string) (filtered []string) + CreateDeleteChange(id string, isSnapshot bool) (res []byte, err error) } type deletionState struct { @@ -124,6 +127,21 @@ func (st *deletionState) FilterJoin(ids ...[]string) (filtered []string) { return } +func (st *deletionState) CreateDeleteChange(id string, isSnapshot bool) (res []byte, err error) { + content := &spacesyncproto.SpaceSettingsContent_ObjectDelete{ + ObjectDelete: &spacesyncproto.ObjectDelete{Id: id}, + } + change := &spacesyncproto.SettingsData{ + Content: []*spacesyncproto.SpaceSettingsContent{ + {content}, + }, + Snapshot: nil, + } + // TODO: add snapshot logic + res, err = change.Marshal() + return +} + func (st *deletionState) exists(id string) bool { if _, exists := st.deleted[id]; exists { return true diff --git a/common/commonspace/settingsdocument/deletionstate/mock_deletionstate/mock_deletionstate.go b/common/commonspace/settingsdocument/deletionstate/mock_deletionstate/mock_deletionstate.go new file mode 100644 index 00000000..12d2ce56 --- /dev/null +++ b/common/commonspace/settingsdocument/deletionstate/mock_deletionstate/mock_deletionstate.go @@ -0,0 +1,136 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate (interfaces: DeletionState) + +// Package mock_deletionstate is a generated GoMock package. +package mock_deletionstate + +import ( + reflect "reflect" + + deletionstate "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" + gomock "github.com/golang/mock/gomock" +) + +// MockDeletionState is a mock of DeletionState interface. +type MockDeletionState struct { + ctrl *gomock.Controller + recorder *MockDeletionStateMockRecorder +} + +// MockDeletionStateMockRecorder is the mock recorder for MockDeletionState. +type MockDeletionStateMockRecorder struct { + mock *MockDeletionState +} + +// NewMockDeletionState creates a new mock instance. +func NewMockDeletionState(ctrl *gomock.Controller) *MockDeletionState { + mock := &MockDeletionState{ctrl: ctrl} + mock.recorder = &MockDeletionStateMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDeletionState) EXPECT() *MockDeletionStateMockRecorder { + return m.recorder +} + +// Add mocks base method. +func (m *MockDeletionState) Add(arg0 []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Add", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Add indicates an expected call of Add. +func (mr *MockDeletionStateMockRecorder) Add(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockDeletionState)(nil).Add), arg0) +} + +// AddObserver mocks base method. +func (m *MockDeletionState) AddObserver(arg0 deletionstate.StateUpdateObserver) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "AddObserver", arg0) +} + +// AddObserver indicates an expected call of AddObserver. +func (mr *MockDeletionStateMockRecorder) AddObserver(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddObserver", reflect.TypeOf((*MockDeletionState)(nil).AddObserver), arg0) +} + +// CreateDeleteChange mocks base method. +func (m *MockDeletionState) CreateDeleteChange(arg0 string, arg1 bool) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateDeleteChange", arg0, arg1) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateDeleteChange indicates an expected call of CreateDeleteChange. +func (mr *MockDeletionStateMockRecorder) CreateDeleteChange(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateDeleteChange", reflect.TypeOf((*MockDeletionState)(nil).CreateDeleteChange), arg0, arg1) +} + +// Delete mocks base method. +func (m *MockDeletionState) Delete(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockDeletionStateMockRecorder) Delete(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockDeletionState)(nil).Delete), arg0) +} + +// Exists mocks base method. +func (m *MockDeletionState) Exists(arg0 string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exists", arg0) + ret0, _ := ret[0].(bool) + return ret0 +} + +// Exists indicates an expected call of Exists. +func (mr *MockDeletionStateMockRecorder) Exists(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockDeletionState)(nil).Exists), arg0) +} + +// FilterJoin mocks base method. +func (m *MockDeletionState) FilterJoin(arg0 ...[]string) []string { + m.ctrl.T.Helper() + varargs := []interface{}{} + for _, a := range arg0 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "FilterJoin", varargs...) + ret0, _ := ret[0].([]string) + return ret0 +} + +// FilterJoin indicates an expected call of FilterJoin. +func (mr *MockDeletionStateMockRecorder) FilterJoin(arg0 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FilterJoin", reflect.TypeOf((*MockDeletionState)(nil).FilterJoin), arg0...) +} + +// GetQueued mocks base method. +func (m *MockDeletionState) GetQueued() []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetQueued") + ret0, _ := ret[0].([]string) + return ret0 +} + +// GetQueued indicates an expected call of GetQueued. +func (mr *MockDeletionStateMockRecorder) GetQueued() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetQueued", reflect.TypeOf((*MockDeletionState)(nil).GetQueued)) +} diff --git a/common/commonspace/settingsdocument/idprovider.go b/common/commonspace/settingsdocument/idprovider.go index 9ced17b3..6e42d0d2 100644 --- a/common/commonspace/settingsdocument/idprovider.go +++ b/common/commonspace/settingsdocument/idprovider.go @@ -21,33 +21,38 @@ func (p *provider) convert(decrypted []byte) (res any, err error) { return deleteChange, nil } +func (p *provider) processChange(change *tree.Change, tr tree.ObjectTree, startId string, ids []string) []string { + // ignoring root change which has empty model or startId change + if change.Model == nil || (change.Id == startId && startId != "") { + return ids + } + + deleteChange := change.Model.(*spacesyncproto.SettingsData) + // getting data from snapshot if we start from it + if change.Id == tr.Root().Id { + ids = deleteChange.Snapshot.DeletedIds + return ids + } + + // otherwise getting data from content + for _, cnt := range deleteChange.Content { + if cnt.GetObjectDelete() != nil { + ids = append(ids, cnt.GetObjectDelete().GetId()) + } + } + return ids +} + func (p *provider) ProvideIds(tr tree.ObjectTree, startId string) (ids []string, lastId string, err error) { - processChange := func(change *tree.Change) bool { - // ignoring root change which has empty model or startId change + process := func(change *tree.Change) bool { lastId = change.Id - if change.Model == nil || (change.Id == startId && startId != "") { - return true - } - - deleteChange := change.Model.(*spacesyncproto.SettingsData) - // getting data from snapshot if we start from it - if change.Id == tr.Root().Id { - ids = deleteChange.Snapshot.DeletedIds - return true - } - - // otherwise getting data from content - for _, cnt := range deleteChange.Content { - if cnt.GetObjectDelete() != nil { - ids = append(ids, cnt.GetObjectDelete().GetId()) - } - } + ids = p.processChange(change, tr, startId, ids) return true } if startId == "" { - err = tr.IterateFrom(tr.ID(), p.convert, processChange) + err = tr.IterateFrom(tr.ID(), p.convert, process) } else { - err = tr.IterateFrom(startId, p.convert, processChange) + err = tr.IterateFrom(startId, p.convert, process) } return } diff --git a/common/commonspace/settingsdocument/settingsdocument.go b/common/commonspace/settingsdocument/settingsdocument.go index 7bb034a6..fc05ad31 100644 --- a/common/commonspace/settingsdocument/settingsdocument.go +++ b/common/commonspace/settingsdocument/settingsdocument.go @@ -6,7 +6,6 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/common/account" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/logger" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" - "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" spacestorage "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree/updatelistener" @@ -78,6 +77,8 @@ func NewSettingsDocument(deps Deps, spaceId string) (doc SettingsDocument) { // this is needed mainly for testing if deps.prov == nil { s.prov = &provider{} + } else { + s.prov = deps.prov } doc = s @@ -113,6 +114,7 @@ func (s *settingsDocument) Init(ctx context.Context) (err error) { if err != nil { return } + s.loop.Run() return } @@ -129,29 +131,24 @@ func (s *settingsDocument) DeleteObject(id string) (err error) { return nil } - content := &spacesyncproto.SpaceSettingsContent_ObjectDelete{ - ObjectDelete: &spacesyncproto.ObjectDelete{Id: id}, - } - change := &spacesyncproto.SettingsData{ - Content: []*spacesyncproto.SpaceSettingsContent{ - {content}, - }, - Snapshot: nil, - } // TODO: add snapshot logic - res, err := change.Marshal() + res, err := s.deletionState.CreateDeleteChange(id, false) if err != nil { return } + + accountData := s.account.Account() _, err = s.AddContent(context.Background(), tree.SignableChangeContent{ Data: res, - Key: s.account.Account().SignKey, - Identity: s.account.Account().Identity, + Key: accountData.SignKey, + Identity: accountData.Identity, IsSnapshot: false, IsEncrypted: false, }) - if err == nil { - s.Update(s) + if err != nil { + return } + + s.Update(s) return } diff --git a/common/commonspace/settingsdocument/settingsdocument_test.go b/common/commonspace/settingsdocument/settingsdocument_test.go index 9ee5a9e4..d69e0fbc 100644 --- a/common/commonspace/settingsdocument/settingsdocument_test.go +++ b/common/commonspace/settingsdocument/settingsdocument_test.go @@ -3,30 +3,53 @@ package settingsdocument import ( "context" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/account/mock_account" - "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate/mock_deletionstate" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/mock_settingsdocument" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage/mock_storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree/mock_synctree" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/synctree/updatelistener" "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter/mock_treegetter" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/account" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/util/keys/asymmetric/signingkey" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + "sync" "testing" "time" ) +type testSyncTreeMock struct { + *mock_synctree.MockSyncTree + m sync.Mutex +} + +func newTestObjMock(mockTree *mock_synctree.MockSyncTree) *testSyncTreeMock { + return &testSyncTreeMock{ + MockSyncTree: mockTree, + } +} + +func (t *testSyncTreeMock) Lock() { + t.m.Lock() +} + +func (t *testSyncTreeMock) Unlock() { + t.m.Unlock() +} + type settingsFixture struct { spaceId string docId string - doc SettingsDocument + doc *settingsDocument ctrl *gomock.Controller treeGetter *mock_treegetter.MockTreeGetter spaceStorage *mock_storage.MockSpaceStorage provider *mock_settingsdocument.MockDeletedIdsProvider deleter *mock_settingsdocument.MockDeleter syncTree *mock_synctree.MockSyncTree - delState *deletionstate.DeletionState + delState *mock_deletionstate.MockDeletionState account *mock_account.MockService } @@ -38,14 +61,16 @@ func newSettingsFixture(t *testing.T) *settingsFixture { acc := mock_account.NewMockService(ctrl) treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) st := mock_storage.NewMockSpaceStorage(ctrl) - delState := deletionstate.NewDeletionState(st) + delState := mock_deletionstate.NewMockDeletionState(ctrl) prov := mock_settingsdocument.NewMockDeletedIdsProvider(ctrl) syncTree := mock_synctree.NewMockSyncTree(ctrl) del := mock_settingsdocument.NewMockDeleter(ctrl) + delState.EXPECT().AddObserver(gomock.Any()) + buildFunc := BuildTreeFunc(func(ctx context.Context, id string, listener updatelistener.UpdateListener) (synctree.SyncTree, error) { require.Equal(t, docId, id) - return syncTree, nil + return newTestObjMock(syncTree), nil }) deps := Deps{ @@ -57,7 +82,7 @@ func newSettingsFixture(t *testing.T) *settingsFixture { prov: prov, del: del, } - doc := NewSettingsDocument(deps, spaceId) + doc := NewSettingsDocument(deps, spaceId).(*settingsDocument) return &settingsFixture{ spaceId: spaceId, docId: docId, @@ -97,12 +122,67 @@ func TestSettingsDocument_DeleteObject(t *testing.T) { fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId) fx.deleter.EXPECT().Delete() - fx.syncTree.EXPECT().Close().Return(nil) err := fx.doc.Init(context.Background()) require.NoError(t, err) - time.Sleep(10 * time.Millisecond) + time.Sleep(100 * time.Millisecond) + delId := "delId" + + fx.delState.EXPECT().Exists(delId).Return(false) + res := []byte("settingsData") + fx.delState.EXPECT().CreateDeleteChange(delId, false).Return(res, nil) + + accountData := &account.AccountData{ + Identity: []byte("id"), + PeerKey: nil, + SignKey: &signingkey.Ed25519PrivateKey{}, + EncKey: nil, + } + fx.account.EXPECT().Account().Return(accountData) + fx.syncTree.EXPECT().AddContent(gomock.Any(), tree.SignableChangeContent{ + Data: res, + Key: accountData.SignKey, + Identity: accountData.Identity, + IsSnapshot: false, + IsEncrypted: false, + }).Return(tree.AddResult{}, nil) + + lastChangeId := "someId" + retIds := []string{"id1", "id2"} + fx.doc.lastChangeId = lastChangeId + fx.provider.EXPECT().ProvideIds(gomock.Not(nil), lastChangeId).Return(retIds, retIds[len(retIds)-1], nil) + fx.delState.EXPECT().Add(retIds).Return(nil) + err = fx.doc.DeleteObject(delId) + require.NoError(t, err) + require.Equal(t, retIds[len(retIds)-1], fx.doc.lastChangeId) + + fx.syncTree.EXPECT().Close().Return(nil) + err = fx.doc.Close() + require.NoError(t, err) +} + +func TestSettingsDocument_Rebuild(t *testing.T) { + fx := newSettingsFixture(t) + defer fx.stop() + + fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId) + fx.deleter.EXPECT().Delete() + + err := fx.doc.Init(context.Background()) + require.NoError(t, err) + time.Sleep(100 * time.Millisecond) + + lastChangeId := "someId" + retIds := []string{"id1", "id2"} + fx.doc.lastChangeId = lastChangeId + fx.provider.EXPECT().ProvideIds(gomock.Not(nil), "").Return(retIds, retIds[len(retIds)-1], nil) + fx.delState.EXPECT().Add(retIds).Return(nil) + + fx.doc.Rebuild(fx.doc) + require.Equal(t, retIds[len(retIds)-1], fx.doc.lastChangeId) + + fx.syncTree.EXPECT().Close().Return(nil) err = fx.doc.Close() require.NoError(t, err) } From 06d68e25ac053c1a189a9790245c6d60fae6bd77 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 29 Nov 2022 19:37:18 +0100 Subject: [PATCH 25/27] Add deletionstate tests --- .../settingsdocument/deleter_test.go | 7 + .../deletionstate/deletionstate_test.go | 127 ++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 common/commonspace/settingsdocument/deleter_test.go create mode 100644 common/commonspace/settingsdocument/deletionstate/deletionstate_test.go diff --git a/common/commonspace/settingsdocument/deleter_test.go b/common/commonspace/settingsdocument/deleter_test.go new file mode 100644 index 00000000..90b9cec4 --- /dev/null +++ b/common/commonspace/settingsdocument/deleter_test.go @@ -0,0 +1,7 @@ +package settingsdocument + +import "testing" + +func TestDeleter_Delete(t *testing.T) { + +} diff --git a/common/commonspace/settingsdocument/deletionstate/deletionstate_test.go b/common/commonspace/settingsdocument/deletionstate/deletionstate_test.go new file mode 100644 index 00000000..d3feeae3 --- /dev/null +++ b/common/commonspace/settingsdocument/deletionstate/deletionstate_test.go @@ -0,0 +1,127 @@ +package deletionstate + +import ( + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage/mock_storage" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + "testing" +) + +type fixture struct { + ctrl *gomock.Controller + delState *deletionState + spaceStorage *mock_storage.MockSpaceStorage +} + +func newFixture(t *testing.T) *fixture { + ctrl := gomock.NewController(t) + spaceStorage := mock_storage.NewMockSpaceStorage(ctrl) + delState := NewDeletionState(spaceStorage).(*deletionState) + return &fixture{ + ctrl: ctrl, + delState: delState, + spaceStorage: spaceStorage, + } +} + +func (fx *fixture) stop() { + fx.ctrl.Finish() +} + +func TestDeletionState_Add(t *testing.T) { + t.Run("add new", func(t *testing.T) { + fx := newFixture(t) + defer fx.stop() + id := "newId" + fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return("", nil) + fx.spaceStorage.EXPECT().SetTreeDeletedStatus(id, storage.TreeDeletedStatusQueued).Return(nil) + err := fx.delState.Add([]string{id}) + require.NoError(t, err) + require.Contains(t, fx.delState.queued, id) + }) + + t.Run("add existing queued", func(t *testing.T) { + fx := newFixture(t) + defer fx.stop() + id := "newId" + fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return(storage.TreeDeletedStatusQueued, nil) + err := fx.delState.Add([]string{id}) + require.NoError(t, err) + require.Contains(t, fx.delState.queued, id) + }) + + t.Run("add existing deleted", func(t *testing.T) { + fx := newFixture(t) + defer fx.stop() + id := "newId" + fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return(storage.TreeDeletedStatusDeleted, nil) + err := fx.delState.Add([]string{id}) + require.NoError(t, err) + require.Contains(t, fx.delState.deleted, id) + }) +} + +func TestDeletionState_GetQueued(t *testing.T) { + fx := newFixture(t) + defer fx.stop() + + fx.delState.queued["id1"] = struct{}{} + fx.delState.queued["id2"] = struct{}{} + + queued := fx.delState.GetQueued() + require.Equal(t, []string{"id1", "id2"}, queued) +} + +func TestDeletionState_FilterJoin(t *testing.T) { + fx := newFixture(t) + defer fx.stop() + + fx.delState.queued["id1"] = struct{}{} + fx.delState.queued["id2"] = struct{}{} + + filtered := fx.delState.FilterJoin([]string{"id1"}, []string{"id3", "id2"}, []string{"id4"}) + require.Equal(t, []string{"id3", "id4"}, filtered) +} + +func TestDeletionState_AddObserver(t *testing.T) { + fx := newFixture(t) + defer fx.stop() + + var queued []string + + fx.delState.AddObserver(func(ids []string) { + queued = ids + }) + id := "newId" + fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return("", nil) + fx.spaceStorage.EXPECT().SetTreeDeletedStatus(id, storage.TreeDeletedStatusQueued).Return(nil) + err := fx.delState.Add([]string{id}) + require.NoError(t, err) + require.Contains(t, fx.delState.queued, id) + require.Equal(t, []string{id}, queued) +} + +func TestDeletionState_Delete(t *testing.T) { + fx := newFixture(t) + defer fx.stop() + + id := "deletedId" + fx.delState.queued[id] = struct{}{} + fx.spaceStorage.EXPECT().SetTreeDeletedStatus(id, storage.TreeDeletedStatusDeleted).Return(nil) + err := fx.delState.Delete(id) + require.NoError(t, err) + require.Contains(t, fx.delState.deleted, id) + require.NotContains(t, fx.delState.queued, id) +} + +func TestDeletionState_Exists(t *testing.T) { + fx := newFixture(t) + defer fx.stop() + + fx.delState.queued["id1"] = struct{}{} + fx.delState.deleted["id2"] = struct{}{} + require.True(t, fx.delState.Exists("id1")) + require.True(t, fx.delState.Exists("id2")) + require.False(t, fx.delState.Exists("id3")) +} \ No newline at end of file From 6bc81244131b79c264fc9b82c7baa47982d1e79a Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 29 Nov 2022 19:45:51 +0100 Subject: [PATCH 26/27] Add deleter tests --- .../settingsdocument/deleter_test.go | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/common/commonspace/settingsdocument/deleter_test.go b/common/commonspace/settingsdocument/deleter_test.go index 90b9cec4..669180d7 100644 --- a/common/commonspace/settingsdocument/deleter_test.go +++ b/common/commonspace/settingsdocument/deleter_test.go @@ -1,7 +1,52 @@ package settingsdocument -import "testing" +import ( + "fmt" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/settingsdocument/deletionstate/mock_deletionstate" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/storage/mock_storage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/treegetter/mock_treegetter" + "github.com/golang/mock/gomock" + "testing" +) func TestDeleter_Delete(t *testing.T) { + ctrl := gomock.NewController(t) + treeGetter := mock_treegetter.NewMockTreeGetter(ctrl) + st := mock_storage.NewMockSpaceStorage(ctrl) + delState := mock_deletionstate.NewMockDeletionState(ctrl) + deleter := newDeleter(st, delState, treeGetter) + + t.Run("deleter delete queued", func(t *testing.T) { + id := "id" + spaceId := "spaceId" + delState.EXPECT().GetQueued().Return([]string{id}) + st.EXPECT().Id().Return(spaceId) + treeGetter.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(nil) + delState.EXPECT().Delete(id).Return(nil) + + deleter.Delete() + }) + + t.Run("deleter delete already deleted", func(t *testing.T) { + id := "id" + spaceId := "spaceId" + delState.EXPECT().GetQueued().Return([]string{id}) + st.EXPECT().Id().Return(spaceId) + treeGetter.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(storage.ErrTreeStorageAlreadyDeleted) + delState.EXPECT().Delete(id).Return(nil) + + deleter.Delete() + }) + + t.Run("deleter delete error", func(t *testing.T) { + id := "id" + spaceId := "spaceId" + delState.EXPECT().GetQueued().Return([]string{id}) + st.EXPECT().Id().Return(spaceId) + treeGetter.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(fmt.Errorf("some error")) + + deleter.Delete() + }) } From 75b5193dc4ed89dfee9597c81c4f9368bebefe0e Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 29 Nov 2022 20:54:44 +0100 Subject: [PATCH 27/27] Add idprovider tests --- .../settingsdocument/idprovider.go | 29 +++--- .../settingsdocument/idprovider_test.go | 94 +++++++++++++++++++ 2 files changed, 109 insertions(+), 14 deletions(-) create mode 100644 common/commonspace/settingsdocument/idprovider_test.go diff --git a/common/commonspace/settingsdocument/idprovider.go b/common/commonspace/settingsdocument/idprovider.go index 6e42d0d2..fd8e1f39 100644 --- a/common/commonspace/settingsdocument/idprovider.go +++ b/common/commonspace/settingsdocument/idprovider.go @@ -12,16 +12,7 @@ type DeletedIdsProvider interface { type provider struct{} -func (p *provider) convert(decrypted []byte) (res any, err error) { - deleteChange := &spacesyncproto.SettingsData{} - err = proto.Unmarshal(decrypted, deleteChange) - if err != nil { - return nil, err - } - return deleteChange, nil -} - -func (p *provider) processChange(change *tree.Change, tr tree.ObjectTree, startId string, ids []string) []string { +func (p *provider) processChange(change *tree.Change, rootId, startId string, ids []string) []string { // ignoring root change which has empty model or startId change if change.Model == nil || (change.Id == startId && startId != "") { return ids @@ -29,7 +20,7 @@ func (p *provider) processChange(change *tree.Change, tr tree.ObjectTree, startI deleteChange := change.Model.(*spacesyncproto.SettingsData) // getting data from snapshot if we start from it - if change.Id == tr.Root().Id { + if change.Id == rootId { ids = deleteChange.Snapshot.DeletedIds return ids } @@ -44,15 +35,25 @@ func (p *provider) processChange(change *tree.Change, tr tree.ObjectTree, startI } func (p *provider) ProvideIds(tr tree.ObjectTree, startId string) (ids []string, lastId string, err error) { + rootId := tr.Root().Id process := func(change *tree.Change) bool { lastId = change.Id - ids = p.processChange(change, tr, startId, ids) + ids = p.processChange(change, rootId, startId, ids) return true } + convert := func(decrypted []byte) (res any, err error) { + deleteChange := &spacesyncproto.SettingsData{} + err = proto.Unmarshal(decrypted, deleteChange) + if err != nil { + return nil, err + } + return deleteChange, nil + } + if startId == "" { - err = tr.IterateFrom(tr.ID(), p.convert, process) + err = tr.IterateFrom(tr.ID(), convert, process) } else { - err = tr.IterateFrom(startId, p.convert, process) + err = tr.IterateFrom(startId, convert, process) } return } diff --git a/common/commonspace/settingsdocument/idprovider_test.go b/common/commonspace/settingsdocument/idprovider_test.go new file mode 100644 index 00000000..435fe210 --- /dev/null +++ b/common/commonspace/settingsdocument/idprovider_test.go @@ -0,0 +1,94 @@ +package settingsdocument + +import ( + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto" + "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree" + mock_tree "github.com/anytypeio/go-anytype-infrastructure-experiments/common/pkg/acl/tree/mock_objecttree" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + "testing" +) + +func TestProvider_ProcessChange(t *testing.T) { + //ctrl := gomock.NewController(t) + //objTree := mock_tree.NewMockObjectTree(ctrl) + prov := &provider{} + //defer ctrl.Finish() + + t.Run("empty model", func(t *testing.T) { + ch := &tree.Change{} + startId := "startId" + rootId := "rootId" + ids := []string{startId} + otherIds := prov.processChange(ch, rootId, startId, ids) + require.Equal(t, []string{startId}, otherIds) + }) + + t.Run("changeId is equal to startId", func(t *testing.T) { + ch := &tree.Change{} + ch.Model = &spacesyncproto.SettingsData{} + ch.Id = "startId" + + startId := "startId" + rootId := "rootId" + ids := []string{startId} + otherIds := prov.processChange(ch, rootId, startId, ids) + require.Equal(t, []string{startId}, otherIds) + }) + + t.Run("changeId is equal to rootId, startId is empty", func(t *testing.T) { + ch := &tree.Change{} + ch.Model = &spacesyncproto.SettingsData{ + Snapshot: &spacesyncproto.SpaceSettingsSnapshot{ + DeletedIds: []string{"id1", "id2"}, + }, + } + ch.Id = "rootId" + + startId := "" + rootId := "rootId" + otherIds := prov.processChange(ch, rootId, startId, nil) + require.Equal(t, []string{"id1", "id2"}, otherIds) + }) + + t.Run("changeId is equal to rootId, startId is empty", func(t *testing.T) { + ch := &tree.Change{} + ch.Model = &spacesyncproto.SettingsData{ + Content: []*spacesyncproto.SpaceSettingsContent{ + {&spacesyncproto.SpaceSettingsContent_ObjectDelete{ + ObjectDelete: &spacesyncproto.ObjectDelete{Id: "id1"}, + }}, + }, + } + ch.Id = "someId" + + startId := "startId" + rootId := "rootId" + otherIds := prov.processChange(ch, rootId, startId, nil) + require.Equal(t, []string{"id1"}, otherIds) + }) +} + +func TestProvider_ProvideIds(t *testing.T) { + ctrl := gomock.NewController(t) + objTree := mock_tree.NewMockObjectTree(ctrl) + prov := &provider{} + defer ctrl.Finish() + + t.Run("startId is empty", func(t *testing.T) { + ch := &tree.Change{Id: "rootId"} + objTree.EXPECT().Root().Return(ch) + objTree.EXPECT().ID().Return("id") + objTree.EXPECT().IterateFrom("id", gomock.Any(), gomock.Any()).Return(nil) + _, _, err := prov.ProvideIds(objTree, "") + require.NoError(t, err) + }) + + t.Run("startId is not empty", func(t *testing.T) { + ch := &tree.Change{Id: "rootId"} + objTree.EXPECT().Root().Return(ch) + objTree.EXPECT().IterateFrom("startId", gomock.Any(), gomock.Any()).Return(nil) + _, _, err := prov.ProvideIds(objTree, "startId") + require.NoError(t, err) + }) +}