diff --git a/Makefile b/Makefile index b1a3cc3e..41922b2c 100644 --- a/Makefile +++ b/Makefile @@ -25,12 +25,13 @@ protos-go: @$(eval P_TIMESTAMP := Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types) @$(eval P_STRUCT := Mgoogle/protobuf/struct.proto=github.com/gogo/protobuf/types) @$(eval P_ACL_CHANGES := M$(P_ACL_CHANGES_PATH_PB)/protos/aclchanges.proto=github.com/anytypeio/go-anytype-infrastructure-experiments/$(P_ACL_CHANGES_PATH_PB)) + @$(eval P_TREE_CHANGES := M$(P_TREE_STORAGE_PATH_PB)/protos/tree.proto=github.com/anytypeio/go-anytype-infrastructure-experiments/$(P_TREE_STORAGE_PATH_PB)) # use if needed $(eval PKGMAP := $$(P_TIMESTAMP),$$(P_STRUCT)) $(GOGO_START) protoc --gogofaster_out=:. $(P_ACL_CHANGES_PATH_PB)/protos/*.proto; mv $(P_ACL_CHANGES_PATH_PB)/protos/*.go $(P_ACL_CHANGES_PATH_PB) $(GOGO_START) protoc --gogofaster_out=:. $(P_TREE_STORAGE_PATH_PB)/protos/*.proto; mv $(P_TREE_STORAGE_PATH_PB)/protos/*.go $(P_TREE_STORAGE_PATH_PB) $(GOGO_START) protoc --gogofaster_out=:. $(P_PLAINTEXT_CHANGES_PATH_PB)/protos/*.proto; mv $(P_PLAINTEXT_CHANGES_PATH_PB)/protos/*.go $(P_PLAINTEXT_CHANGES_PATH_PB) - $(eval PKGMAP := $$(P_ACL_CHANGES)) + $(eval PKGMAP := $$(P_ACL_CHANGES),$$(P_TREE_CHANGES)) $(GOGO_START) protoc --gogofaster_out=$(PKGMAP):. $(P_SYNC_CHANGES_PATH_PB)/protos/*.proto; mv $(P_SYNC_CHANGES_PATH_PB)/protos/*.go $(P_SYNC_CHANGES_PATH_PB) build: diff --git a/pkg/acl/acltree/acltree.go b/pkg/acl/acltree/acltree.go index ed1c1a45..4b091433 100644 --- a/pkg/acl/acltree/acltree.go +++ b/pkg/acl/acltree/acltree.go @@ -6,6 +6,7 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb" "sync" ) @@ -47,6 +48,7 @@ var ErrNoCommonSnapshot = errors.New("trees doesn't have a common snapshot") type ACLTree interface { RWLocker ID() string + Header() *treepb.TreeHeader ACLState() *ACLState AddContent(ctx context.Context, f func(builder ChangeBuilder) error) (*Change, error) AddRawChanges(ctx context.Context, changes ...*aclpb.RawChange) (AddResult, error) @@ -67,6 +69,7 @@ type aclTree struct { updateListener TreeUpdateListener id string + header *treepb.TreeHeader fullTree *Tree aclTreeFromStart *Tree // TODO: right now we don't use it, we can probably have only local var for now. This tree is built from start of the document aclState *ACLState @@ -118,6 +121,10 @@ func BuildACLTree( if err != nil { return nil, err } + aclTree.header, err = t.Header() + if err != nil { + return nil, err + } listener.Rebuild(aclTree) @@ -224,6 +231,10 @@ func (a *aclTree) ID() string { return a.id } +func (a *aclTree) Header() *treepb.TreeHeader { + return a.header +} + func (a *aclTree) ACLState() *ACLState { return a.aclState } diff --git a/pkg/acl/acltree/acltreestorage.go b/pkg/acl/acltree/acltreestorage.go index 51d7d86d..df03898a 100644 --- a/pkg/acl/acltree/acltreestorage.go +++ b/pkg/acl/acltree/acltreestorage.go @@ -4,13 +4,16 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb" + "github.com/anytypeio/go-anytype-infrastructure-experiments/util/cid" "github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey" + "github.com/gogo/protobuf/proto" ) -func BuildTreeStorageWithACL( +func CreateNewTreeStorageWithACL( acc *account.AccountData, build func(builder ChangeBuilder) error, - create func(change *aclpb.RawChange) (treestorage.TreeStorage, error)) (treestorage.TreeStorage, error) { + create treestorage.CreatorFunc) (treestorage.TreeStorage, error) { bld := newChangeBuilder() bld.Init( newACLState(acc.Identity, acc.EncKey, signingkey.NewEd25519PubKeyDecoder()), @@ -32,8 +35,12 @@ func BuildTreeStorageWithACL( Signature: change.Signature(), Id: change.CID(), } + header, id, err := createTreeHeaderAndId(rawChange) + if err != nil { + return nil, err + } - thr, err := create(rawChange) + thr, err := create(id, header, []*aclpb.RawChange{rawChange}) if err != nil { return nil, err } @@ -44,3 +51,20 @@ func BuildTreeStorageWithACL( } return thr, nil } + +func createTreeHeaderAndId(change *aclpb.RawChange) (*treepb.TreeHeader, string, error) { + header := &treepb.TreeHeader{ + FirstChangeId: change.Id, + IsWorkspace: false, + } + marshalledHeader, err := proto.Marshal(header) + if err != nil { + return nil, "", err + } + treeId, err := cid.NewCIDFromBytes(marshalledHeader) + if err != nil { + return nil, "", err + } + + return header, treeId, nil +} diff --git a/pkg/acl/acltree/acltreestorage_test.go b/pkg/acl/acltree/acltreestorage_test.go index 674dd700..8730a542 100644 --- a/pkg/acl/acltree/acltreestorage_test.go +++ b/pkg/acl/acltree/acltreestorage_test.go @@ -19,7 +19,7 @@ func Test_BuildTreeStorageWithACL(t *testing.T) { SignKey: keychain.SigningKeys["A"], EncKey: keychain.EncryptionKeys["A"], } - thr, err := BuildTreeStorageWithACL( + thr, err := CreateNewTreeStorageWithACL( data, func(builder ChangeBuilder) error { return builder.UserAdd( diff --git a/pkg/acl/example/plaintextdocument/document.go b/pkg/acl/example/plaintextdocument/document.go index b25b2f8e..d97bf430 100644 --- a/pkg/acl/example/plaintextdocument/document.go +++ b/pkg/acl/example/plaintextdocument/document.go @@ -118,7 +118,7 @@ func NewInMemoryPlainTextDocument(acc *account.AccountData, text string) (PlainT func NewPlainTextDocument( acc *account.AccountData, - create func(change *aclpb.RawChange) (treestorage.TreeStorage, error), + create treestorage.CreatorFunc, text string) (PlainTextDocument, error) { changeBuilder := func(builder acltree.ChangeBuilder) error { err := builder.UserAdd(acc.Identity, acc.EncKey.GetPublic(), aclpb.ACLChange_Admin) @@ -128,7 +128,7 @@ func NewPlainTextDocument( builder.AddChangeContent(createInitialChangeContent(text)) return nil } - t, err := acltree.BuildTreeStorageWithACL( + t, err := acltree.CreateNewTreeStorageWithACL( acc, changeBuilder, create) diff --git a/pkg/acl/treestorage/inmemory.go b/pkg/acl/treestorage/inmemory.go index b9015116..adfa9af8 100644 --- a/pkg/acl/treestorage/inmemory.go +++ b/pkg/acl/treestorage/inmemory.go @@ -6,7 +6,6 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb" - "github.com/anytypeio/go-anytype-infrastructure-experiments/util/cid" "github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice" "github.com/gogo/protobuf/proto" "sync" @@ -22,29 +21,25 @@ type inMemoryTreeStorage struct { sync.RWMutex } -func NewInMemoryTreeStorage(firstChange *aclpb.RawChange) (TreeStorage, error) { - header := &treepb.TreeHeader{ - FirstChangeId: firstChange.Id, - IsWorkspace: false, - } - marshalledHeader, err := proto.Marshal(header) - if err != nil { - return nil, err - } - treeId, err := cid.NewCIDFromBytes(marshalledHeader) - if err != nil { - return nil, err - } +type CreatorFunc = func(string, *treepb.TreeHeader, []*aclpb.RawChange) (TreeStorage, error) - changes := make(map[string]*aclpb.RawChange) - changes[firstChange.Id] = firstChange +func NewInMemoryTreeStorage( + treeId string, + header *treepb.TreeHeader, + changes []*aclpb.RawChange) (TreeStorage, error) { + allChanges := make(map[string]*aclpb.RawChange) + var orphans []string + for _, ch := range changes { + allChanges[ch.Id] = ch + orphans = append(orphans, ch.Id) + } return &inMemoryTreeStorage{ id: treeId, header: header, - heads: []string{firstChange.Id}, - orphans: nil, - changes: changes, + heads: nil, + orphans: orphans, + changes: allChanges, RWMutex: sync.RWMutex{}, }, nil } @@ -137,27 +132,28 @@ func (t *inMemoryTreeStorage) GetChange(ctx context.Context, changeId string) (* type inMemoryTreeStorageProvider struct { trees map[string]TreeStorage + sync.RWMutex } func (i *inMemoryTreeStorageProvider) TreeStorage(treeId string) (TreeStorage, error) { + i.RLock() + defer i.RUnlock() if tree, exists := i.trees[treeId]; exists { return tree, nil } return nil, ErrUnknownTreeId } -func (i *inMemoryTreeStorageProvider) InsertTree(tree TreeStorage) error { - if tree == nil { - return fmt.Errorf("tree should not be nil") - } - - id, err := tree.TreeID() +func (i *inMemoryTreeStorageProvider) CreateTreeStorage(treeId string, header *treepb.TreeHeader, changes []*aclpb.RawChange) (TreeStorage, error) { + i.Lock() + defer i.Unlock() + res, err := NewInMemoryTreeStorage(treeId, header, changes) if err != nil { - return err + return nil, err } - i.trees[id] = tree - return nil + i.trees[treeId] = res + return res, nil } func NewInMemoryTreeStorageProvider() Provider { diff --git a/pkg/acl/treestorage/provider.go b/pkg/acl/treestorage/provider.go index fd442655..41b472e9 100644 --- a/pkg/acl/treestorage/provider.go +++ b/pkg/acl/treestorage/provider.go @@ -1,10 +1,14 @@ package treestorage -import "errors" +import ( + "errors" + "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb" + "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb" +) var ErrUnknownTreeId = errors.New("tree does not exist") type Provider interface { TreeStorage(treeId string) (TreeStorage, error) - InsertTree(tree TreeStorage) error + CreateTreeStorage(treeId string, header *treepb.TreeHeader, changes []*aclpb.RawChange) (TreeStorage, error) } diff --git a/pkg/acl/treestorage/treepb/protos/tree.proto b/pkg/acl/treestorage/treepb/protos/tree.proto index 2b2a707f..eb342673 100644 --- a/pkg/acl/treestorage/treepb/protos/tree.proto +++ b/pkg/acl/treestorage/treepb/protos/tree.proto @@ -1,5 +1,5 @@ syntax = "proto3"; -package anytype; +package tree; option go_package = "treepb"; message TreeHeader { diff --git a/pkg/acl/treestorage/treepb/tree.pb.go b/pkg/acl/treestorage/treepb/tree.pb.go index 32ce731f..7baa874c 100644 --- a/pkg/acl/treestorage/treepb/tree.pb.go +++ b/pkg/acl/treestorage/treepb/tree.pb.go @@ -75,7 +75,7 @@ func (m *TreeHeader) GetIsWorkspace() bool { } func init() { - proto.RegisterType((*TreeHeader)(nil), "anytype.TreeHeader") + proto.RegisterType((*TreeHeader)(nil), "tree.TreeHeader") } func init() { @@ -83,18 +83,18 @@ func init() { } var fileDescriptor_e7d760b855878644 = []byte{ - // 170 bytes of a gzipped FileDescriptorProto + // 165 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0x29, 0xc8, 0x4e, 0xd7, 0x4f, 0x4c, 0xce, 0xd1, 0x2f, 0x29, 0x4a, 0x4d, 0x2d, 0x2e, 0xc9, 0x2f, 0x4a, 0x4c, 0x4f, 0x05, 0xb3, 0x0b, 0x92, 0xf4, 0x0b, 0x8a, 0xf2, 0x4b, 0xf2, 0x8b, 0xc1, 0x3c, 0x3d, 0x30, 0x5b, 0x88, - 0x3d, 0x31, 0xaf, 0xb2, 0xa4, 0xb2, 0x20, 0x55, 0x29, 0x84, 0x8b, 0x2b, 0xa4, 0x28, 0x35, 0xd5, - 0x23, 0x35, 0x31, 0x25, 0xb5, 0x48, 0x48, 0x85, 0x8b, 0x37, 0x2d, 0xb3, 0xa8, 0xb8, 0xc4, 0x39, - 0x23, 0x31, 0x2f, 0x3d, 0xd5, 0x33, 0x45, 0x82, 0x51, 0x81, 0x51, 0x83, 0x33, 0x08, 0x55, 0x50, - 0x48, 0x81, 0x8b, 0x3b, 0xb3, 0x38, 0x3c, 0xbf, 0x28, 0xbb, 0xb8, 0x20, 0x31, 0x39, 0x55, 0x82, - 0x49, 0x81, 0x51, 0x83, 0x23, 0x08, 0x59, 0xc8, 0x49, 0xe1, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x8f, - 0xe4, 0x18, 0x1f, 0x3c, 0x92, 0x63, 0x9c, 0xf0, 0x58, 0x8e, 0xe1, 0xc2, 0x63, 0x39, 0x86, 0x1b, - 0x8f, 0xe5, 0x18, 0xa2, 0xd8, 0x20, 0x4e, 0x4a, 0x62, 0x03, 0xbb, 0xc3, 0x18, 0x10, 0x00, 0x00, - 0xff, 0xff, 0xb1, 0x3f, 0x66, 0x17, 0xb7, 0x00, 0x00, 0x00, + 0x05, 0xc4, 0x56, 0x0a, 0xe1, 0xe2, 0x0a, 0x29, 0x4a, 0x4d, 0xf5, 0x48, 0x4d, 0x4c, 0x49, 0x2d, + 0x12, 0x52, 0xe1, 0xe2, 0x4d, 0xcb, 0x2c, 0x2a, 0x2e, 0x71, 0xce, 0x48, 0xcc, 0x4b, 0x4f, 0xf5, + 0x4c, 0x91, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x42, 0x15, 0x14, 0x52, 0xe0, 0xe2, 0xce, 0x2c, + 0x0e, 0xcf, 0x2f, 0xca, 0x2e, 0x2e, 0x48, 0x4c, 0x4e, 0x95, 0x60, 0x52, 0x60, 0xd4, 0xe0, 0x08, + 0x42, 0x16, 0x72, 0x52, 0x38, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, + 0x18, 0x27, 0x3c, 0x96, 0x63, 0xb8, 0xf0, 0x58, 0x8e, 0xe1, 0xc6, 0x63, 0x39, 0x86, 0x28, 0x36, + 0x88, 0x7b, 0x92, 0xd8, 0xc0, 0x8e, 0x30, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0x09, 0x4f, 0xc6, + 0xec, 0xb4, 0x00, 0x00, 0x00, } func (m *TreeHeader) Marshal() (dAtA []byte, err error) { diff --git a/service/sync/requesthandler/requesthandler.go b/service/sync/requesthandler/requesthandler.go index 491f3c4b..7bdaf488 100644 --- a/service/sync/requesthandler/requesthandler.go +++ b/service/sync/requesthandler/requesthandler.go @@ -6,6 +6,7 @@ import ( "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/acltree" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb" "github.com/anytypeio/go-anytype-infrastructure-experiments/service/account" "github.com/anytypeio/go-anytype-infrastructure-experiments/service/sync/client" "github.com/anytypeio/go-anytype-infrastructure-experiments/service/sync/syncpb" @@ -66,7 +67,7 @@ func (r *requestHandler) HandleHeadUpdate(ctx context.Context, senderId string, shouldFullSync := !slice.UnsortedEquals(update.Heads, tree.Heads()) snapshotPath = tree.SnapshotPath() if shouldFullSync { - fullRequest, err = r.prepareFullSyncRequest(update.TreeId, update.SnapshotPath, tree) + fullRequest, err = r.prepareFullSyncRequest(update.TreeId, update.TreeHeader, update.SnapshotPath, tree) if err != nil { return err } @@ -75,8 +76,10 @@ func (r *requestHandler) HandleHeadUpdate(ctx context.Context, senderId string, }) // if there are no such tree if err == treestorage.ErrUnknownTreeId { + // TODO: maybe we can optimize this by sending the header and stuff right away, so when the tree is created we are able to add it on first request fullRequest = &syncpb.SyncFullRequest{ - TreeId: update.TreeId, + TreeId: update.TreeId, + TreeHeader: update.TreeHeader, } } // if we have incompatible heads, or we haven't seen the tree at all @@ -93,6 +96,7 @@ func (r *requestHandler) HandleHeadUpdate(ctx context.Context, senderId string, Changes: result.Added, SnapshotPath: snapshotPath, TreeId: update.TreeId, + TreeHeader: update.TreeHeader, } return r.client.NotifyHeadsChanged(newUpdate) } @@ -106,7 +110,7 @@ func (r *requestHandler) HandleFullSyncRequest(ctx context.Context, senderId str err = r.treeCache.Do(ctx, request.TreeId, func(tree acltree.ACLTree) error { // TODO: check if we already have those changes - // if we have non empty request + // if we have non-empty request if len(request.Heads) != 0 { result, err = tree.AddRawChanges(ctx, request.Changes...) if err != nil { @@ -135,6 +139,7 @@ func (r *requestHandler) HandleFullSyncRequest(ctx context.Context, senderId str Changes: result.Added, SnapshotPath: snapshotPath, TreeId: request.TreeId, + TreeHeader: request.TreeHeader, } return r.client.NotifyHeadsChanged(newUpdate) } @@ -154,17 +159,18 @@ func (r *requestHandler) HandleFullSyncResponse(ctx context.Context, senderId st snapshotPath = tree.SnapshotPath() return nil }) - if err == treestorage.ErrUnknownTreeId { - // TODO: probably sometimes we should notify about this (e.g. if client created new document) - return r.createTree(response) - } // if error or nothing has changed - if err != nil || len(result.Added) == 0 { + if (err != nil || len(result.Added) == 0) && err != treestorage.ErrUnknownTreeId { return err } - - // TODO: probably here we should not send an update message, because the other node had already sent it after updating with our data - // otherwise sending heads update message + // if we have a new tree + if err == treestorage.ErrUnknownTreeId { + err = r.createTree(ctx, response) + if err != nil { + return err + } + } + // sending heads update message newUpdate := &syncpb.SyncHeadUpdate{ Heads: result.Heads, Changes: result.Added, @@ -174,7 +180,7 @@ func (r *requestHandler) HandleFullSyncResponse(ctx context.Context, senderId st return r.client.NotifyHeadsChanged(newUpdate) } -func (r *requestHandler) prepareFullSyncRequest(treeId string, theirPath []string, tree acltree.ACLTree) (*syncpb.SyncFullRequest, error) { +func (r *requestHandler) prepareFullSyncRequest(treeId string, header *treepb.TreeHeader, theirPath []string, tree acltree.ACLTree) (*syncpb.SyncFullRequest, error) { ourChanges, err := tree.ChangesAfterCommonSnapshot(theirPath) if err != nil { return nil, err @@ -184,10 +190,15 @@ func (r *requestHandler) prepareFullSyncRequest(treeId string, theirPath []strin Changes: ourChanges, TreeId: treeId, SnapshotPath: tree.SnapshotPath(), + TreeHeader: header, }, nil } -func (r *requestHandler) prepareFullSyncResponse(treeId string, theirPath []string, theirChanges []*aclpb.RawChange, tree acltree.ACLTree) (*syncpb.SyncFullResponse, error) { +func (r *requestHandler) prepareFullSyncResponse( + treeId string, + theirPath []string, + theirChanges []*aclpb.RawChange, + tree acltree.ACLTree) (*syncpb.SyncFullResponse, error) { // TODO: we can probably use the common snapshot calculated on the request step from previous peer ourChanges, err := tree.ChangesAfterCommonSnapshot(theirPath) if err != nil { @@ -211,10 +222,10 @@ func (r *requestHandler) prepareFullSyncResponse(treeId string, theirPath []stri Changes: final, TreeId: treeId, SnapshotPath: tree.SnapshotPath(), + TreeHeader: tree.Header(), }, nil } -func (r *requestHandler) createTree(response *syncpb.SyncFullResponse) error { - // TODO: write create tree functionality - return nil +func (r *requestHandler) createTree(ctx context.Context, response *syncpb.SyncFullResponse) error { + return r.treeCache.Add(ctx, response.TreeId, response.TreeHeader, response.Changes) } diff --git a/service/sync/syncpb/protos/sync.proto b/service/sync/syncpb/protos/sync.proto index b67e18a1..efdc1f7c 100644 --- a/service/sync/syncpb/protos/sync.proto +++ b/service/sync/syncpb/protos/sync.proto @@ -1,8 +1,9 @@ syntax = "proto3"; -package anytype; +package sync; option go_package = "syncpb"; import "pkg/acl/aclchanges/aclpb/protos/aclchanges.proto"; +import "pkg/acl/treestorage/treepb/protos/tree.proto"; message Sync { message HeadUpdate { @@ -10,6 +11,7 @@ message Sync { repeated acl.RawChange changes = 2; string treeId = 3; repeated string snapshotPath = 4; + tree.TreeHeader treeHeader = 5; } message Full { @@ -19,6 +21,7 @@ message Sync { repeated acl.RawChange changes = 2; string treeId = 3; repeated string snapshotPath = 4; + tree.TreeHeader treeHeader = 5; } message Response { @@ -26,6 +29,7 @@ message Sync { repeated acl.RawChange changes = 2; string treeId = 3; repeated string snapshotPath = 4; + tree.TreeHeader treeHeader = 5; } } } \ No newline at end of file diff --git a/service/sync/syncpb/sync.pb.go b/service/sync/syncpb/sync.pb.go index f06b2d27..1cd6afb0 100644 --- a/service/sync/syncpb/sync.pb.go +++ b/service/sync/syncpb/sync.pb.go @@ -6,6 +6,7 @@ package syncpb import ( fmt "fmt" aclpb "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb" + treepb "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb" proto "github.com/gogo/protobuf/proto" io "io" math "math" @@ -64,6 +65,7 @@ type SyncHeadUpdate struct { Changes []*aclpb.RawChange `protobuf:"bytes,2,rep,name=changes,proto3" json:"changes,omitempty"` TreeId string `protobuf:"bytes,3,opt,name=treeId,proto3" json:"treeId,omitempty"` SnapshotPath []string `protobuf:"bytes,4,rep,name=snapshotPath,proto3" json:"snapshotPath,omitempty"` + TreeHeader *treepb.TreeHeader `protobuf:"bytes,5,opt,name=treeHeader,proto3" json:"treeHeader,omitempty"` } func (m *SyncHeadUpdate) Reset() { *m = SyncHeadUpdate{} } @@ -127,6 +129,13 @@ func (m *SyncHeadUpdate) GetSnapshotPath() []string { return nil } +func (m *SyncHeadUpdate) GetTreeHeader() *treepb.TreeHeader { + if m != nil { + return m.TreeHeader + } + return nil +} + type SyncFull struct { } @@ -169,6 +178,7 @@ type SyncFullRequest struct { Changes []*aclpb.RawChange `protobuf:"bytes,2,rep,name=changes,proto3" json:"changes,omitempty"` TreeId string `protobuf:"bytes,3,opt,name=treeId,proto3" json:"treeId,omitempty"` SnapshotPath []string `protobuf:"bytes,4,rep,name=snapshotPath,proto3" json:"snapshotPath,omitempty"` + TreeHeader *treepb.TreeHeader `protobuf:"bytes,5,opt,name=treeHeader,proto3" json:"treeHeader,omitempty"` } func (m *SyncFullRequest) Reset() { *m = SyncFullRequest{} } @@ -232,11 +242,19 @@ func (m *SyncFullRequest) GetSnapshotPath() []string { return nil } +func (m *SyncFullRequest) GetTreeHeader() *treepb.TreeHeader { + if m != nil { + return m.TreeHeader + } + return nil +} + type SyncFullResponse struct { Heads []string `protobuf:"bytes,1,rep,name=heads,proto3" json:"heads,omitempty"` Changes []*aclpb.RawChange `protobuf:"bytes,2,rep,name=changes,proto3" json:"changes,omitempty"` TreeId string `protobuf:"bytes,3,opt,name=treeId,proto3" json:"treeId,omitempty"` SnapshotPath []string `protobuf:"bytes,4,rep,name=snapshotPath,proto3" json:"snapshotPath,omitempty"` + TreeHeader *treepb.TreeHeader `protobuf:"bytes,5,opt,name=treeHeader,proto3" json:"treeHeader,omitempty"` } func (m *SyncFullResponse) Reset() { *m = SyncFullResponse{} } @@ -300,12 +318,19 @@ func (m *SyncFullResponse) GetSnapshotPath() []string { return nil } +func (m *SyncFullResponse) GetTreeHeader() *treepb.TreeHeader { + if m != nil { + return m.TreeHeader + } + return nil +} + func init() { - proto.RegisterType((*Sync)(nil), "anytype.Sync") - proto.RegisterType((*SyncHeadUpdate)(nil), "anytype.Sync.HeadUpdate") - proto.RegisterType((*SyncFull)(nil), "anytype.Sync.Full") - proto.RegisterType((*SyncFullRequest)(nil), "anytype.Sync.Full.Request") - proto.RegisterType((*SyncFullResponse)(nil), "anytype.Sync.Full.Response") + proto.RegisterType((*Sync)(nil), "sync.Sync") + proto.RegisterType((*SyncHeadUpdate)(nil), "sync.Sync.HeadUpdate") + proto.RegisterType((*SyncFull)(nil), "sync.Sync.Full") + proto.RegisterType((*SyncFullRequest)(nil), "sync.Sync.Full.Request") + proto.RegisterType((*SyncFullResponse)(nil), "sync.Sync.Full.Response") } func init() { @@ -313,25 +338,27 @@ func init() { } var fileDescriptor_5f66cdd599c6466f = []byte{ - // 273 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x2d, 0x4e, 0x2d, 0x2a, - 0xcb, 0x4c, 0x4e, 0xd5, 0x2f, 0xae, 0xcc, 0x4b, 0x06, 0x13, 0x05, 0x49, 0xfa, 0x05, 0x45, 0xf9, - 0x25, 0xf9, 0xc5, 0x60, 0x9e, 0x1e, 0x98, 0x2d, 0xc4, 0x9e, 0x98, 0x57, 0x59, 0x52, 0x59, 0x90, - 0x2a, 0x65, 0x50, 0x90, 0x9d, 0xae, 0x9f, 0x98, 0x9c, 0x03, 0xc2, 0xc9, 0x19, 0x89, 0x79, 0xe9, - 0xa9, 0xc5, 0x20, 0x26, 0x42, 0x13, 0x42, 0x1c, 0xa2, 0x55, 0x69, 0x35, 0x33, 0x17, 0x4b, 0x70, - 0x65, 0x5e, 0xb2, 0x54, 0x07, 0x23, 0x17, 0x97, 0x47, 0x6a, 0x62, 0x4a, 0x68, 0x41, 0x4a, 0x62, - 0x49, 0xaa, 0x90, 0x08, 0x17, 0x6b, 0x46, 0x6a, 0x62, 0x4a, 0xb1, 0x04, 0xa3, 0x02, 0xb3, 0x06, - 0x67, 0x10, 0x84, 0x23, 0xa4, 0xc1, 0xc5, 0x0e, 0xd5, 0x2e, 0xc1, 0xa4, 0xc0, 0xac, 0xc1, 0x6d, - 0xc4, 0xa7, 0x97, 0x98, 0x9c, 0xa3, 0x17, 0x94, 0x58, 0xee, 0x0c, 0x16, 0x0e, 0x82, 0x49, 0x0b, - 0x89, 0x71, 0xb1, 0x95, 0x14, 0xa5, 0xa6, 0x7a, 0xa6, 0x48, 0x30, 0x2b, 0x30, 0x6a, 0x70, 0x06, - 0x41, 0x79, 0x42, 0x4a, 0x5c, 0x3c, 0xc5, 0x79, 0x89, 0x05, 0xc5, 0x19, 0xf9, 0x25, 0x01, 0x89, - 0x25, 0x19, 0x12, 0x2c, 0x60, 0xe3, 0x51, 0xc4, 0xa4, 0xa6, 0x33, 0x71, 0xb1, 0xb8, 0x95, 0xe6, - 0xe4, 0x48, 0xb5, 0x32, 0x72, 0xb1, 0x07, 0xa5, 0x16, 0x96, 0xa6, 0x16, 0x97, 0x0c, 0xa8, 0x83, - 0xda, 0x18, 0xb9, 0x38, 0x82, 0x52, 0x8b, 0x0b, 0xf2, 0xf3, 0x8a, 0x07, 0x34, 0x64, 0x9c, 0x14, - 0x4e, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, 0x8e, 0xf1, 0xc1, 0x23, 0x39, 0xc6, 0x09, 0x8f, 0xe5, - 0x18, 0x2e, 0x3c, 0x96, 0x63, 0xb8, 0xf1, 0x58, 0x8e, 0x21, 0x8a, 0x0d, 0x92, 0x38, 0x92, 0xd8, - 0xc0, 0xd1, 0x6a, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0x90, 0x29, 0xbc, 0x62, 0x3a, 0x02, 0x00, - 0x00, + // 314 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x52, 0xc1, 0x4a, 0xc3, 0x30, + 0x18, 0x5e, 0xb6, 0x6e, 0x73, 0xff, 0x44, 0x24, 0x88, 0x94, 0x1e, 0x42, 0x19, 0x08, 0x3d, 0x48, + 0x37, 0xe6, 0x1b, 0x28, 0x88, 0xde, 0x24, 0xea, 0xc5, 0x5b, 0xd6, 0xfe, 0xac, 0x62, 0x69, 0x63, + 0x93, 0x29, 0xbe, 0x85, 0x6f, 0xa3, 0x78, 0xf3, 0xe6, 0x71, 0x47, 0x8f, 0xd2, 0x3e, 0x86, 0x17, + 0x49, 0xba, 0x59, 0x7d, 0x84, 0x1d, 0x12, 0xbe, 0xef, 0xcb, 0xff, 0xe5, 0xfb, 0x13, 0x7e, 0x38, + 0x50, 0x58, 0x3c, 0xdc, 0x46, 0x38, 0x56, 0x4f, 0x59, 0x64, 0x37, 0x39, 0x1b, 0xcb, 0x22, 0xd7, + 0xb9, 0xb2, 0x2c, 0xb4, 0x98, 0x3a, 0x06, 0x7b, 0x13, 0x79, 0x37, 0x1f, 0x8b, 0x28, 0x35, 0x2b, + 0x4a, 0x44, 0x36, 0x47, 0x65, 0x60, 0xe3, 0x68, 0xf4, 0xda, 0xe7, 0x1d, 0xae, 0x1d, 0xba, 0x40, + 0x54, 0x3a, 0x2f, 0xc4, 0x1c, 0x2d, 0x6e, 0x3c, 0x86, 0xd5, 0xd5, 0xa3, 0x77, 0x07, 0x9c, 0x4b, + 0x13, 0xf4, 0x46, 0x00, 0xce, 0x50, 0xc4, 0xd7, 0x32, 0x16, 0x1a, 0xe9, 0x1e, 0x74, 0x13, 0x14, + 0xb1, 0x72, 0x89, 0xdf, 0x09, 0x06, 0xbc, 0x26, 0x34, 0x80, 0xfe, 0x2a, 0xcc, 0x6d, 0xfb, 0x9d, + 0x60, 0x38, 0xdd, 0x09, 0x45, 0x94, 0x86, 0x5c, 0x3c, 0x9e, 0x58, 0x99, 0xaf, 0x8f, 0xe9, 0x3e, + 0xf4, 0x4c, 0xca, 0x79, 0xec, 0x76, 0x7c, 0x12, 0x0c, 0xf8, 0x8a, 0xd1, 0x11, 0x6c, 0xab, 0x4c, + 0x48, 0x95, 0xe4, 0xfa, 0x42, 0xe8, 0xc4, 0x75, 0xec, 0xf5, 0xff, 0x34, 0x3a, 0x01, 0x30, 0xd5, + 0xa6, 0x1b, 0x2c, 0xdc, 0xae, 0x4f, 0x82, 0xe1, 0x74, 0x37, 0xb4, 0x4d, 0x5f, 0xfd, 0xea, 0xfc, + 0x4f, 0x8d, 0xf7, 0xdd, 0x06, 0xe7, 0x74, 0x91, 0xa6, 0xde, 0x0b, 0x81, 0x3e, 0xc7, 0xfb, 0x05, + 0x2a, 0xbd, 0x61, 0x4f, 0x78, 0x25, 0xb0, 0xc5, 0x51, 0xc9, 0x3c, 0x53, 0x1b, 0xf6, 0xfb, 0xc7, + 0xfe, 0x47, 0xc9, 0xc8, 0xb2, 0x64, 0xe4, 0xab, 0x64, 0xe4, 0xb9, 0x62, 0xad, 0x65, 0xc5, 0x5a, + 0x9f, 0x15, 0x6b, 0xdd, 0xf4, 0xea, 0xe9, 0x9e, 0xf5, 0xec, 0xb0, 0x1d, 0xfd, 0x04, 0x00, 0x00, + 0xff, 0xff, 0x52, 0xcd, 0x15, 0xea, 0xfb, 0x02, 0x00, 0x00, } func (m *Sync) Marshal() (dAtA []byte, err error) { @@ -377,6 +404,18 @@ func (m *SyncHeadUpdate) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.TreeHeader != nil { + { + size, err := m.TreeHeader.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintSync(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a + } if len(m.SnapshotPath) > 0 { for iNdEx := len(m.SnapshotPath) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.SnapshotPath[iNdEx]) @@ -462,6 +501,18 @@ func (m *SyncFullRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.TreeHeader != nil { + { + size, err := m.TreeHeader.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintSync(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a + } if len(m.SnapshotPath) > 0 { for iNdEx := len(m.SnapshotPath) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.SnapshotPath[iNdEx]) @@ -524,6 +575,18 @@ func (m *SyncFullResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.TreeHeader != nil { + { + size, err := m.TreeHeader.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintSync(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a + } if len(m.SnapshotPath) > 0 { for iNdEx := len(m.SnapshotPath) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.SnapshotPath[iNdEx]) @@ -614,6 +677,10 @@ func (m *SyncHeadUpdate) Size() (n int) { n += 1 + l + sovSync(uint64(l)) } } + if m.TreeHeader != nil { + l = m.TreeHeader.Size() + n += 1 + l + sovSync(uint64(l)) + } return n } @@ -654,6 +721,10 @@ func (m *SyncFullRequest) Size() (n int) { n += 1 + l + sovSync(uint64(l)) } } + if m.TreeHeader != nil { + l = m.TreeHeader.Size() + n += 1 + l + sovSync(uint64(l)) + } return n } @@ -685,6 +756,10 @@ func (m *SyncFullResponse) Size() (n int) { n += 1 + l + sovSync(uint64(l)) } } + if m.TreeHeader != nil { + l = m.TreeHeader.Size() + n += 1 + l + sovSync(uint64(l)) + } return n } @@ -903,6 +978,42 @@ func (m *SyncHeadUpdate) Unmarshal(dAtA []byte) error { } m.SnapshotPath = append(m.SnapshotPath, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TreeHeader", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthSync + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthSync + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.TreeHeader == nil { + m.TreeHeader = &treepb.TreeHeader{} + } + if err := m.TreeHeader.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipSync(dAtA[iNdEx:]) @@ -1133,6 +1244,42 @@ func (m *SyncFullRequest) Unmarshal(dAtA []byte) error { } m.SnapshotPath = append(m.SnapshotPath, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TreeHeader", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthSync + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthSync + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.TreeHeader == nil { + m.TreeHeader = &treepb.TreeHeader{} + } + if err := m.TreeHeader.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipSync(dAtA[iNdEx:]) @@ -1313,6 +1460,42 @@ func (m *SyncFullResponse) Unmarshal(dAtA []byte) error { } m.SnapshotPath = append(m.SnapshotPath, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TreeHeader", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSync + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthSync + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthSync + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.TreeHeader == nil { + m.TreeHeader = &treepb.TreeHeader{} + } + if err := m.TreeHeader.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipSync(dAtA[iNdEx:]) diff --git a/service/treecache/service.go b/service/treecache/service.go index cd3cf884..82460917 100644 --- a/service/treecache/service.go +++ b/service/treecache/service.go @@ -3,8 +3,10 @@ package treecache import ( "context" "github.com/anytypeio/go-anytype-infrastructure-experiments/app" + "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/acltree" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage" + "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/ocache" "github.com/anytypeio/go-anytype-infrastructure-experiments/service/account" ) @@ -13,7 +15,7 @@ const CName = "treecache" type Service interface { Do(ctx context.Context, treeId string, f func(tree acltree.ACLTree) error) error - Add(ctx context.Context, treeId string, tree acltree.ACLTree) error + Add(ctx context.Context, treeId string, header *treepb.TreeHeader, changes []*aclpb.RawChange) error } type service struct { @@ -38,13 +40,20 @@ func (s *service) Do(ctx context.Context, treeId string, f func(tree acltree.ACL return f(tree.(acltree.ACLTree)) } -func (s *service) Add(ctx context.Context, treeId string, tree acltree.ACLTree) error { - return s.cache.Add(treeId, tree) +func (s *service) Add(ctx context.Context, treeId string, header *treepb.TreeHeader, changes []*aclpb.RawChange) error { + _, err := s.treeProvider.CreateTreeStorage(treeId, header, changes) + if err != nil { + return err + } + // forcing the tree to build + _, err = s.cache.Get(ctx, treeId) + return err } func (s *service) Init(ctx context.Context, a *app.App) (err error) { s.cache = ocache.New(s.loadTree) s.account = a.MustComponent(account.CName).(account.Service) + s.treeProvider = treestorage.NewInMemoryTreeStorageProvider() // TODO: for test we should load some predefined keys return nil }