benjamyn/any-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #7 from anytypeio/new-tree-structure

Browse Source
This commit is contained in:
Mikhail Rakhmanov 2022-09-08 08:09:00 +02:00 committed by GitHub
commit 2e47b23827
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
85 changed files with 6498 additions and 5316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Makefile
View File

@ -13,25 +13,22 @@ endif
export PATH=$(GOPATH)/bin:$(shell echo $$PATH)
# TODO: folders were changed, so we should update Makefile and protos generation
protos-go:
proto:
@echo 'Generating protobuf packages (Go)...'
# Uncomment if needed
@$(eval ROOT_PKG := pkg)
@$(eval GOGO_START := GOGO_NO_UNDERSCORE=1 GOGO_EXPORT_ONEOF_INTERFACE=1)
@$(eval P_TREE_STORAGE_PATH_PB := $(ROOT_PKG)/acl/treestorage/treepb)
@$(eval P_ACL_CHANGES_PATH_PB := $(ROOT_PKG)/acl/aclchanges/aclpb)
@$(eval P_PLAINTEXT_CHANGES_PATH_PB := $(ROOT_PKG)/acl/testutils/testchanges/testchangepb)
@$(eval P_SYNC_CHANGES_PATH_PB := syncproto)
@$(eval P_TEST_CHANGES_PATH_PB := $(ROOT_PKG)/acl/testutils/testchanges)
@$(eval P_TIMESTAMP := Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types)
@$(eval P_STRUCT := Mgoogle/protobuf/struct.proto=github.com/gogo/protobuf/types)
@$(eval P_ACL_CHANGES := M$(P_ACL_CHANGES_PATH_PB)/protos/aclchanges.proto=github.com/anytypeio/go-anytype-infrastructure-experiments/$(P_ACL_CHANGES_PATH_PB))
@$(eval P_TREE_CHANGES := M$(P_TREE_STORAGE_PATH_PB)/protos/tree.proto=github.com/anytypeio/go-anytype-infrastructure-experiments/$(P_TREE_STORAGE_PATH_PB))
# use if needed $(eval PKGMAP := $$(P_TIMESTAMP),$$(P_STRUCT))
$(GOGO_START) protoc --gogofaster_out=:. $(P_ACL_CHANGES_PATH_PB)/protos/*.proto; mv $(P_ACL_CHANGES_PATH_PB)/protos/*.go $(P_ACL_CHANGES_PATH_PB)
$(GOGO_START) protoc --gogofaster_out=:. $(P_TREE_STORAGE_PATH_PB)/protos/*.proto; mv $(P_TREE_STORAGE_PATH_PB)/protos/*.go $(P_TREE_STORAGE_PATH_PB)
$(GOGO_START) protoc --gogofaster_out=:. $(P_PLAINTEXT_CHANGES_PATH_PB)/protos/*.proto; mv $(P_PLAINTEXT_CHANGES_PATH_PB)/protos/*.go $(P_PLAINTEXT_CHANGES_PATH_PB)
$(eval PKGMAP := $$(P_ACL_CHANGES),$$(P_TREE_CHANGES))
$(GOGO_START) protoc --gogofaster_out=:. $(P_TEST_CHANGES_PATH_PB)/proto/*.proto
$(eval PKGMAP := $$(P_ACL_CHANGES))
$(GOGO_START) protoc --gogofaster_out=$(PKGMAP):. $(P_SYNC_CHANGES_PATH_PB)/proto/*.proto
$(GOGO_START) protoc --gogofaster_out=$(PKGMAP):. service/space/spacesync/protos/*.proto

4
cmd/node/node.go
View File

@ -10,12 +10,13 @@ import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/api"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/configuration"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/document"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/net/dialer"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/net/pool"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/net/rpc/server"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/net/secure"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/node"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/sync/document"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/storage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/sync/message"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/sync/requesthandler"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/treecache"
@ -98,6 +99,7 @@ func Bootstrap(a *app.App) {
Register(server.New()).
Register(dialer.New()).
Register(pool.NewPool()).
Register(storage.New()).
Register(configuration.New()).
Register(document.New()).
Register(message.New()).

28
etc/acl.yml Normal file
View File

@ -0,0 +1,28 @@
records:
- identity: A
aclChanges:
- userAdd:
identity: A
permission: admin
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
- userAdd:
identity: B
permission: admin
encryptionKey: key.Enc.B
encryptedReadKeys: [key.Read.1]
readKey: key.Read.1
keys:
Enc:
- name: A
value: JgG4CcCbae1qEpe7mKpBzsHjZhXUmDSNVNX2B1gxFZsJyMX4V6kBQUott9zRWyeXaW1ZmpzuxDXnwSQpAnNurhXyGa9iQaAPqzY9A9VWBPD33Yy1eW7TRuVemzToh8jJQKQKnZNbF8ucTWV9qahusKzyvN8uyhrqoW2tAPfA9S3E3ognCuqbLSW6yjE2rBKayvyS1BVwzjSd6FZK4DDyjfU3pbEVjut3wytGEAn9af6sNMmyCnf2MX5vLovWs9rU8av61wD4z7HTsXyGFx4K75N4Go249Hpe9SKAT6HxhRc3yvj63krPLiQV5yMuH2UeMUXBDekUQyNmBEdn9wrur7mLqB67Bc6tcc2PP8XApBCdWJHvHjN4FktSpaG5vbCqoZbLD1oCbk36q2x9s6XM8pydVqD1J9P3nTbfgMb5pJCTFjNtgKeuKv6wjfJeA9jF1VhcJQisfsahgv9MvZ9M8FJpZTq1zKUhYDCRnZxUkraoMS5yNNVdDzaUckKEDthqik7BMWCWT79vq7uVgMwEvGwGi76gtoMg1159bbPMLZ4bdPVfhH2S9QjPrzQfwZSrzB2YeVPjWpaXDeLDity5H8n1NK2oniAQR6gE71n81neSptsuhV6o6QpQ89AU8y57XmEsou4VEryn8vUxBHhULLxrLNUouxyWamCeFiDjk5cSN6koQsf9BYKSNTPFTrwjTKForDokMhcPdMtFktKwjv7u9UEGcY4MKvNzZZkc77gHiP8bqVtdNNoLpTFUC5SZ9i7bKdHvK12HpSy7yzzPeMXJ9UwhLxkok1g81ngTbN1yxRhvYXyHZFtguCR9kvGojDjka91MTBtk551qDw9eCn2xZT9U8jqzBCjdpvSg3mRWKMPnYAGB7m7u1ye165wyGFvzcHAx3vtXjxAqLUeKYZCjv2m6V9D2Y4qH1TQNddWqH14T1JVMis971UCH9Ddpj6a3387oUnufD1P6HZN2ieJCvptrmbGVvxJYYSvmVf1dkwbtqurDRNWD7TJ7gf6iqSP549C9bxP4GpLt3ygjHmMtcuUzstBuztvunJUnQhfnJxqU6LjRdsFzm53wGWgXNxab7ZvQcPyLwsevn1b98FGPnVpS5iY4LjmqW4ugrC6HgrbsjrXiKzR1yZKhLQkCbLzPoaHb8iB5iBnCr7d4yf5CtfpFRqgoqMFdK5LNZYmDX4HzUKN6A7wC3gGiSRFTLcgGZeSMkB5Pa61CZBU7WCQgFxykycE9HRA7PiQa496GWDCV15teToCpFRsAa6jDmR1MGXPeLRqQgve49VXnQN5FL7c1VuEv5SWjeTuCnMB47DJKBaP7eKJNKgLwETALzSCMF3nRiRgeb15kfoS4BbrJ5yupjrvwmbmvNg1AYFFS5sYNWft7K8v87wQvBakRtGP71Kp8NX77XFtu6xdB7sR6jpfC6qJPyB9akWNXgCrWy9kE4ih42gwAZdUugNZ9YtEsgRM3pwb6qJhkAPyEJtrxrja859PCAgqPSQiPQN33PaMkgQ6HJknu8CrjKRiXAycZ16KLUkHV64TNhEjPTcX1a7rqpD131AYMWX8d7CCdc9Ys7RUb6BwguuNSh8rJK3x4AkMDSUsaE8ynKvpC7RXZpJ9Nxfhd
- name: B
value: JgG4CcCbae1qEpe7mKXzp7m5hNc56SSyZd9DwUaEStKJrq7RToAC2Vgd3i6hKRwa58zCWeN6Wjc3o6qrdKPEPRvcyEPysamajVo5mdQiUgWAmr97pGEsyjuRjQoC2GY2LvLiEQxEgwFgJxKGMHMiaWMtDfxCDUaDEm4bu5RdMhqRZekAWho6c3WoEeruSr14iX1TrocFNfBkBY7CjEw8kcywXCTNgtvhb2Qiwgj5AxEF4wyw4bzaNA9ctXb1hoHPFVMu6C51pkFY7jUD9zwyH3ukgnAewkGAcPNbKmaTAtMosKRVaAN97mAwXh2VRt1hWmRvVk7r76EjnVKhD4vbsKZc56RVcHTVWRVdhU7FGyPsiE5rSQAz1JQGYzxnZpX7EG77CyrmUGyfueVfRHhwY2oq8A4uQCRaQxSaJHYLowjXSxh8DQ2V6MTqyzti32C27utBYdHzLVCJSGkmdzGwrFcHqsq7nLDxmvJVErPvyReixEe8kFmqopJ3e6LLm8WdYw9K6JYBjXnEfwPzm7Von9sf3dcaGDUHYfttMyeke7fAXJkvPRje69hYVyzdQGAauuojzGkkvQWCSMK1KCMNMznRaPDCNvofrQhYrub24WhmwpKhorufdfW8Cb4T6reBDCtaWVsbuinjtL6F6Sui5aYHJFLJ6e4pPewr1P4EuZYRbMBZwN5KvDLhTGLBuBnaTqUUdF6bj2U22NoRYMogiHiftqKqiexKNDXX1Zg9RQEvxgjuVo6SBW42mVEA8agrLhruRqCmiduJxVrfqLNGeYXHXrcmMEgW7uosJbPXvTcfRvdFWS1ov7oSALvj6vhDQ28Yi9D2ETNdNsfVWAFQuwvPpW7CHQGXTitprVbqH8JYxNZuGygcLmr5efbB22Vzu4ntd1HoraQpG12qeDEUA7tXYUpoYyuSdWwKPjSAMtaQcCSfVrhKQHQuKJargrVrez8vjWuwLfvSucV7ZHe7gjqvYgULdE1ubRCRSd7DuLjEN2Vd6obzV2c3MRet7ZSf4Sp88WM5AuTyW7BjArBc4S3gUQ8rYaiZ8Tu7NCxkEzbFwWRaemZkwfvcsX3XxqjyF37tFSGkEqE5kuBvpZW72675LkDffj7kH1zA8yE6dVujJjWsNYVFJWndUtz5Vy2KCdZAbBgq19q4AtsxWPodU2N3yZXzFAFAzTrxS6V4P7Scpdau1avgRvHLcBQPunA37xaYMy8YMifJwtmRY25mnAQwZAk3eANk7tXwZd58SDnciLNvARJvwKzTQBXcshkwyy52SX8XmXDJsPnRLaHmiYBJ63Yzr5XpZuuAtxb9qrWG2NHCNxfomHokWacV1hjZPPd6ZxT1FuRozB6Qt2NLcyqY7bnTcQJb1jPUaTAGXXCR8WVmmmYo2fDQe8CdBmgyPvbzNTEJUyScBz4RdycB5PZap4SurJCWtHbuMyQbQUB6jJgURDstfXS5Akfe4oruNq9rnYcNtnsDJPtrhXHBqzDizmf1BDxR5FB2RCxzCgeAfg8WQ1Ug9PVAGTzob6ZqCrGXzWXEUniZnf1vjr7QhGKBYXEX9SWDoSMUpP4FreVDTnx15ijRZTV3p8xG5fE9e36TnugRVvTyq7XzmyPBjW2r66f1bior
Sign:
- name: A
value: 3id6ddLcoNoe9rDgGM88ET8T6TnvHm5GFqFdN6kBzn7Q8d6VUGgjeT59CNWFiaofdeRnHBvX2A5ZacMXvfwaYEFuCbug
- name: B
value: 3iiLPj6wMUQpPwTBNZcUgkbXub1jumg4AEV9LfMyFHZVc84GLyAjVbVvH6EAGhcNrxRxL82aW4BimhDZCpLsRCqx5vwj
Read:
- name: 1
value: bamccoi5jdypwnjkiuuogkawvhkbowha4qg756uhnbkecr5vt3h4q

15
etc/path.go Normal file
View File

@ -0,0 +1,15 @@
package etc
import (
"path/filepath"
"runtime"
)
var (
_, b, _, _ = runtime.Caller(0)
basepath = filepath.Dir(b)
)
func Path() string {
return basepath
}

3
pkg/acl/account/accountdata.go
View File

@ -1,6 +1,7 @@
package account
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/encryptionkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
)
@ -9,5 +10,5 @@ type AccountData struct { // TODO: create a convenient constructor for this
Identity string // TODO: this is essentially the same as sign key
SignKey signingkey.PrivKey
EncKey encryptionkey.PrivKey
Decoder signingkey.PubKeyDecoder
Decoder keys.Decoder
}

1605
pkg/acl/aclchanges/aclpb/aclchanges.pb.go
View File

File diff suppressed because it is too large Load Diff

37
pkg/acl/aclchanges/aclpb/protos/aclchanges.proto
View File

@ -8,6 +8,12 @@ message RawChange {
string id = 3;
}
message RawRecord {
bytes payload = 1;
bytes signature = 2;
string id = 3;
}
// the element of change tree used to store and internal apply smartBlock history
message ACLChange {
repeated string treeHeadIds = 1;
@ -109,3 +115,34 @@ message ACLChange {
Removed = 3;
}
}
message Change {
repeated string treeHeadIds = 1;
string aclHeadId = 2;
string snapshotBaseId = 3; // we will only have one base snapshot for both
bytes changesData = 4;
uint64 currentReadKeyHash = 5;
int64 timestamp = 6;
string identity = 7;
bool isSnapshot = 8;
}
message Record {
string prevId = 1;
string identity = 2;
bytes data = 3;
uint64 currentReadKeyHash = 4;
int64 timestamp = 5;
}
message Header {
string firstId = 1;
string aclListId = 2;
string workspaceId = 3;
DocType docType = 4;
enum DocType {
ACL = 0;
DocTree = 1;
}
}

4
pkg/acl/aclchanges/change.go
View File

@ -1,11 +1,11 @@
package aclchanges
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/gogo/protobuf/proto"
)
type Change interface {
ProtoChange() *aclpb.ACLChange
ProtoChange() proto.Marshaler
DecryptedChangeContent() []byte
Signature() []byte
CID() string

186
pkg/acl/acltree/aclstatebuilder.go
View File

@ -1,186 +0,0 @@
package acltree
import (
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/encryptionkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
)
type aclStateBuilder struct {
tree *Tree
identity string
key encryptionkey.PrivKey
decoder signingkey.PubKeyDecoder
}
type decreasedPermissionsParameters struct {
users []*aclpb.ACLChangeUserPermissionChange
startChange string
}
func newACLStateBuilder(decoder signingkey.PubKeyDecoder, accountData *account.AccountData) *aclStateBuilder {
return &aclStateBuilder{
decoder: decoder,
identity: accountData.Identity,
key: accountData.EncKey,
}
}
func (sb *aclStateBuilder) Init(tree *Tree) error {
sb.tree = tree
return nil
}
func (sb *aclStateBuilder) Build() (*ACLState, error) {
state, _, err := sb.BuildBefore("")
return state, err
}
// TODO: we can probably have only one state builder, because we can Build both at the same time
func (sb *aclStateBuilder) BuildBefore(beforeId string) (*ACLState, bool, error) {
var (
err error
startChange = sb.tree.root
foundId bool
idSeenMap = make(map[string][]*Change)
decreasedPermissions *decreasedPermissionsParameters
)
root := sb.tree.Root()
if !root.IsSnapshot {
return nil, false, fmt.Errorf("root should always be a snapshot")
}
state, err := newACLStateFromSnapshotChange(
root.Content,
sb.identity,
sb.key,
sb.decoder)
if err != nil {
return nil, false, fmt.Errorf("could not build ACLState from snapshot: %w", err)
}
idSeenMap[startChange.Content.Identity] = append(idSeenMap[startChange.Content.Identity], startChange)
if startChange.Content.GetChangesData() != nil {
key, exists := state.userReadKeys[startChange.Content.CurrentReadKeyHash]
if !exists {
return nil, false, fmt.Errorf("no first snapshot")
}
err = startChange.DecryptContents(key)
if err != nil {
return nil, false, fmt.Errorf("failed to decrypt contents of first snapshot")
}
}
if beforeId == startChange.Id {
return state, true, nil
}
for {
// TODO: we should optimize this method to just remember last state of iterator and not iterate from the start and skip if nothing was removed from the Tree
sb.tree.IterateSkip(sb.tree.root.Id, startChange.Id, func(c *Change) (isContinue bool) {
defer func() {
if err == nil {
startChange = c
} else if err != ErrDocumentForbidden {
//log.Errorf("marking change %s as invalid: %v", c.Id, err)
sb.tree.RemoveInvalidChange(c.Id)
}
}()
// not applying root change
if c.Id == startChange.Id {
return true
}
idSeenMap[c.Content.Identity] = append(idSeenMap[c.Content.Identity], c)
if c.Content.GetAclData() != nil {
err = state.applyChange(c.Content)
if err != nil {
return false
}
// if we have some users who have less permissions now
users := state.getPermissionDecreasedUsers(c.Content)
if len(users) > 0 {
decreasedPermissions = &decreasedPermissionsParameters{
users: users,
startChange: c.Id,
}
return false
}
}
// the user can't make changes
if !state.hasPermission(c.Content.Identity, aclpb.ACLChange_Writer) && !state.hasPermission(c.Content.Identity, aclpb.ACLChange_Admin) {
err = fmt.Errorf("user %s cannot make changes", c.Content.Identity)
return false
}
// decrypting contents on the fly
if c.Content.GetChangesData() != nil {
key, exists := state.userReadKeys[c.Content.CurrentReadKeyHash]
if !exists {
err = fmt.Errorf("failed to find key with hash: %d", c.Content.CurrentReadKeyHash)
return false
}
err = c.DecryptContents(key)
if err != nil {
err = fmt.Errorf("failed to decrypt contents for hash: %d", c.Content.CurrentReadKeyHash)
return false
}
}
if c.Id == beforeId {
foundId = true
return false
}
return true
})
// if we have users with decreased permissions
if decreasedPermissions != nil {
var removed bool
validChanges := sb.tree.dfs(decreasedPermissions.startChange)
for _, permChange := range decreasedPermissions.users {
seenChanges := idSeenMap[permChange.Identity]
for _, seen := range seenChanges {
// if we find some invalid changes
if _, exists := validChanges[seen.Id]; !exists {
// if the user didn't have enough permission to make changes
if seen.IsACLChange() || permChange.Permissions > aclpb.ACLChange_Writer {
removed = true
sb.tree.RemoveInvalidChange(seen.Id)
}
}
}
}
decreasedPermissions = nil
if removed {
// starting from the beginning but with updated Tree
return sb.BuildBefore(beforeId)
}
} else if err == nil {
// we can finish the acl state building process
break
}
// the user is forbidden to access the document
if err == ErrDocumentForbidden {
return nil, foundId, err
}
// otherwise we have to continue from the change which we had
err = nil
}
return state, foundId, err
}

520
pkg/acl/acltree/acltree.go
View File

@ -1,520 +0,0 @@
package acltree
import (
"context"
"errors"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb"
"go.uber.org/zap"
"sync"
)
type AddResultSummary int
const (
AddResultSummaryNothing AddResultSummary = iota
AddResultSummaryAppend
AddResultSummaryRebuild
)
type AddResult struct {
OldHeads []string
Heads []string
Added []*aclpb.RawChange
// TODO: add summary for changes
Summary AddResultSummary
}
type TreeUpdateListener interface {
Update(tree ACLTree)
Rebuild(tree ACLTree)
}
type NoOpListener struct{}
func (n NoOpListener) Update(tree ACLTree) {}
func (n NoOpListener) Rebuild(tree ACLTree) {}
type RWLocker interface {
sync.Locker
RLock()
RUnlock()
}
var ErrNoCommonSnapshot = errors.New("trees doesn't have a common snapshot")
type ACLTree interface {
RWLocker
ID() string
Header() *treepb.TreeHeader
ACLState() *ACLState
AddContent(ctx context.Context, f func(builder ChangeBuilder) error) (*aclpb.RawChange, error)
AddRawChanges(ctx context.Context, changes ...*aclpb.RawChange) (AddResult, error)
Heads() []string
Root() *Change
Iterate(func(change *Change) bool)
IterateFrom(string, func(change *Change) bool)
HasChange(string) bool
SnapshotPath() []string
ChangesAfterCommonSnapshot(snapshotPath []string) ([]*aclpb.RawChange, error)
Storage() treestorage.TreeStorage
DebugDump() (string, error)
Close() error
}
type aclTree struct {
treeStorage treestorage.TreeStorage
accountData *account.AccountData
updateListener TreeUpdateListener
id string
header *treepb.TreeHeader
fullTree *Tree
aclTreeFromStart *Tree // TODO: right now we don't use it, we can probably have only local var for now. This tree is built from start of the document
aclState *ACLState
treeBuilder *treeBuilder
aclTreeBuilder *aclTreeBuilder
aclStateBuilder *aclStateBuilder
snapshotValidator *snapshotValidator
changeBuilder *changeBuilder
sync.RWMutex
}
func BuildACLTree(
t treestorage.TreeStorage,
acc *account.AccountData,
listener TreeUpdateListener) (ACLTree, error) {
aclTreeBuilder := newACLTreeBuilder(t, acc.Decoder)
treeBuilder := newTreeBuilder(t, acc.Decoder)
snapshotValidator := newSnapshotValidator(acc.Decoder, acc) // TODO: this looks weird, change it
aclStateBuilder := newACLStateBuilder(acc.Decoder, acc)
changeBuilder := newChangeBuilder()
aclTree := &aclTree{
treeStorage: t,
accountData: acc,
fullTree: nil,
aclState: nil,
treeBuilder: treeBuilder,
aclTreeBuilder: aclTreeBuilder,
aclStateBuilder: aclStateBuilder,
snapshotValidator: snapshotValidator,
changeBuilder: changeBuilder,
updateListener: listener,
}
err := aclTree.rebuildFromStorage(false)
if err != nil {
return nil, err
}
err = aclTree.removeOrphans()
if err != nil {
return nil, err
}
err = t.SetHeads(aclTree.Heads())
if err != nil {
return nil, err
}
aclTree.id, err = t.TreeID()
if err != nil {
return nil, err
}
aclTree.header, err = t.Header()
if err != nil {
return nil, err
}
listener.Rebuild(aclTree)
return aclTree, nil
}
// TODO: this is not used for now, in future we should think about not making full tree rebuild
//func (a *aclTree) rebuildFromTree(validateSnapshot bool) (err error) {
// if validateSnapshot {
// err = a.snapshotValidator.Init(a.aclTreeFromStart)
// if err != nil {
// return err
// }
//
// valid, err := a.snapshotValidator.ValidateSnapshot(a.fullTree.root)
// if err != nil {
// return err
// }
// if !valid {
// return a.rebuildFromStorage(true)
// }
// }
//
// err = a.aclStateBuilder.Init(a.fullTree)
// if err != nil {
// return err
// }
//
// a.aclState, err = a.aclStateBuilder.Build()
// if err != nil {
// return err
// }
//
// return nil
//}
func (a *aclTree) removeOrphans() error {
// removing attached or invalid orphans
var toRemove []string
orphans, err := a.treeStorage.Orphans()
if err != nil {
return err
}
for _, orphan := range orphans {
if _, exists := a.fullTree.attached[orphan]; exists {
toRemove = append(toRemove, orphan)
}
if _, exists := a.fullTree.invalidChanges[orphan]; exists {
toRemove = append(toRemove, orphan)
}
}
return a.treeStorage.RemoveOrphans(toRemove...)
}
func (a *aclTree) rebuildFromStorage(fromStart bool) error {
a.treeBuilder.Init()
a.aclTreeBuilder.Init()
var err error
a.fullTree, err = a.treeBuilder.Build(fromStart)
if err != nil {
return err
}
// TODO: remove this from context as this is used only to validate snapshot
a.aclTreeFromStart, err = a.aclTreeBuilder.Build()
if err != nil {
return err
}
if !fromStart {
err = a.snapshotValidator.Init(a.aclTreeFromStart)
if err != nil {
return err
}
valid, err := a.snapshotValidator.ValidateSnapshot(a.fullTree.root)
if err != nil {
return err
}
if !valid {
return a.rebuildFromStorage(true)
}
}
// TODO: there is a question how we can validate not only that the full tree is built correctly
// but also that the ACL prev ids are not messed up. I think we should probably compare the resulting
// acl state with the acl state which is built in aclTreeFromStart
err = a.aclStateBuilder.Init(a.fullTree)
if err != nil {
return err
}
a.aclState, err = a.aclStateBuilder.Build()
if err != nil {
return err
}
return nil
}
func (a *aclTree) ID() string {
return a.id
}
func (a *aclTree) Header() *treepb.TreeHeader {
return a.header
}
func (a *aclTree) ACLState() *ACLState {
return a.aclState
}
func (a *aclTree) Storage() treestorage.TreeStorage {
return a.treeStorage
}
func (a *aclTree) AddContent(ctx context.Context, build func(builder ChangeBuilder) error) (*aclpb.RawChange, error) {
// TODO: add snapshot creation logic
defer func() {
// TODO: should this be called in a separate goroutine to prevent accidental cycles (tree->updater->tree)
a.updateListener.Update(a)
}()
a.changeBuilder.Init(a.aclState, a.fullTree, a.accountData)
err := build(a.changeBuilder)
if err != nil {
return nil, err
}
ch, marshalled, err := a.changeBuilder.BuildAndApply()
if err != nil {
return nil, err
}
a.fullTree.AddFast(ch)
rawCh := &aclpb.RawChange{
Payload: marshalled,
Signature: ch.Signature(),
Id: ch.Id,
}
err = a.treeStorage.AddRawChange(rawCh)
if err != nil {
return nil, err
}
err = a.treeStorage.SetHeads([]string{ch.Id})
if err != nil {
return nil, err
}
return rawCh, nil
}
func (a *aclTree) AddRawChanges(ctx context.Context, rawChanges ...*aclpb.RawChange) (AddResult, error) {
// TODO: make proper error handling, because there are a lot of corner cases where this will break
var err error
var mode Mode
var changes []*Change // TODO: = addChangesBuf[:0] ...
for _, ch := range rawChanges {
change, err := NewFromRawChange(ch)
// TODO: think what if we will have incorrect signatures on rawChanges, how everything will work
if err != nil {
continue
}
changes = append(changes, change)
}
defer func() {
if err != nil {
return
}
err = a.removeOrphans()
if err != nil {
return
}
err = a.treeStorage.SetHeads(a.fullTree.Heads())
if err != nil {
return
}
switch mode {
case Append:
a.updateListener.Update(a)
case Rebuild:
a.updateListener.Rebuild(a)
default:
break
}
}()
getAddedChanges := func() []*aclpb.RawChange {
var added []*aclpb.RawChange
for _, ch := range rawChanges {
if _, exists := a.fullTree.attached[ch.Id]; exists {
added = append(added, ch)
}
}
return added
}
for _, ch := range changes {
err = a.treeStorage.AddChange(ch)
if err != nil {
return AddResult{}, err
}
err = a.treeStorage.AddOrphans(ch.Id)
if err != nil {
return AddResult{}, err
}
}
prevHeads := a.fullTree.Heads()
mode = a.fullTree.Add(changes...)
switch mode {
case Nothing:
return AddResult{
OldHeads: prevHeads,
Heads: prevHeads,
Summary: AddResultSummaryNothing,
}, nil
case Rebuild:
err = a.rebuildFromStorage(false)
if err != nil {
return AddResult{}, err
}
return AddResult{
OldHeads: prevHeads,
Heads: a.fullTree.Heads(),
Added: getAddedChanges(),
Summary: AddResultSummaryRebuild,
}, nil
default:
// just rebuilding the state from start without reloading everything from tree storage
// as an optimization we could've started from current heads, but I didn't implement that
a.aclState, err = a.aclStateBuilder.Build()
if err != nil {
return AddResult{}, err
}
return AddResult{
OldHeads: prevHeads,
Heads: a.fullTree.Heads(),
Added: getAddedChanges(),
Summary: AddResultSummaryAppend,
}, nil
}
}
func (a *aclTree) Iterate(f func(change *Change) bool) {
a.fullTree.Iterate(a.fullTree.RootId(), f)
}
func (a *aclTree) IterateFrom(s string, f func(change *Change) bool) {
a.fullTree.Iterate(s, f)
}
func (a *aclTree) HasChange(s string) bool {
_, attachedExists := a.fullTree.attached[s]
_, unattachedExists := a.fullTree.unAttached[s]
_, invalidExists := a.fullTree.invalidChanges[s]
return attachedExists || unattachedExists || invalidExists
}
func (a *aclTree) Heads() []string {
return a.fullTree.Heads()
}
func (a *aclTree) Root() *Change {
return a.fullTree.Root()
}
func (a *aclTree) Close() error {
return nil
}
func (a *aclTree) SnapshotPath() []string {
// TODO: think about caching this
var path []string
// TODO: think that the user may have not all of the snapshots locally
currentSnapshotId := a.fullTree.RootId()
for currentSnapshotId != "" {
sn, err := a.treeBuilder.loadChange(currentSnapshotId)
if err != nil {
break
}
path = append(path, currentSnapshotId)
currentSnapshotId = sn.SnapshotId
}
return path
}
func (a *aclTree) ChangesAfterCommonSnapshot(theirPath []string) ([]*aclpb.RawChange, error) {
// TODO: think about when the clients will have their full acl tree and thus full snapshots
// but no changes after some of the snapshots
var (
isNewDocument = len(theirPath) == 0
ourPath = a.SnapshotPath()
// by default returning everything we have
commonSnapshot = ourPath[len(ourPath)-1] // TODO: root snapshot, probably it is better to have a specific method in treestorage
err error
)
// if this is non-empty request
if !isNewDocument {
commonSnapshot, err = a.commonSnapshotForTwoPaths(ourPath, theirPath)
if err != nil {
return nil, err
}
}
var rawChanges []*aclpb.RawChange
// using custom load function to skip verification step and save raw changes
load := func(id string) (*Change, error) {
raw, err := a.treeStorage.GetChange(context.Background(), id)
if err != nil {
return nil, err
}
aclChange, err := a.treeBuilder.makeUnverifiedACLChange(raw)
if err != nil {
return nil, err
}
ch := NewChange(id, aclChange)
rawChanges = append(rawChanges, raw)
return ch, nil
}
// we presume that we have everything after the common snapshot, though this may not be the case in case of clients and only ACL tree changes
log.With(
zap.Strings("heads", a.fullTree.Heads()),
zap.String("breakpoint", commonSnapshot),
zap.String("id", a.id)).
Debug("getting all changes from common snapshot")
_, err = a.treeBuilder.dfs(a.fullTree.Heads(), commonSnapshot, load)
if err != nil {
return nil, err
}
if isNewDocument {
// adding snapshot to raw changes
_, err = load(commonSnapshot)
if err != nil {
return nil, err
}
}
log.With(
zap.Int("len(changes)", len(rawChanges)),
zap.String("id", a.id)).
Debug("returning all changes after common snapshot")
return rawChanges, nil
}
func (a *aclTree) DebugDump() (string, error) {
return a.fullTree.Graph()
}
func (a *aclTree) commonSnapshotForTwoPaths(ourPath []string, theirPath []string) (string, error) {
var i int
var j int
log.With(zap.Strings("our path", ourPath), zap.Strings("their path", theirPath)).
Debug("finding common snapshot for two paths")
OuterLoop:
// find starting point from the right
for i = len(ourPath) - 1; i >= 0; i-- {
for j = len(theirPath) - 1; j >= 0; j-- {
// most likely there would be only one comparison, because mostly the snapshot path will start from the root for nodes
if ourPath[i] == theirPath[j] {
break OuterLoop
}
}
}
if i < 0 || j < 0 {
return "", ErrNoCommonSnapshot
}
// find last common element of the sequence moving from right to left
for i >= 0 && j >= 0 {
if ourPath[i] == theirPath[j] {
i--
j--
}
}
return ourPath[i+1], nil
}

262
pkg/acl/acltree/acltree_test.go
View File

@ -1,262 +0,0 @@
package acltree
import (
"context"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/treestoragebuilder"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"testing"
"github.com/stretchr/testify/assert"
)
type mockListener struct{}
func (m *mockListener) Update(tree ACLTree) {}
func (m *mockListener) Rebuild(tree ACLTree) {}
func TestACLTree_UserJoinBuild(t *testing.T) {
thr, err := treestoragebuilder.NewTreeStorageBuilderWithTestName("userjoinexample.yml")
if err != nil {
t.Fatal(err)
}
keychain := thr.GetKeychain()
accountData := &account.AccountData{
Identity: keychain.GetIdentity("A"),
SignKey: keychain.SigningKeys["A"],
EncKey: keychain.EncryptionKeys["A"],
Decoder: signingkey.NewEd25519PubKeyDecoder(),
}
listener := &mockListener{}
tree, err := BuildACLTree(thr, accountData, listener)
if err != nil {
t.Fatalf("should Build acl ACLState without err: %v", err)
}
aclState := tree.ACLState()
aId := keychain.GeneratedIdentities["A"]
bId := keychain.GeneratedIdentities["B"]
cId := keychain.GeneratedIdentities["C"]
assert.Equal(t, aclState.identity, aId)
assert.Equal(t, aclState.userStates[aId].Permissions, aclpb.ACLChange_Admin)
assert.Equal(t, aclState.userStates[bId].Permissions, aclpb.ACLChange_Writer)
assert.Equal(t, aclState.userStates[cId].Permissions, aclpb.ACLChange_Reader)
var changeIds []string
tree.Iterate(func(c *Change) (isContinue bool) {
changeIds = append(changeIds, c.Id)
return true
})
assert.Equal(t, changeIds, []string{"A.1.1", "A.1.2", "B.1.1", "B.1.2"})
}
func TestACLTree_UserJoinUpdate_Append(t *testing.T) {
thr, err := treestoragebuilder.NewTreeStorageBuilderWithTestName("userjoinexampleupdate.yml")
if err != nil {
t.Fatal(err)
}
keychain := thr.GetKeychain()
accountData := &account.AccountData{
Identity: keychain.GetIdentity("A"),
SignKey: keychain.SigningKeys["A"],
EncKey: keychain.EncryptionKeys["A"],
Decoder: signingkey.NewEd25519PubKeyDecoder(),
}
listener := &mockListener{}
tree, err := BuildACLTree(thr, accountData, listener)
if err != nil {
t.Fatalf("should Build acl ACLState without err: %v", err)
}
rawChanges := thr.GetUpdates("append")
res, err := tree.AddRawChanges(context.Background(), rawChanges...)
assert.Equal(t, res.Summary, AddResultSummaryAppend)
aclState := tree.ACLState()
aId := keychain.GeneratedIdentities["A"]
bId := keychain.GeneratedIdentities["B"]
cId := keychain.GeneratedIdentities["C"]
dId := keychain.GeneratedIdentities["D"]
assert.Equal(t, aclState.identity, aId)
assert.Equal(t, aclState.userStates[aId].Permissions, aclpb.ACLChange_Admin)
assert.Equal(t, aclState.userStates[bId].Permissions, aclpb.ACLChange_Writer)
assert.Equal(t, aclState.userStates[cId].Permissions, aclpb.ACLChange_Reader)
assert.Equal(t, aclState.userStates[dId].Permissions, aclpb.ACLChange_Writer)
var changeIds []string
tree.Iterate(func(c *Change) (isContinue bool) {
changeIds = append(changeIds, c.Id)
return true
})
assert.Equal(t, changeIds, []string{"A.1.1", "A.1.2", "B.1.1", "B.1.2", "B.1.3", "A.1.4"})
}
func TestACLTree_UserJoinUpdate_Rebuild(t *testing.T) {
thr, err := treestoragebuilder.NewTreeStorageBuilderWithTestName("userjoinexampleupdate.yml")
if err != nil {
t.Fatal(err)
}
keychain := thr.GetKeychain()
accountData := &account.AccountData{
Identity: keychain.GetIdentity("A"),
SignKey: keychain.SigningKeys["A"],
EncKey: keychain.EncryptionKeys["A"],
Decoder: signingkey.NewEd25519PubKeyDecoder(),
}
listener := &mockListener{}
tree, err := BuildACLTree(thr, accountData, listener)
if err != nil {
t.Fatalf("should Build acl ACLState without err: %v", err)
}
rawChanges := thr.GetUpdates("rebuild")
res, err := tree.AddRawChanges(context.Background(), rawChanges...)
assert.Equal(t, res.Summary, AddResultSummaryRebuild)
aclState := tree.ACLState()
aId := keychain.GeneratedIdentities["A"]
bId := keychain.GeneratedIdentities["B"]
cId := keychain.GeneratedIdentities["C"]
dId := keychain.GeneratedIdentities["D"]
assert.Equal(t, aclState.identity, aId)
assert.Equal(t, aclState.userStates[aId].Permissions, aclpb.ACLChange_Admin)
assert.Equal(t, aclState.userStates[bId].Permissions, aclpb.ACLChange_Writer)
assert.Equal(t, aclState.userStates[cId].Permissions, aclpb.ACLChange_Reader)
assert.Equal(t, aclState.userStates[dId].Permissions, aclpb.ACLChange_Writer)
var changeIds []string
tree.Iterate(func(c *Change) (isContinue bool) {
changeIds = append(changeIds, c.Id)
return true
})
assert.Equal(t, changeIds, []string{"A.1.1", "A.1.2", "B.1.1", "B.1.2", "A.1.4"})
}
func TestACLTree_UserRemoveBuild(t *testing.T) {
thr, err := treestoragebuilder.NewTreeStorageBuilderWithTestName("userremoveexample.yml")
if err != nil {
t.Fatal(err)
}
keychain := thr.GetKeychain()
accountData := &account.AccountData{
Identity: keychain.GetIdentity("A"),
SignKey: keychain.SigningKeys["A"],
EncKey: keychain.EncryptionKeys["A"],
Decoder: signingkey.NewEd25519PubKeyDecoder(),
}
listener := &mockListener{}
tree, err := BuildACLTree(thr, accountData, listener)
if err != nil {
t.Fatalf("should Build acl ACLState without err: %v", err)
}
aclState := tree.ACLState()
aId := keychain.GeneratedIdentities["A"]
assert.Equal(t, aclState.identity, aId)
assert.Equal(t, aclState.userStates[aId].Permissions, aclpb.ACLChange_Admin)
var changeIds []string
tree.Iterate(func(c *Change) (isContinue bool) {
changeIds = append(changeIds, c.Id)
return true
})
assert.Equal(t, changeIds, []string{"A.1.1", "A.1.2", "B.1.1", "A.1.3", "A.1.4"})
}
func TestACLTree_UserRemoveBeforeBuild(t *testing.T) {
thr, err := treestoragebuilder.NewTreeStorageBuilderWithTestName("userremovebeforeexample.yml")
if err != nil {
t.Fatal(err)
}
keychain := thr.GetKeychain()
accountData := &account.AccountData{
Identity: keychain.GetIdentity("A"),
SignKey: keychain.SigningKeys["A"],
EncKey: keychain.EncryptionKeys["A"],
Decoder: signingkey.NewEd25519PubKeyDecoder(),
}
listener := &mockListener{}
tree, err := BuildACLTree(thr, accountData, listener)
if err != nil {
t.Fatalf("should Build acl ACLState without err: %v", err)
}
aclState := tree.ACLState()
for _, s := range []string{"A", "C", "E"} {
assert.Equal(t, aclState.userStates[keychain.GetIdentity(s)].Permissions, aclpb.ACLChange_Admin)
}
assert.Equal(t, aclState.identity, keychain.GetIdentity("A"))
assert.Nil(t, aclState.userStates[keychain.GetIdentity("B")])
var changeIds []string
tree.Iterate(func(c *Change) (isContinue bool) {
changeIds = append(changeIds, c.Id)
return true
})
assert.Equal(t, changeIds, []string{"A.1.1", "B.1.1", "A.1.2", "A.1.3"})
}
func TestACLTree_InvalidSnapshotBuild(t *testing.T) {
thr, err := treestoragebuilder.NewTreeStorageBuilderWithTestName("invalidsnapshotexample.yml")
if err != nil {
t.Fatal(err)
}
keychain := thr.GetKeychain()
accountData := &account.AccountData{
Identity: keychain.GetIdentity("A"),
SignKey: keychain.SigningKeys["A"],
EncKey: keychain.EncryptionKeys["A"],
Decoder: signingkey.NewEd25519PubKeyDecoder(),
}
listener := &mockListener{}
tree, err := BuildACLTree(thr, accountData, listener)
if err != nil {
t.Fatalf("should Build acl ACLState without err: %v", err)
}
aclState := tree.ACLState()
for _, s := range []string{"A", "B", "C", "D", "E", "F"} {
assert.Equal(t, aclState.userStates[keychain.GetIdentity(s)].Permissions, aclpb.ACLChange_Admin)
}
assert.Equal(t, aclState.identity, keychain.GetIdentity("A"))
var changeIds []string
tree.Iterate(func(c *Change) (isContinue bool) {
changeIds = append(changeIds, c.Id)
return true
})
assert.Equal(t, []string{"A.1.1", "B.1.1", "A.1.2", "A.1.3", "B.1.2"}, changeIds)
}
func TestACLTree_ValidSnapshotBuild(t *testing.T) {
thr, err := treestoragebuilder.NewTreeStorageBuilderWithTestName("validsnapshotexample.yml")
if err != nil {
t.Fatal(err)
}
keychain := thr.GetKeychain()
accountData := &account.AccountData{
Identity: keychain.GetIdentity("A"),
SignKey: keychain.SigningKeys["A"],
EncKey: keychain.EncryptionKeys["A"],
Decoder: signingkey.NewEd25519PubKeyDecoder(),
}
listener := &mockListener{}
tree, err := BuildACLTree(thr, accountData, listener)
if err != nil {
t.Fatalf("should Build acl ACLState without err: %v", err)
}
aclState := tree.ACLState()
for _, s := range []string{"A", "B", "C", "D", "E", "F"} {
assert.Equal(t, aclState.userStates[keychain.GetIdentity(s)].Permissions, aclpb.ACLChange_Admin)
}
assert.Equal(t, aclState.identity, keychain.GetIdentity("A"))
var changeIds []string
tree.Iterate(func(c *Change) (isContinue bool) {
changeIds = append(changeIds, c.Id)
return true
})
assert.Equal(t, []string{"A.1.2", "A.1.3", "B.1.2"}, changeIds)
}

154
pkg/acl/acltree/acltreebuilder.go
View File

@ -1,154 +0,0 @@
package acltree
import (
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice"
)
type aclTreeBuilder struct {
cache map[string]*Change
identityKeys map[string]signingkey.PubKey
signingPubKeyDecoder signingkey.PubKeyDecoder
tree *Tree
treeStorage treestorage.TreeStorage
*changeLoader
}
func newACLTreeBuilder(t treestorage.TreeStorage, decoder signingkey.PubKeyDecoder) *aclTreeBuilder {
return &aclTreeBuilder{
signingPubKeyDecoder: decoder,
treeStorage: t,
changeLoader: newChangeLoader(
t,
decoder,
NewACLChange),
}
}
func (tb *aclTreeBuilder) Init() {
tb.cache = make(map[string]*Change)
tb.identityKeys = make(map[string]signingkey.PubKey)
tb.tree = &Tree{}
tb.changeLoader.Init(tb.cache, tb.identityKeys)
}
func (tb *aclTreeBuilder) Build() (*Tree, error) {
var headsAndOrphans []string
orphans, err := tb.treeStorage.Orphans()
if err != nil {
return nil, err
}
heads, err := tb.treeStorage.Heads()
if err != nil {
return nil, err
}
headsAndOrphans = append(headsAndOrphans, orphans...)
headsAndOrphans = append(headsAndOrphans, heads...)
aclHeads, err := tb.getACLHeads(headsAndOrphans)
if err != nil {
return nil, err
}
if err = tb.buildTreeFromStart(aclHeads); err != nil {
return nil, fmt.Errorf("buildTree error: %v", err)
}
tb.cache = nil
return tb.tree, nil
}
func (tb *aclTreeBuilder) buildTreeFromStart(heads []string) (err error) {
changes, root, err := tb.dfsFromStart(heads)
if err != nil {
return err
}
tb.tree.AddFast(root)
tb.tree.AddFast(changes...)
return
}
func (tb *aclTreeBuilder) dfsFromStart(heads []string) (buf []*Change, root *Change, err error) {
var possibleRoots []*Change
stack := make([]string, len(heads), len(heads)*2)
copy(stack, heads)
buf = make([]*Change, 0, len(stack)*2)
uniqMap := make(map[string]struct{})
for len(stack) > 0 {
id := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if _, exists := uniqMap[id]; exists {
continue
}
ch, err := tb.loadChange(id)
if err != nil {
continue
}
uniqMap[id] = struct{}{}
buf = append(buf, ch)
for _, prev := range ch.PreviousIds {
stack = append(stack, prev)
}
if len(ch.PreviousIds) == 0 {
possibleRoots = append(possibleRoots, ch)
}
}
header, err := tb.treeStorage.Header()
if err != nil {
return nil, nil, err
}
for _, r := range possibleRoots {
if r.Id == header.FirstChangeId {
return buf, r, nil
}
}
return nil, nil, fmt.Errorf("could not find root change")
}
func (tb *aclTreeBuilder) getACLHeads(heads []string) (aclTreeHeads []string, err error) {
for _, head := range heads {
if slice.FindPos(aclTreeHeads, head) != -1 { // do not scan known heads
continue
}
precedingHeads, err := tb.getPrecedingACLHeads(head)
if err != nil {
continue
}
for _, aclHead := range precedingHeads {
if slice.FindPos(aclTreeHeads, aclHead) != -1 {
continue
}
aclTreeHeads = append(aclTreeHeads, aclHead)
}
}
if len(aclTreeHeads) == 0 {
return nil, fmt.Errorf("no usable ACL heads in tree storage")
}
return aclTreeHeads, nil
}
func (tb *aclTreeBuilder) getPrecedingACLHeads(head string) ([]string, error) {
headChange, err := tb.loadChange(head)
if err != nil {
return nil, err
}
if headChange.Content.GetAclData() != nil {
return []string{head}, nil
} else {
return headChange.PreviousIds, nil
}
}

70
pkg/acl/acltree/acltreestorage.go
View File

@ -1,70 +0,0 @@
package acltree
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/cid"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"github.com/gogo/protobuf/proto"
)
func CreateNewTreeStorageWithACL(
acc *account.AccountData,
build func(builder ChangeBuilder) error,
create treestorage.CreatorFunc) (treestorage.TreeStorage, error) {
bld := newChangeBuilder()
bld.Init(
newACLState(acc.Identity, acc.EncKey, signingkey.NewEd25519PubKeyDecoder()),
&Tree{},
acc)
err := build(bld)
if err != nil {
return nil, err
}
bld.SetMakeSnapshot(true)
change, payload, err := bld.BuildAndApply()
if err != nil {
return nil, err
}
rawChange := &aclpb.RawChange{
Payload: payload,
Signature: change.Signature(),
Id: change.CID(),
}
header, id, err := createTreeHeaderAndId(rawChange)
if err != nil {
return nil, err
}
thr, err := create(id, header, []*aclpb.RawChange{rawChange})
if err != nil {
return nil, err
}
err = thr.SetHeads([]string{change.CID()})
if err != nil {
return nil, err
}
return thr, nil
}
func createTreeHeaderAndId(change *aclpb.RawChange) (*treepb.TreeHeader, string, error) {
header := &treepb.TreeHeader{
FirstChangeId: change.Id,
IsWorkspace: false,
}
marshalledHeader, err := proto.Marshal(header)
if err != nil {
return nil, "", err
}
treeId, err := cid.NewCIDFromBytes(marshalledHeader)
if err != nil {
return nil, "", err
}
return header, treeId, nil
}

63
pkg/acl/acltree/acltreestorage_test.go
View File

@ -1,63 +0,0 @@
package acltree
import (
"context"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/treestoragebuilder"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage"
"github.com/stretchr/testify/assert"
"testing"
)
func Test_BuildTreeStorageWithACL(t *testing.T) {
keychain := treestoragebuilder.NewKeychain()
keychain.AddSigningKey("A")
keychain.AddEncryptionKey("A")
data := &account.AccountData{
Identity: keychain.GetIdentity("A"),
SignKey: keychain.SigningKeys["A"],
EncKey: keychain.EncryptionKeys["A"],
}
thr, err := CreateNewTreeStorageWithACL(
data,
func(builder ChangeBuilder) error {
return builder.UserAdd(
keychain.GetIdentity("A"),
keychain.EncryptionKeys["A"].GetPublic(),
aclpb.ACLChange_Admin)
},
treestorage.NewInMemoryTreeStorage)
if err != nil {
t.Fatalf("build should not return error")
}
heads, err := thr.Heads()
if err != nil {
t.Fatalf("should return heads: %v", err)
}
if len(heads) == 0 {
t.Fatalf("tree storage should have non-empty heads")
}
header, err := thr.Header()
if err != nil {
t.Fatalf("tree storage header should return without error: %v", err)
}
assert.Equal(t, heads[0], header.FirstChangeId)
treeId, err := thr.TreeID()
if err != nil {
t.Fatalf("tree id should return without error: %v", err)
}
assert.NotEmpty(t, treeId)
ch, err := thr.GetChange(context.Background(), header.FirstChangeId)
if err != nil {
t.Fatalf("get change should not return error: %v", err)
}
_, err = NewFromRawChange(ch)
if err != nil {
t.Fatalf("we should be able to unmarshall change: %v", err)
}
}

97
pkg/acl/acltree/change.go
View File

@ -1,97 +0,0 @@
package acltree
import (
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/gogo/protobuf/proto"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/symmetric"
)
type ChangeContent struct {
ChangesData proto.Marshaler
ACLData *aclpb.ACLChangeACLData
Id string // TODO: this is just for testing, because id should be created automatically from content
}
// Change is an abstract type for all types of changes
type Change struct {
Next []*Change
Unattached []*Change
PreviousIds []string
Id string
SnapshotId string
IsSnapshot bool
DecryptedDocumentChange []byte
Content *aclpb.ACLChange
Sign []byte
}
func (ch *Change) DecryptContents(key *symmetric.Key) error {
if ch.Content.ChangesData == nil {
return nil
}
decrypted, err := key.Decrypt(ch.Content.ChangesData)
if err != nil {
return fmt.Errorf("failed to decrypt changes data: %w", err)
}
ch.DecryptedDocumentChange = decrypted
return nil
}
func (ch *Change) IsACLChange() bool {
return ch.Content.GetAclData() != nil
}
func NewFromRawChange(rawChange *aclpb.RawChange) (*Change, error) {
unmarshalled := &aclpb.ACLChange{}
err := proto.Unmarshal(rawChange.Payload, unmarshalled)
if err != nil {
return nil, err
}
ch := NewChange(rawChange.Id, unmarshalled)
ch.Sign = rawChange.Signature
return ch, nil
}
func NewChange(id string, ch *aclpb.ACLChange) *Change {
return &Change{
Next: nil,
PreviousIds: ch.TreeHeadIds,
Id: id,
Content: ch,
SnapshotId: ch.SnapshotBaseId,
IsSnapshot: ch.GetAclData().GetAclSnapshot() != nil,
}
}
func NewACLChange(id string, ch *aclpb.ACLChange) *Change {
return &Change{
Next: nil,
PreviousIds: ch.AclHeadIds,
Id: id,
Content: ch,
SnapshotId: ch.SnapshotBaseId,
IsSnapshot: ch.GetAclData().GetAclSnapshot() != nil,
}
}
func (ch *Change) ProtoChange() *aclpb.ACLChange {
return ch.Content
}
func (ch *Change) DecryptedChangeContent() []byte {
return ch.DecryptedDocumentChange
}
func (ch *Change) Signature() []byte {
return ch.Sign
}
func (ch *Change) CID() string {
return ch.Id
}

99
pkg/acl/acltree/changeloader.go
View File

@ -1,99 +0,0 @@
package acltree
import (
"context"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"time"
"github.com/gogo/protobuf/proto"
)
type changeLoader struct {
cache map[string]*Change
identityKeys map[string]signingkey.PubKey
signingPubKeyDecoder signingkey.PubKeyDecoder
treeStorage treestorage.TreeStorage
changeCreator func(id string, ch *aclpb.ACLChange) *Change
}
func newChangeLoader(
treeStorage treestorage.TreeStorage,
signingPubKeyDecoder signingkey.PubKeyDecoder,
changeCreator func(id string, ch *aclpb.ACLChange) *Change) *changeLoader {
return &changeLoader{
signingPubKeyDecoder: signingPubKeyDecoder,
treeStorage: treeStorage,
changeCreator: changeCreator,
}
}
func (c *changeLoader) Init(cache map[string]*Change,
identityKeys map[string]signingkey.PubKey) {
c.cache = cache
c.identityKeys = identityKeys
}
func (c *changeLoader) loadChange(id string) (ch *Change, err error) {
if ch, ok := c.cache[id]; ok {
return ch, nil
}
// TODO: Add virtual changes logic
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
change, err := c.treeStorage.GetChange(ctx, id)
if err != nil {
return nil, err
}
aclChange, err := c.makeVerifiedACLChange(change)
if err != nil {
return nil, err
}
ch = c.changeCreator(id, aclChange)
c.cache[id] = ch
return ch, nil
}
func (c *changeLoader) verify(identity string, payload, signature []byte) (isVerified bool, err error) {
identityKey, exists := c.identityKeys[identity]
if !exists {
identityKey, err = c.signingPubKeyDecoder.DecodeFromString(identity)
if err != nil {
return
}
c.identityKeys[identity] = identityKey
}
return identityKey.Verify(payload, signature)
}
func (c *changeLoader) makeVerifiedACLChange(change *aclpb.RawChange) (aclChange *aclpb.ACLChange, err error) {
aclChange = new(aclpb.ACLChange)
// TODO: think what should we do with such cases, because this can be used by attacker to break our Tree
if err = proto.Unmarshal(change.Payload, aclChange); err != nil {
return
}
var verified bool
verified, err = c.verify(aclChange.Identity, change.Payload, change.Signature)
if err != nil {
return
}
if !verified {
err = fmt.Errorf("the signature of the payload cannot be verified")
return
}
return
}
func (c *changeLoader) makeUnverifiedACLChange(change *aclpb.RawChange) (aclChange *aclpb.ACLChange, err error) {
aclChange = new(aclpb.ACLChange)
err = proto.Unmarshal(change.Payload, aclChange)
return
}

50
pkg/acl/acltree/snapshotvalidator.go
View File

@ -1,50 +0,0 @@
package acltree
import (
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/encryptionkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
)
type snapshotValidator struct {
aclTree *Tree
identity string
key encryptionkey.PrivKey
decoder signingkey.PubKeyDecoder
stateBuilder *aclStateBuilder
}
func newSnapshotValidator(
decoder signingkey.PubKeyDecoder,
accountData *account.AccountData) *snapshotValidator {
return &snapshotValidator{
identity: accountData.Identity,
key: accountData.EncKey,
decoder: decoder,
stateBuilder: newACLStateBuilder(decoder, accountData),
}
}
func (s *snapshotValidator) Init(aclTree *Tree) error {
s.aclTree = aclTree
return s.stateBuilder.Init(aclTree)
}
func (s *snapshotValidator) ValidateSnapshot(ch *Change) (bool, error) {
st, found, err := s.stateBuilder.BuildBefore(ch.Id)
if err != nil {
return false, err
}
if !found {
return false, fmt.Errorf("didn't find snapshot in ACL Tree")
}
otherSt, err := newACLStateFromSnapshotChange(ch.Content, s.identity, s.key, s.decoder)
if err != nil {
return false, err
}
return st.equal(otherSt), nil
}

64
pkg/acl/acltree/treebuilder_test.go
View File

@ -1,64 +0,0 @@
package acltree
//func createTreeFromThread(t thread.Thread, fromStart bool) (*Tree, error) {
// treeBuilder := newTreeBuilder(t, keys.NewEd25519Decoder())
// treeBuilder.Init()
// return treeBuilder.Build(fromStart)
//}
//
//func TestACLTreeBuilder_UserJoinCorrectHeadsAndLen(t *testing.T) {
// thread, err := threadbuilder.NewThreadBuilderWithTestName("threadbuilder/userjoinexample.yml")
// if err != nil {
// t.Fatal(err)
// }
//
// res, err := createTreeFromThread(thread)
// if err != nil {
// t.Fatalf("build Tree should not result in an error: %v", res)
// }
//
// assert.equal(t, res.Heads(), []string{"C.1.1"})
// assert.equal(t, res.Len(), 4)
//}
//
//func TestTreeBuilder_UserJoinTestTreeIterate(t *testing.T) {
// thread, err := threadbuilder.NewThreadBuilderWithTestName("threadbuilder/userjoinexample.yml")
// if err != nil {
// t.Fatal(err)
// }
//
// res, err := createTreeFromThread(thread)
// if err != nil {
// t.Fatalf("build Tree should not result in an error: %v", res)
// }
//
// assert.equal(t, res.Heads(), []string{"C.1.1"})
// assert.equal(t, res.Len(), 4)
// var changeIds []string
// res.iterate(res.root, func(c *Change) (isContinue bool) {
// changeIds = append(changeIds, c.Id)
// return true
// })
// assert.equal(t, changeIds, []string{"A.1.1", "A.1.2", "B.1.1", "C.1.1"})
//}
//
//func TestTreeBuilder_UserRemoveTestTreeIterate(t *testing.T) {
// thread, err := threadbuilder.NewThreadBuilderWithTestName("threadbuilder/userremoveexample.yml")
// if err != nil {
// t.Fatal(err)
// }
//
// res, err := createTreeFromThread(thread)
// if err != nil {
// t.Fatalf("build Tree should not result in an error: %v", res)
// }
//
// assert.equal(t, res.Heads(), []string{"A.1.3"})
// assert.equal(t, res.Len(), 4)
// var changeIds []string
// res.iterate(res.root, func(c *Change) (isContinue bool) {
// changeIds = append(changeIds, c.Id)
// return true
// })
// assert.equal(t, changeIds, []string{"A.1.1", "A.1.2", "B.1.1", "A.1.3"})
//}

158
pkg/acl/acltree/treeiterator.go
View File

@ -1,158 +0,0 @@
package acltree
import "sync"
var itPool = &sync.Pool{
New: func() interface{} {
return &iterator{}
},
}
func newIterator() *iterator {
return itPool.Get().(*iterator)
}
func freeIterator(i *iterator) {
itPool.Put(i)
}
type iterator struct {
compBuf []*Change
queue []*Change
doneMap map[*Change]struct{}
breakpoint *Change
f func(c *Change) bool
}
func (i *iterator) iterateSkip(start *Change, skipBefore *Change, f func(c *Change) (isContinue bool)) {
skipping := true
i.iterate(start, func(c *Change) (isContinue bool) {
if skipping && c != skipBefore {
return true
}
skipping = false
return f(c)
})
}
func (i *iterator) iterate(start *Change, f func(c *Change) (isContinue bool)) {
if start == nil {
return
}
// reset
i.queue = i.queue[:0]
i.compBuf = i.compBuf[:0]
i.doneMap = make(map[*Change]struct{})
i.queue = append(i.queue, start)
i.breakpoint = nil
i.f = f
for len(i.queue) > 0 {
c := i.queue[0]
i.queue = i.queue[1:]
nl := len(c.Next)
if nl == 1 {
if !i.iterateLin(c) {
return
}
if i.breakpoint != nil {
i.toQueue(i.breakpoint)
i.breakpoint = nil
}
} else {
_, done := i.doneMap[c]
if !done {
if !f(c) {
return
}
i.doneMap[c] = struct{}{}
}
if nl != 0 {
for _, next := range c.Next {
i.toQueue(next)
}
}
}
}
}
func (i *iterator) iterateLin(c *Change) bool {
for len(c.Next) == 1 {
_, done := i.doneMap[c]
if !done {
if !i.f(c) {
return false
}
i.doneMap[c] = struct{}{}
}
c = c.Next[0]
if len(c.PreviousIds) > 1 {
break
}
}
if len(c.Next) == 0 && len(c.PreviousIds) <= 1 {
if !i.f(c) {
return false
}
i.doneMap[c] = struct{}{}
} else {
i.breakpoint = c
}
return true
}
func (i *iterator) comp(c1, c2 *Change) uint8 {
if c1.Id == c2.Id {
return 0
}
i.compBuf = i.compBuf[:0]
i.compBuf = append(i.compBuf, c1.Next...)
var uniq = make(map[*Change]struct{})
var appendUniqueToBuf = func(next []*Change) {
for _, n := range next {
if _, ok := uniq[n]; !ok {
i.compBuf = append(i.compBuf, n)
uniq[n] = struct{}{}
}
}
}
var used int
for len(i.compBuf)-used > 0 {
l := len(i.compBuf) - used
for _, n := range i.compBuf[used:] {
delete(uniq, n)
if n.Id == c2.Id {
return 1
} else {
appendUniqueToBuf(n.Next)
}
}
used += l
}
return 2
}
func (i *iterator) toQueue(c *Change) {
var pos = -1
For:
for idx, qc := range i.queue {
switch i.comp(c, qc) {
// exists
case 0:
return
//
case 1:
pos = idx
break For
}
}
if pos == -1 {
i.queue = append(i.queue, c)
} else if pos == 0 {
i.queue = append([]*Change{c}, i.queue...)
} else {
i.queue = append(i.queue[:pos], append([]*Change{c}, i.queue[pos:]...)...)
}
}

169
pkg/acl/example/plaintextdocument/document.go
View File

@ -1,169 +0,0 @@
package plaintextdocument
import (
"context"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
aclpb "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/acltree"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/testchanges/testchangepb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage"
"github.com/gogo/protobuf/proto"
)
type PlainTextDocument interface {
Text() string
AddText(ctx context.Context, text string) error
}
// TODO: this struct is not thread-safe, so use it wisely :-)
type plainTextDocument struct {
heads []string
aclTree acltree.ACLTree
state *DocumentState
}
func (p *plainTextDocument) Text() string {
if p.state != nil {
return p.state.Text
}
return ""
}
func (p *plainTextDocument) AddText(ctx context.Context, text string) error {
_, err := p.aclTree.AddContent(ctx, func(builder acltree.ChangeBuilder) error {
builder.AddChangeContent(
&testchangepb.PlainTextChangeData{
Content: []*testchangepb.PlainTextChangeContent{
createAppendTextChangeContent(text),
},
})
return nil
})
return err
}
func (p *plainTextDocument) Update(tree acltree.ACLTree) {
p.aclTree = tree
var err error
defer func() {
if err != nil {
fmt.Println("rebuild has returned error:", err)
}
}()
prevHeads := p.heads
p.heads = tree.Heads()
startId := prevHeads[0]
tree.IterateFrom(startId, func(change *acltree.Change) (isContinue bool) {
if change.Id == startId {
return true
}
if change.DecryptedDocumentChange != nil {
p.state, err = p.state.ApplyChange(change.DecryptedDocumentChange, change.Id)
if err != nil {
return false
}
}
return true
})
}
func (p *plainTextDocument) Rebuild(tree acltree.ACLTree) {
p.aclTree = tree
p.heads = tree.Heads()
var startId string
var err error
defer func() {
if err != nil {
fmt.Println("rebuild has returned error:", err)
}
}()
rootChange := tree.Root()
if rootChange.DecryptedDocumentChange == nil {
err = fmt.Errorf("root doesn't have decrypted change")
return
}
state, err := BuildDocumentStateFromChange(rootChange.DecryptedDocumentChange, rootChange.Id)
if err != nil {
return
}
startId = rootChange.Id
tree.Iterate(func(change *acltree.Change) (isContinue bool) {
if startId == change.Id {
return true
}
if change.DecryptedDocumentChange != nil {
state, err = state.ApplyChange(change.DecryptedDocumentChange, change.Id)
if err != nil {
return false
}
}
return true
})
if err != nil {
return
}
p.state = state
}
func NewInMemoryPlainTextDocument(acc *account.AccountData, text string) (PlainTextDocument, error) {
return NewPlainTextDocument(acc, treestorage.NewInMemoryTreeStorage, text)
}
func NewPlainTextDocument(
acc *account.AccountData,
create treestorage.CreatorFunc,
text string) (PlainTextDocument, error) {
changeBuilder := func(builder acltree.ChangeBuilder) error {
err := builder.UserAdd(acc.Identity, acc.EncKey.GetPublic(), aclpb.ACLChange_Admin)
if err != nil {
return err
}
builder.AddChangeContent(createInitialChangeContent(text))
return nil
}
t, err := acltree.CreateNewTreeStorageWithACL(
acc,
changeBuilder,
create)
if err != nil {
return nil, err
}
doc := &plainTextDocument{
heads: nil,
aclTree: nil,
state: nil,
}
tree, err := acltree.BuildACLTree(t, acc, doc)
if err != nil {
return nil, err
}
doc.aclTree = tree
return doc, nil
}
func createInitialChangeContent(text string) proto.Marshaler {
return &testchangepb.PlainTextChangeData{
Content: []*testchangepb.PlainTextChangeContent{
createAppendTextChangeContent(text),
},
Snapshot: &testchangepb.PlainTextChangeSnapshot{Text: text},
}
}
func createAppendTextChangeContent(text string) *testchangepb.PlainTextChangeContent {
return &testchangepb.PlainTextChangeContent{
Value: &testchangepb.PlainTextChangeContentValueOfTextAppend{
TextAppend: &testchangepb.PlainTextChangeTextAppend{
Text: text,
},
},
}
}

58
pkg/acl/example/plaintextdocument/document_test.go
View File

@ -1,58 +0,0 @@
package plaintextdocument
import (
"context"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/treestoragebuilder"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"github.com/stretchr/testify/assert"
"testing"
)
func TestDocument_NewPlainTextDocument(t *testing.T) {
keychain := treestoragebuilder.NewKeychain()
keychain.AddSigningKey("A")
keychain.AddEncryptionKey("A")
data := &account.AccountData{
Identity: keychain.GetIdentity("A"),
SignKey: keychain.SigningKeys["A"],
EncKey: keychain.EncryptionKeys["A"],
Decoder: signingkey.NewEd25519PubKeyDecoder(),
}
doc, err := NewPlainTextDocument(data, treestorage.NewInMemoryTreeStorage, "Some text")
if err != nil {
t.Fatalf("should not create document with error: %v", err)
}
assert.Equal(t, doc.Text(), "Some text")
}
func TestDocument_PlainTextDocument_AddText(t *testing.T) {
keychain := treestoragebuilder.NewKeychain()
keychain.AddSigningKey("A")
keychain.AddEncryptionKey("A")
data := &account.AccountData{
Identity: keychain.GetIdentity("A"),
SignKey: keychain.SigningKeys["A"],
EncKey: keychain.EncryptionKeys["A"],
Decoder: signingkey.NewEd25519PubKeyDecoder(),
}
doc, err := NewPlainTextDocument(data, treestorage.NewInMemoryTreeStorage, "Some text")
if err != nil {
t.Fatalf("should not create document with error: %v", err)
}
err = doc.AddText(context.Background(), "Next")
if err != nil {
t.Fatalf("should be able to add document: %v", err)
}
assert.Equal(t, doc.Text(), "Some text|Next")
err = doc.AddText(context.Background(), "Shmext")
if err != nil {
t.Fatalf("should be able to add document: %v", err)
}
assert.Equal(t, doc.Text(), "Some text|Next|Shmext")
}

59
pkg/acl/example/plaintextdocument/plaintextdocstate.go
View File

@ -1,59 +0,0 @@
package plaintextdocument
import (
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/testchanges/testchangepb"
"github.com/gogo/protobuf/proto"
)
type DocumentState struct {
LastChangeId string
Text string
}
func NewDocumentState(text string, id string) *DocumentState {
return &DocumentState{
LastChangeId: id,
Text: text,
}
}
func BuildDocumentStateFromChange(change []byte, id string) (*DocumentState, error) {
var changesData testchangepb.PlainTextChangeData
err := proto.Unmarshal(change, &changesData)
if err != nil {
return nil, err
}
if changesData.GetSnapshot() == nil {
return nil, fmt.Errorf("could not create state from empty snapshot")
}
return NewDocumentState(changesData.GetSnapshot().GetText(), id), nil
}
func (p *DocumentState) ApplyChange(change []byte, id string) (*DocumentState, error) {
var changesData testchangepb.PlainTextChangeData
err := proto.Unmarshal(change, &changesData)
if err != nil {
return nil, err
}
for _, content := range changesData.GetContent() {
err = p.applyChange(content)
if err != nil {
return nil, err
}
}
p.LastChangeId = id
return p, nil
}
func (p *DocumentState) applyChange(ch *testchangepb.PlainTextChangeContent) error {
switch {
case ch.GetTextAppend() != nil:
text := ch.GetTextAppend().GetText()
p.Text += "|" + text
}
return nil
}

183
pkg/acl/acltree/aclstate.go → pkg/acl/list/aclstate.go
View File

@ -1,135 +1,181 @@
package acltree
package list
import (
"bytes"
"errors"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app/logger"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/encryptionkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/symmetric"
"github.com/gogo/protobuf/proto"
"go.uber.org/zap"
"hash/fnv"
)
var log = logger.NewNamed("acllist").Sugar()
var ErrNoSuchUser = errors.New("no such user")
var ErrFailedToDecrypt = errors.New("failed to decrypt key")
var ErrUserRemoved = errors.New("user was removed from the document")
var ErrDocumentForbidden = errors.New("your user was forbidden access to the document")
var ErrUserAlreadyExists = errors.New("user already exists")
var ErrNoSuchRecord = errors.New("no such record")
var ErrInsufficientPermissions = errors.New("insufficient permissions")
var ErrNoReadKey = errors.New("acl state doesn't have a read key")
type UserPermissionPair struct {
Identity string
Permission aclpb.ACLChangeUserPermissions
}
type ACLState struct {
currentReadKeyHash uint64
userReadKeys map[uint64]*symmetric.Key
userStates map[string]*aclpb.ACLChangeUserState
userInvites map[string]*aclpb.ACLChangeUserInvite
signingPubKeyDecoder signingkey.PubKeyDecoder
signingPubKeyDecoder keys.Decoder
encryptionKey encryptionkey.PrivKey
identity string
permissionsAtRecord map[string][]UserPermissionPair
}
func newACLState(
func newACLStateWithIdentity(
identity string,
encryptionKey encryptionkey.PrivKey,
signingPubKeyDecoder signingkey.PubKeyDecoder) *ACLState {
decoder keys.Decoder) *ACLState {
return &ACLState{
identity: identity,
encryptionKey: encryptionKey,
userReadKeys: make(map[uint64]*symmetric.Key),
userStates: make(map[string]*aclpb.ACLChangeUserState),
userInvites: make(map[string]*aclpb.ACLChangeUserInvite),
signingPubKeyDecoder: signingPubKeyDecoder,
signingPubKeyDecoder: decoder,
permissionsAtRecord: make(map[string][]UserPermissionPair),
}
}
func newACLStateFromSnapshotChange(
snapshotChange *aclpb.ACLChange,
identity string,
encryptionKey encryptionkey.PrivKey,
signingPubKeyDecoder signingkey.PubKeyDecoder) (*ACLState, error) {
st := &ACLState{
identity: identity,
encryptionKey: encryptionKey,
func newACLState(decoder keys.Decoder) *ACLState {
return &ACLState{
signingPubKeyDecoder: decoder,
userReadKeys: make(map[uint64]*symmetric.Key),
userStates: make(map[string]*aclpb.ACLChangeUserState),
userInvites: make(map[string]*aclpb.ACLChangeUserInvite),
signingPubKeyDecoder: signingPubKeyDecoder,
permissionsAtRecord: make(map[string][]UserPermissionPair),
}
err := st.recreateFromSnapshotChange(snapshotChange)
if err != nil {
return nil, err
}
return st, nil
}
func (st *ACLState) recreateFromSnapshotChange(snapshotChange *aclpb.ACLChange) error {
snapshot := snapshotChange.GetAclData().GetAclSnapshot()
if snapshot == nil {
return fmt.Errorf("could not create state from snapshot, because it is nil")
}
state := snapshot.AclState
for _, userState := range state.UserStates {
st.userStates[userState.Identity] = userState
}
func (st *ACLState) CurrentReadKeyHash() uint64 {
return st.currentReadKeyHash
}
userState, exists := st.userStates[st.identity]
func (st *ACLState) CurrentReadKey() (*symmetric.Key, error) {
key, exists := st.userReadKeys[st.currentReadKeyHash]
if !exists {
return ErrNoSuchUser
return nil, ErrNoReadKey
}
for _, key := range userState.EncryptedReadKeys {
key, hash, err := st.decryptReadKeyAndHash(key)
if err != nil {
return ErrFailedToDecrypt
return key, nil
}
func (st *ACLState) UserReadKeys() map[uint64]*symmetric.Key {
return st.userReadKeys
}
func (st *ACLState) PermissionsAtRecord(id string, identity string) (UserPermissionPair, error) {
permissions, ok := st.permissionsAtRecord[id]
if !ok {
log.Errorf("missing record at id %s", id)
return UserPermissionPair{}, ErrNoSuchRecord
}
for _, perm := range permissions {
if perm.Identity == identity {
return perm, nil
}
st.userReadKeys[hash] = key
}
st.currentReadKeyHash = snapshotChange.CurrentReadKeyHash
if snapshot.GetAclState().GetInvites() != nil {
st.userInvites = snapshot.GetAclState().GetInvites()
}
return nil
return UserPermissionPair{}, ErrNoSuchUser
}
func (st *ACLState) makeSnapshot() *aclpb.ACLChangeACLSnapshot {
var userStates []*aclpb.ACLChangeUserState
for _, st := range st.userStates {
userStates = append(userStates, st)
func (st *ACLState) applyRecord(record *aclpb.Record) (err error) {
// TODO: this should be probably changed
aclData := &aclpb.ACLChangeACLData{}
err = proto.Unmarshal(record.Data, aclData)
if err != nil {
return
}
return &aclpb.ACLChangeACLSnapshot{AclState: &aclpb.ACLChangeACLState{
ReadKeyHashes: nil,
UserStates: userStates, // TODO: make states and invites in same format
Invites: st.userInvites,
}}
}
func (st *ACLState) applyChange(change *aclpb.ACLChange) (err error) {
defer func() {
if err != nil {
return
}
st.currentReadKeyHash = change.CurrentReadKeyHash
st.currentReadKeyHash = record.CurrentReadKeyHash
}()
return st.applyChangeData(aclData, record.CurrentReadKeyHash, record.Identity)
}
func (st *ACLState) applyChangeAndUpdate(recordWrapper *Record) (err error) {
change := recordWrapper.Content
aclData := &aclpb.ACLChangeACLData{}
if recordWrapper.ParsedModel != nil {
aclData = recordWrapper.ParsedModel.(*aclpb.ACLChangeACLData)
} else {
err = proto.Unmarshal(change.Data, aclData)
if err != nil {
return
}
recordWrapper.ParsedModel = aclData
}
err = st.applyChangeData(aclData, recordWrapper.Content.CurrentReadKeyHash, recordWrapper.Content.Identity)
if err != nil {
return err
}
var permissions []UserPermissionPair
for _, state := range st.userStates {
permission := UserPermissionPair{
Identity: state.Identity,
Permission: state.Permissions,
}
permissions = append(permissions, permission)
}
st.permissionsAtRecord[recordWrapper.Id] = permissions
log.Infof("adding permissions at record %s", recordWrapper.Id)
return nil
}
func (st *ACLState) applyChangeData(changeData *aclpb.ACLChangeACLData, hash uint64, identity string) (err error) {
defer func() {
if err != nil {
return
}
st.currentReadKeyHash = hash
}()
// we can't check this for the user which is joining, because it will not be in our list
// the same is for the first change to be added
skipIdentityCheck := st.isUserJoin(change) || (st.currentReadKeyHash == 0 && st.isUserAdd(change))
skipIdentityCheck := st.isUserJoin(changeData) || (st.currentReadKeyHash == 0 && st.isUserAdd(changeData, identity))
if !skipIdentityCheck {
// we check signature when we add this to the Tree, so no need to do it here
if _, exists := st.userStates[change.Identity]; !exists {
if _, exists := st.userStates[identity]; !exists {
err = ErrNoSuchUser
return
}
if !st.hasPermission(change.Identity, aclpb.ACLChange_Admin) {
err = fmt.Errorf("user %s must have admin permissions", change.Identity)
if !st.hasPermission(identity, aclpb.ACLChange_Admin) {
err = fmt.Errorf("user %s must have admin permissions", identity)
return
}
}
for _, ch := range change.GetAclData().GetAclContent() {
for _, ch := range changeData.GetAclContent() {
if err = st.applyChangeContent(ch); err != nil {
//log.Infof("error while applying changes: %v; ignore", err)
log.Info("error while applying changes: %v; ignore", zap.Error(err))
return err
}
}
@ -137,7 +183,6 @@ func (st *ACLState) applyChange(change *aclpb.ACLChange) (err error) {
return nil
}
// TODO: remove changeId, because it is not needed
func (st *ACLState) applyChangeContent(ch *aclpb.ACLChangeACLContentValue) error {
switch {
case ch.GetUserPermissionChange() != nil:
@ -193,7 +238,7 @@ func (st *ACLState) applyUserJoin(ch *aclpb.ACLChangeUserJoin) error {
return fmt.Errorf("failed to decode signing identity as bytes")
}
res, err := verificationKey.Verify(rawSignedId, signature)
res, err := verificationKey.(signingkey.PubKey).Verify(rawSignedId, signature)
if err != nil {
return fmt.Errorf("verification returned error: %w", err)
}
@ -318,15 +363,15 @@ func (st *ACLState) hasPermission(identity string, permission aclpb.ACLChangeUse
return state.Permissions == permission
}
func (st *ACLState) isUserJoin(ch *aclpb.ACLChange) bool {
func (st *ACLState) isUserJoin(data *aclpb.ACLChangeACLData) bool {
// if we have a UserJoin, then it should always be the first one applied
return ch.AclData.GetAclContent() != nil && ch.AclData.GetAclContent()[0].GetUserJoin() != nil
return data.GetAclContent() != nil && data.GetAclContent()[0].GetUserJoin() != nil
}
func (st *ACLState) isUserAdd(ch *aclpb.ACLChange) bool {
func (st *ACLState) isUserAdd(data *aclpb.ACLChangeACLData, identity string) bool {
// if we have a UserAdd, then it should always be the first one applied
userAdd := ch.AclData.GetAclContent()[0].GetUserAdd()
return ch.AclData.GetAclContent() != nil && userAdd != nil && userAdd.GetIdentity() == ch.Identity
userAdd := data.GetAclContent()[0].GetUserAdd()
return data.GetAclContent() != nil && userAdd != nil && userAdd.GetIdentity() == identity
}
func (st *ACLState) getPermissionDecreasedUsers(ch *aclpb.ACLChange) (identities []*aclpb.ACLChangeUserPermissionChange) {
@ -409,3 +454,7 @@ func (st *ACLState) GetUserStates() map[string]*aclpb.ACLChangeUserState {
// TODO: we should provide better API that would not allow to change this map from the outside
return st.userStates
}
func (st *ACLState) isNodeIdentity() bool {
return st.identity == ""
}

48
pkg/acl/list/aclstatebuilder.go Normal file
View File

@ -0,0 +1,48 @@
package list
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/encryptionkey"
)
type aclStateBuilder struct {
identity string
key encryptionkey.PrivKey
decoder keys.Decoder
}
func newACLStateBuilderWithIdentity(decoder keys.Decoder, accountData *account.AccountData) *aclStateBuilder {
return &aclStateBuilder{
decoder: decoder,
identity: accountData.Identity,
key: accountData.EncKey,
}
}
func newACLStateBuilder(decoder keys.Decoder) *aclStateBuilder {
return &aclStateBuilder{
decoder: decoder,
}
}
func (sb *aclStateBuilder) Build(records []*Record) (*ACLState, error) {
var (
err error
state *ACLState
)
if sb.key != nil {
state = newACLStateWithIdentity(sb.identity, sb.key, sb.decoder)
} else {
state = newACLState(sb.decoder)
}
for _, rec := range records {
err = state.applyChangeAndUpdate(rec)
if err != nil {
return nil, err
}
}
return state, err
}

86
pkg/acl/acltree/changebuilder.go → pkg/acl/list/changebuilder.go
View File

@ -1,4 +1,4 @@
package acltree
package list
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
@ -15,35 +15,27 @@ type MarshalledChange = []byte
type ACLChangeBuilder interface {
UserAdd(identity string, encryptionKey encryptionkey.PubKey, permissions aclpb.ACLChangeUserPermissions) error
AddId(id string) // TODO: this is only for testing
SetMakeSnapshot(bool) // TODO: who should decide this? probably ACLTree so we can delete it
AddId(id string) // TODO: this is only for testing
}
type ChangeBuilder interface {
ACLChangeBuilder
AddChangeContent(marshaler proto.Marshaler) // user code should be responsible for making regular snapshots
}
type changeBuilder struct {
type aclChangeBuilder struct {
aclState *ACLState
tree *Tree
list ACLList
acc *account.AccountData
aclData *aclpb.ACLChangeACLData
changeContent proto.Marshaler
id string
makeSnapshot bool
readKey *symmetric.Key
readKeyHash uint64
aclData *aclpb.ACLChangeACLData
id string
readKey *symmetric.Key
readKeyHash uint64
}
func newChangeBuilder() *changeBuilder {
return &changeBuilder{}
func newACLChangeBuilder() *aclChangeBuilder {
return &aclChangeBuilder{}
}
func (c *changeBuilder) Init(state *ACLState, tree *Tree, acc *account.AccountData) {
func (c *aclChangeBuilder) Init(state *ACLState, list ACLList, acc *account.AccountData) {
c.aclState = state
c.tree = tree
c.list = list
c.acc = acc
c.aclData = &aclpb.ACLChangeACLData{}
@ -60,15 +52,11 @@ func (c *changeBuilder) Init(state *ACLState, tree *Tree, acc *account.AccountDa
}
}
func (c *changeBuilder) AddId(id string) {
func (c *aclChangeBuilder) AddId(id string) {
c.id = id
}
func (c *changeBuilder) SetMakeSnapshot(b bool) {
c.makeSnapshot = b
}
func (c *changeBuilder) UserAdd(identity string, encryptionKey encryptionkey.PubKey, permissions aclpb.ACLChangeUserPermissions) error {
func (c *aclChangeBuilder) UserAdd(identity string, encryptionKey encryptionkey.PubKey, permissions aclpb.ACLChangeUserPermissions) error {
var allKeys []*symmetric.Key
if c.aclState.currentReadKeyHash != 0 {
for _, key := range c.aclState.userReadKeys {
@ -105,41 +93,25 @@ func (c *changeBuilder) UserAdd(identity string, encryptionKey encryptionkey.Pub
return nil
}
func (c *changeBuilder) BuildAndApply() (*Change, []byte, error) {
aclChange := &aclpb.ACLChange{
TreeHeadIds: c.tree.Heads(),
AclHeadIds: c.tree.ACLHeads(),
SnapshotBaseId: c.tree.RootId(),
AclData: c.aclData,
func (c *aclChangeBuilder) BuildAndApply() (*Record, []byte, error) {
aclRecord := &aclpb.Record{
PrevId: c.list.Head().Id,
CurrentReadKeyHash: c.readKeyHash,
Timestamp: int64(time.Now().Nanosecond()),
Identity: c.acc.Identity,
}
err := c.aclState.applyChange(aclChange)
marshalledData, err := proto.Marshal(c.aclData)
if err != nil {
return nil, nil, err
}
aclRecord.Data = marshalledData
err = c.aclState.applyRecord(aclRecord)
if err != nil {
return nil, nil, err
}
if c.makeSnapshot {
c.aclData.AclSnapshot = c.aclState.makeSnapshot()
}
var marshalled []byte
if c.changeContent != nil {
marshalled, err = c.changeContent.Marshal()
if err != nil {
return nil, nil, err
}
encrypted, err := c.aclState.userReadKeys[c.aclState.currentReadKeyHash].
Encrypt(marshalled)
if err != nil {
return nil, nil, err
}
aclChange.ChangesData = encrypted
}
fullMarshalledChange, err := proto.Marshal(aclChange)
fullMarshalledChange, err := proto.Marshal(aclRecord)
if err != nil {
return nil, nil, err
}
@ -151,13 +123,9 @@ func (c *changeBuilder) BuildAndApply() (*Change, []byte, error) {
if err != nil {
return nil, nil, err
}
ch := NewChange(id, aclChange)
ch.DecryptedDocumentChange = marshalled
ch := NewRecord(id, aclRecord)
ch.ParsedModel = c.aclData
ch.Sign = signature
return ch, fullMarshalledChange, nil
}
func (c *changeBuilder) AddChangeContent(marshaler proto.Marshaler) {
c.changeContent = marshaler
}

179
pkg/acl/list/list.go Normal file
View File

@ -0,0 +1,179 @@
package list
import (
"context"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
"go.uber.org/zap"
"sync"
)
type IterFunc = func(record *Record) (IsContinue bool)
type RWLocker interface {
sync.Locker
RLock()
RUnlock()
}
type ACLList interface {
RWLocker
ID() string
Header() *aclpb.Header
Records() []*Record
ACLState() *ACLState
IsAfter(first string, second string) (bool, error)
Head() *Record
Get(id string) (*Record, error)
Iterate(iterFunc IterFunc)
IterateFrom(startId string, iterFunc IterFunc)
Close() (err error)
}
type aclList struct {
header *aclpb.Header
records []*Record
indexes map[string]int
id string
builder *aclStateBuilder
aclState *ACLState
sync.RWMutex
}
func BuildACLListWithIdentity(acc *account.AccountData, storage storage.ListStorage) (ACLList, error) {
builder := newACLStateBuilderWithIdentity(acc.Decoder, acc)
return buildWithACLStateBuilder(builder, storage)
}
func BuildACLList(decoder keys.Decoder, storage storage.ListStorage) (ACLList, error) {
return buildWithACLStateBuilder(newACLStateBuilder(decoder), storage)
}
func buildWithACLStateBuilder(builder *aclStateBuilder, storage storage.ListStorage) (ACLList, error) {
header, err := storage.Header()
if err != nil {
return nil, err
}
id, err := storage.ID()
if err != nil {
return nil, err
}
rawRecord, err := storage.Head()
if err != nil {
return nil, err
}
record, err := NewFromRawRecord(rawRecord)
if err != nil {
return nil, err
}
records := []*Record{record}
for record.Content.PrevId != "" {
rawRecord, err = storage.GetRawRecord(context.Background(), record.Content.PrevId)
if err != nil {
return nil, err
}
record, err = NewFromRawRecord(rawRecord)
if err != nil {
return nil, err
}
records = append(records, record)
}
indexes := make(map[string]int)
for i, j := 0, len(records)-1; i < j; i, j = i+1, j-1 {
records[i], records[j] = records[j], records[i]
indexes[records[i].Id] = i
indexes[records[j].Id] = j
}
// adding missed index if needed
if len(records)%2 != 0 {
indexes[records[len(records)/2].Id] = len(records) / 2
}
log.With(zap.String("head id", records[len(records)-1].Id), zap.String("list id", id)).
Info("building acl tree")
state, err := builder.Build(records)
if err != nil {
return nil, err
}
return &aclList{
header: header,
records: records,
indexes: indexes,
builder: builder,
aclState: state,
id: id,
RWMutex: sync.RWMutex{},
}, nil
}
func (a *aclList) Records() []*Record {
return a.records
}
func (a *aclList) ID() string {
return a.id
}
func (a *aclList) Header() *aclpb.Header {
return a.header
}
func (a *aclList) ACLState() *ACLState {
return a.aclState
}
func (a *aclList) IsAfter(first string, second string) (bool, error) {
firstRec, okFirst := a.indexes[first]
secondRec, okSecond := a.indexes[second]
if !okFirst || !okSecond {
return false, fmt.Errorf("not all entries are there: first (%t), second (%t)", okFirst, okSecond)
}
return firstRec >= secondRec, nil
}
func (a *aclList) Head() *Record {
return a.records[len(a.records)-1]
}
func (a *aclList) Get(id string) (*Record, error) {
recIdx, ok := a.indexes[id]
if !ok {
return nil, fmt.Errorf("no such record")
}
return a.records[recIdx], nil
}
func (a *aclList) Iterate(iterFunc IterFunc) {
for _, rec := range a.records {
if !iterFunc(rec) {
return
}
}
}
func (a *aclList) IterateFrom(startId string, iterFunc IterFunc) {
recIdx, ok := a.indexes[startId]
if !ok {
return
}
for i := recIdx; i < len(a.records); i++ {
if !iterFunc(a.records[i]) {
return
}
}
}
func (a *aclList) Close() (err error) {
return nil
}

92
pkg/acl/list/list_test.go Normal file
View File

@ -0,0 +1,92 @@
package list
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/acllistbuilder"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing"
)
func TestAclList_ACLState_UserInviteAndJoin(t *testing.T) {
st, err := acllistbuilder.NewListStorageWithTestName("userjoinexample.yml")
require.NoError(t, err, "building storage should not result in error")
keychain := st.(*acllistbuilder.ACLListStorageBuilder).GetKeychain()
aclList, err := BuildACLList(signingkey.NewEDPubKeyDecoder(), st)
require.NoError(t, err, "building acl list should be without error")
idA := keychain.GetIdentity("A")
idB := keychain.GetIdentity("B")
idC := keychain.GetIdentity("C")
// checking final state
assert.Equal(t, aclpb.ACLChange_Admin, aclList.ACLState().GetUserStates()[idA].Permissions)
assert.Equal(t, aclpb.ACLChange_Writer, aclList.ACLState().GetUserStates()[idB].Permissions)
assert.Equal(t, aclpb.ACLChange_Reader, aclList.ACLState().GetUserStates()[idC].Permissions)
assert.Equal(t, aclList.Head().Content.CurrentReadKeyHash, aclList.ACLState().CurrentReadKeyHash())
var records []*Record
aclList.Iterate(func(record *Record) (IsContinue bool) {
records = append(records, record)
return true
})
// checking permissions at specific records
assert.Equal(t, 3, len(records))
_, err = aclList.ACLState().PermissionsAtRecord(records[1].Id, idB)
assert.Error(t, err, "B should have no permissions at record 1")
perm, err := aclList.ACLState().PermissionsAtRecord(records[2].Id, idB)
assert.NoError(t, err, "should have no error with permissions of B in the record 2")
assert.Equal(t, UserPermissionPair{
Identity: idB,
Permission: aclpb.ACLChange_Writer,
}, perm)
}
func TestAclList_ACLState_UserJoinAndRemove(t *testing.T) {
st, err := acllistbuilder.NewListStorageWithTestName("userremoveexample.yml")
require.NoError(t, err, "building storage should not result in error")
keychain := st.(*acllistbuilder.ACLListStorageBuilder).GetKeychain()
aclList, err := BuildACLList(signingkey.NewEDPubKeyDecoder(), st)
require.NoError(t, err, "building acl list should be without error")
idA := keychain.GetIdentity("A")
idB := keychain.GetIdentity("B")
idC := keychain.GetIdentity("C")
// checking final state
assert.Equal(t, aclpb.ACLChange_Admin, aclList.ACLState().GetUserStates()[idA].Permissions)
assert.Equal(t, aclpb.ACLChange_Reader, aclList.ACLState().GetUserStates()[idC].Permissions)
assert.Equal(t, aclList.Head().Content.CurrentReadKeyHash, aclList.ACLState().CurrentReadKeyHash())
_, exists := aclList.ACLState().GetUserStates()[idB]
assert.Equal(t, false, exists)
var records []*Record
aclList.Iterate(func(record *Record) (IsContinue bool) {
records = append(records, record)
return true
})
// checking permissions at specific records
assert.Equal(t, 4, len(records))
assert.NotEqual(t, records[2].Content.CurrentReadKeyHash, aclList.ACLState().CurrentReadKeyHash())
perm, err := aclList.ACLState().PermissionsAtRecord(records[2].Id, idB)
assert.NoError(t, err, "should have no error with permissions of B in the record 2")
assert.Equal(t, UserPermissionPair{
Identity: idB,
Permission: aclpb.ACLChange_Writer,
}, perm)
_, err = aclList.ACLState().PermissionsAtRecord(records[3].Id, idB)
assert.Error(t, err, "B should have no permissions at record 3, because user should be removed")
}

34
pkg/acl/list/record.go Normal file
View File

@ -0,0 +1,34 @@
package list
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/gogo/protobuf/proto"
)
type Record struct {
Id string
Content *aclpb.Record
ParsedModel interface{}
Sign []byte
}
func NewRecord(id string, aclRecord *aclpb.Record) *Record {
return &Record{
Id: id,
Content: aclRecord,
}
}
func NewFromRawRecord(rawRec *aclpb.RawRecord) (*Record, error) {
aclRec := &aclpb.Record{}
err := proto.Unmarshal(rawRec.Payload, aclRec)
if err != nil {
return nil, err
}
return &Record{
Id: rawRec.Id,
Content: aclRec,
Sign: rawRec.Signature,
}, nil
}

191
pkg/acl/storage/inmemory.go Normal file
View File

@ -0,0 +1,191 @@
package storage
import (
"context"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"sync"
)
type inMemoryACLListStorage struct {
header *aclpb.Header
records []*aclpb.RawRecord
id string
sync.RWMutex
}
func NewInMemoryACLListStorage(
id string,
header *aclpb.Header,
records []*aclpb.RawRecord) (ListStorage, error) {
return &inMemoryACLListStorage{
id: id,
header: header,
records: records,
RWMutex: sync.RWMutex{},
}, nil
}
func (i *inMemoryACLListStorage) Header() (*aclpb.Header, error) {
i.RLock()
defer i.RUnlock()
return i.header, nil
}
func (i *inMemoryACLListStorage) Head() (*aclpb.RawRecord, error) {
i.RLock()
defer i.RUnlock()
return i.records[len(i.records)-1], nil
}
func (i *inMemoryACLListStorage) GetRawRecord(ctx context.Context, id string) (*aclpb.RawRecord, error) {
i.RLock()
defer i.RUnlock()
for _, rec := range i.records {
if rec.Id == id {
return rec, nil
}
}
return nil, fmt.Errorf("no such record")
}
func (i *inMemoryACLListStorage) AddRawRecord(ctx context.Context, rec *aclpb.RawRecord) error {
panic("implement me")
}
func (i *inMemoryACLListStorage) ID() (string, error) {
i.RLock()
defer i.RUnlock()
return i.id, nil
}
type inMemoryTreeStorage struct {
id string
header *aclpb.Header
heads []string
changes map[string]*aclpb.RawChange
sync.RWMutex
}
func NewInMemoryTreeStorage(
treeId string,
header *aclpb.Header,
heads []string,
changes []*aclpb.RawChange) (TreeStorage, error) {
allChanges := make(map[string]*aclpb.RawChange)
for _, ch := range changes {
allChanges[ch.Id] = ch
}
return &inMemoryTreeStorage{
id: treeId,
header: header,
heads: heads,
changes: allChanges,
RWMutex: sync.RWMutex{},
}, nil
}
func (t *inMemoryTreeStorage) ID() (string, error) {
t.RLock()
defer t.RUnlock()
return t.id, nil
}
func (t *inMemoryTreeStorage) Header() (*aclpb.Header, error) {
t.RLock()
defer t.RUnlock()
return t.header, nil
}
func (t *inMemoryTreeStorage) Heads() ([]string, error) {
t.RLock()
defer t.RUnlock()
return t.heads, nil
}
func (t *inMemoryTreeStorage) SetHeads(heads []string) error {
t.Lock()
defer t.Unlock()
t.heads = t.heads[:0]
for _, h := range heads {
t.heads = append(t.heads, h)
}
return nil
}
func (t *inMemoryTreeStorage) AddRawChange(change *aclpb.RawChange) error {
t.Lock()
defer t.Unlock()
// TODO: better to do deep copy
t.changes[change.Id] = change
return nil
}
func (t *inMemoryTreeStorage) GetRawChange(ctx context.Context, changeId string) (*aclpb.RawChange, error) {
t.RLock()
defer t.RUnlock()
if res, exists := t.changes[changeId]; exists {
return res, nil
}
return nil, fmt.Errorf("could not get change with id: %s", changeId)
}
type inMemoryStorageProvider struct {
objects map[string]Storage
sync.RWMutex
}
func (i *inMemoryStorageProvider) AddStorage(id string, st Storage) error {
i.Lock()
defer i.Unlock()
if _, exists := i.objects[id]; exists {
return fmt.Errorf("storage already exists")
}
i.objects[id] = st
return nil
}
func (i *inMemoryStorageProvider) Storage(id string) (Storage, error) {
i.RLock()
defer i.RUnlock()
if tree, exists := i.objects[id]; exists {
return tree, nil
}
return nil, ErrUnknownTreeId
}
func (i *inMemoryStorageProvider) CreateTreeStorage(payload TreeStorageCreatePayload) (TreeStorage, error) {
i.Lock()
defer i.Unlock()
res, err := NewInMemoryTreeStorage(payload.TreeId, payload.Header, payload.Heads, payload.Changes)
if err != nil {
return nil, err
}
i.objects[payload.TreeId] = res
return res, nil
}
func (i *inMemoryStorageProvider) CreateACLListStorage(payload ACLListStorageCreatePayload) (ListStorage, error) {
i.Lock()
defer i.Unlock()
res, err := NewInMemoryACLListStorage(payload.ListId, payload.Header, payload.Records)
if err != nil {
return nil, err
}
i.objects[payload.ListId] = res
return res, nil
}
func NewInMemoryTreeStorageProvider() Provider {
return &inMemoryStorageProvider{
objects: make(map[string]Storage),
}
}

14
pkg/acl/storage/liststorage.go Normal file
View File

@ -0,0 +1,14 @@
package storage
import (
"context"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
)
type ListStorage interface {
Storage
Head() (*aclpb.RawRecord, error)
GetRawRecord(ctx context.Context, id string) (*aclpb.RawRecord, error)
AddRawRecord(ctx context.Context, rec *aclpb.RawRecord) error
}

28
pkg/acl/storage/provider.go Normal file
View File

@ -0,0 +1,28 @@
package storage
import (
"errors"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
)
var ErrUnknownTreeId = errors.New("tree does not exist")
type TreeStorageCreatePayload struct {
TreeId string
Header *aclpb.Header
Changes []*aclpb.RawChange
Heads []string
}
type ACLListStorageCreatePayload struct {
ListId string
Header *aclpb.Header
Records []*aclpb.RawRecord
}
type Provider interface {
Storage(id string) (Storage, error)
AddStorage(id string, st Storage) error
CreateTreeStorage(payload TreeStorageCreatePayload) (TreeStorage, error)
CreateACLListStorage(payload ACLListStorageCreatePayload) (ListStorage, error)
}

8
pkg/acl/storage/storage.go Normal file
View File

@ -0,0 +1,8 @@
package storage
import "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
type Storage interface {
ID() (string, error)
Header() (*aclpb.Header, error)
}

17
pkg/acl/storage/treestorage.go Normal file
View File

@ -0,0 +1,17 @@
package storage
import (
"context"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
)
type TreeStorage interface {
Storage
Heads() ([]string, error)
SetHeads(heads []string) error
AddRawChange(change *aclpb.RawChange) error
GetRawChange(ctx context.Context, recordID string) (*aclpb.RawChange, error)
}
type TreeStorageCreatorFunc = func(payload TreeStorageCreatePayload) (TreeStorage, error)

97
pkg/acl/testutils/treestoragebuilder/keychain.go → pkg/acl/testutils/acllistbuilder/keychain.go
View File

@ -1,4 +1,4 @@
package treestoragebuilder
package acllistbuilder
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/encryptionkey"
@ -50,69 +50,110 @@ func (k *Keychain) ParseKeys(keys *Keys) {
}
}
func (k *Keychain) AddEncryptionKey(name string) {
if _, exists := k.EncryptionKeys[name]; exists {
func (k *Keychain) AddEncryptionKey(key *Key) {
if _, exists := k.EncryptionKeys[key.Name]; exists {
return
}
newPrivKey, _, err := encryptionkey.GenerateRandomRSAKeyPair(2048)
if err != nil {
panic(err)
var (
newPrivKey encryptionkey.PrivKey
err error
)
if key.Value == "generated" {
newPrivKey, _, err = encryptionkey.GenerateRandomRSAKeyPair(2048)
if err != nil {
panic(err)
}
} else {
decoder := encryptionkey.NewRSAPrivKeyDecoder()
privKey, err := decoder.DecodeFromString(key.Value)
if err != nil {
panic(err)
}
newPrivKey = privKey.(encryptionkey.PrivKey)
}
k.EncryptionKeys[name] = newPrivKey
k.EncryptionKeys[key.Name] = newPrivKey
}
func (k *Keychain) AddSigningKey(name string) {
if _, exists := k.SigningKeys[name]; exists {
func (k *Keychain) AddSigningKey(key *Key) {
if _, exists := k.SigningKeys[key.Name]; exists {
return
}
newPrivKey, pubKey, err := signingkey.GenerateRandomEd25519KeyPair()
if err != nil {
panic(err)
var (
newPrivKey signingkey.PrivKey
pubKey signingkey.PubKey
err error
)
if key.Value == "generated" {
newPrivKey, pubKey, err = signingkey.GenerateRandomEd25519KeyPair()
if err != nil {
panic(err)
}
} else {
decoder := signingkey.NewEDPrivKeyDecoder()
privKey, err := decoder.DecodeFromString(key.Value)
if err != nil {
panic(err)
}
newPrivKey = privKey.(signingkey.PrivKey)
pubKey = newPrivKey.GetPublic()
}
k.SigningKeys[name] = newPrivKey
k.SigningKeys[key.Name] = newPrivKey
res, err := k.coder.EncodeToString(pubKey)
if err != nil {
panic(err)
}
k.SigningKeysByIdentity[res] = newPrivKey
k.GeneratedIdentities[name] = res
k.GeneratedIdentities[key.Name] = res
}
func (k *Keychain) AddReadKey(name string) {
if _, exists := k.ReadKeys[name]; exists {
func (k *Keychain) AddReadKey(key *Key) {
if _, exists := k.ReadKeys[key.Name]; exists {
return
}
key, _ := symmetric.NewRandom()
var (
rkey *symmetric.Key
err error
)
if key.Value == "generated" {
rkey, err = symmetric.NewRandom()
if err != nil {
panic("should be able to generate symmetric key")
}
} else {
rkey, err = symmetric.FromString(key.Value)
if err != nil {
panic("should be able to parse symmetric key")
}
}
hasher := fnv.New64()
hasher.Write(key.Bytes())
hasher.Write(rkey.Bytes())
k.ReadKeys[name] = &SymKey{
k.ReadKeys[key.Name] = &SymKey{
Hash: hasher.Sum64(),
Key: key,
Key: rkey,
}
k.ReadKeysByHash[hasher.Sum64()] = &SymKey{
Hash: hasher.Sum64(),
Key: key,
Key: rkey,
}
}
func (k *Keychain) AddKey(key string) {
parts := strings.Split(key, ".")
func (k *Keychain) AddKey(key *Key) {
parts := strings.Split(key.Name, ".")
if len(parts) != 3 {
panic("cannot parse a key")
}
name := parts[2]
switch parts[1] {
case "Sign":
k.AddSigningKey(name)
k.AddSigningKey(key)
case "Enc":
k.AddEncryptionKey(name)
k.AddEncryptionKey(key)
case "Read":
k.AddReadKey(name)
k.AddReadKey(key)
default:
panic("incorrect format")
}

322
pkg/acl/testutils/acllistbuilder/liststoragebuilder.go Normal file
View File

@ -0,0 +1,322 @@
package acllistbuilder
import (
"context"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/yamltests"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/cid"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/encryptionkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"io/ioutil"
"path"
"time"
"github.com/gogo/protobuf/proto"
"gopkg.in/yaml.v3"
)
type ACLListStorageBuilder struct {
aclList string
records []*aclpb.Record
rawRecords []*aclpb.RawRecord
indexes map[string]int
keychain *Keychain
header *aclpb.Header
id string
}
func NewACLListStorageBuilder(keychain *Keychain) *ACLListStorageBuilder {
return &ACLListStorageBuilder{
records: make([]*aclpb.Record, 0),
indexes: make(map[string]int),
keychain: keychain,
}
}
func NewListStorageWithTestName(name string) (storage.ListStorage, error) {
filePath := path.Join(yamltests.Path(), name)
return NewACLListStorageBuilderFromFile(filePath)
}
func NewACLListStorageBuilderFromFile(file string) (*ACLListStorageBuilder, error) {
content, err := ioutil.ReadFile(file)
if err != nil {
return nil, err
}
ymlTree := YMLList{}
err = yaml.Unmarshal(content, &ymlTree)
if err != nil {
return nil, err
}
tb := NewACLListStorageBuilder(NewKeychain())
tb.Parse(&ymlTree)
return tb, nil
}
func (t *ACLListStorageBuilder) createRaw(rec *aclpb.Record) *aclpb.RawRecord {
aclMarshaled, err := proto.Marshal(rec)
if err != nil {
panic("should be able to marshal final acl message!")
}
signature, err := t.keychain.SigningKeysByIdentity[rec.Identity].Sign(aclMarshaled)
if err != nil {
panic("should be able to sign final acl message!")
}
id, _ := cid.NewCIDFromBytes(aclMarshaled)
return &aclpb.RawRecord{
Payload: aclMarshaled,
Signature: signature,
Id: id,
}
}
func (t *ACLListStorageBuilder) getRecord(idx int) *aclpb.RawRecord {
return t.rawRecords[idx]
}
func (t *ACLListStorageBuilder) Head() (*aclpb.RawRecord, error) {
return t.getRecord(len(t.records) - 1), nil
}
func (t *ACLListStorageBuilder) Header() (*aclpb.Header, error) {
return t.header, nil
}
func (t *ACLListStorageBuilder) GetRawRecord(ctx context.Context, id string) (*aclpb.RawRecord, error) {
recIdx, ok := t.indexes[id]
if !ok {
return nil, fmt.Errorf("no such record")
}
return t.getRecord(recIdx), nil
}
func (t *ACLListStorageBuilder) AddRawRecord(ctx context.Context, rec *aclpb.RawRecord) error {
panic("implement me")
}
func (t *ACLListStorageBuilder) ID() (string, error) {
return t.id, nil
}
func (t *ACLListStorageBuilder) GetRawRecords() []*aclpb.RawRecord {
return t.rawRecords
}
func (t *ACLListStorageBuilder) GetKeychain() *Keychain {
return t.keychain
}
func (t *ACLListStorageBuilder) Parse(tree *YMLList) {
// Just to clarify - we are generating new identities for the ones that
// are specified in the yml file, because our identities should be Ed25519
// the same thing is happening for the encryption keys
t.keychain.ParseKeys(&tree.Keys)
prevId := ""
for idx, rec := range tree.Records {
newRecord := t.parseRecord(rec, prevId)
rawRecord := t.createRaw(newRecord)
t.records = append(t.records, newRecord)
t.rawRecords = append(t.rawRecords, t.createRaw(newRecord))
t.indexes[rawRecord.Id] = idx
prevId = rawRecord.Id
}
t.createHeaderAndId()
}
func (t *ACLListStorageBuilder) parseRecord(rec *Record, prevId string) *aclpb.Record {
k := t.keychain.GetKey(rec.ReadKey).(*SymKey)
var aclChangeContents []*aclpb.ACLChangeACLContentValue
for _, ch := range rec.AclChanges {
aclChangeContent := t.parseACLChange(ch)
aclChangeContents = append(aclChangeContents, aclChangeContent)
}
data := &aclpb.ACLChangeACLData{
AclContent: aclChangeContents,
}
bytes, _ := data.Marshal()
return &aclpb.Record{
PrevId: prevId,
Identity: t.keychain.GetIdentity(rec.Identity),
Data: bytes,
CurrentReadKeyHash: k.Hash,
Timestamp: time.Now().Unix(),
}
}
func (t *ACLListStorageBuilder) parseACLChange(ch *ACLChange) (convCh *aclpb.ACLChangeACLContentValue) {
switch {
case ch.UserAdd != nil:
add := ch.UserAdd
encKey := t.keychain.GetKey(add.EncryptionKey).(encryptionkey.PrivKey)
rawKey, _ := encKey.GetPublic().Raw()
convCh = &aclpb.ACLChangeACLContentValue{
Value: &aclpb.ACLChangeACLContentValueValueOfUserAdd{
UserAdd: &aclpb.ACLChangeUserAdd{
Identity: t.keychain.GetIdentity(add.Identity),
EncryptionKey: rawKey,
EncryptedReadKeys: t.encryptReadKeys(add.EncryptedReadKeys, encKey),
Permissions: t.convertPermission(add.Permission),
},
},
}
case ch.UserJoin != nil:
join := ch.UserJoin
encKey := t.keychain.
GetKey(join.EncryptionKey).(encryptionkey.PrivKey)
rawKey, _ := encKey.GetPublic().Raw()
idKey, _ := t.keychain.SigningKeys[join.Identity].GetPublic().Raw()
signKey := t.keychain.GetKey(join.AcceptSignature).(signingkey.PrivKey)
signature, err := signKey.Sign(idKey)
if err != nil {
panic(err)
}
convCh = &aclpb.ACLChangeACLContentValue{
Value: &aclpb.ACLChangeACLContentValueValueOfUserJoin{
UserJoin: &aclpb.ACLChangeUserJoin{
Identity: t.keychain.GetIdentity(join.Identity),
EncryptionKey: rawKey,
AcceptSignature: signature,
UserInviteId: join.InviteId,
EncryptedReadKeys: t.encryptReadKeys(join.EncryptedReadKeys, encKey),
},
},
}
case ch.UserInvite != nil:
invite := ch.UserInvite
rawAcceptKey, _ := t.keychain.GetKey(invite.AcceptKey).(signingkey.PrivKey).GetPublic().Raw()
encKey := t.keychain.
GetKey(invite.EncryptionKey).(encryptionkey.PrivKey)
rawEncKey, _ := encKey.GetPublic().Raw()
convCh = &aclpb.ACLChangeACLContentValue{
Value: &aclpb.ACLChangeACLContentValueValueOfUserInvite{
UserInvite: &aclpb.ACLChangeUserInvite{
AcceptPublicKey: rawAcceptKey,
EncryptPublicKey: rawEncKey,
EncryptedReadKeys: t.encryptReadKeys(invite.EncryptedReadKeys, encKey),
Permissions: t.convertPermission(invite.Permissions),
InviteId: invite.InviteId,
},
},
}
case ch.UserConfirm != nil:
confirm := ch.UserConfirm
convCh = &aclpb.ACLChangeACLContentValue{
Value: &aclpb.ACLChangeACLContentValueValueOfUserConfirm{
UserConfirm: &aclpb.ACLChangeUserConfirm{
Identity: t.keychain.GetIdentity(confirm.Identity),
UserAddId: confirm.UserAddId,
},
},
}
case ch.UserPermissionChange != nil:
permissionChange := ch.UserPermissionChange
convCh = &aclpb.ACLChangeACLContentValue{
Value: &aclpb.ACLChangeACLContentValueValueOfUserPermissionChange{
UserPermissionChange: &aclpb.ACLChangeUserPermissionChange{
Identity: t.keychain.GetIdentity(permissionChange.Identity),
Permissions: t.convertPermission(permissionChange.Permission),
},
},
}
case ch.UserRemove != nil:
remove := ch.UserRemove
newReadKey := t.keychain.GetKey(remove.NewReadKey).(*SymKey)
var replaces []*aclpb.ACLChangeReadKeyReplace
for _, id := range remove.IdentitiesLeft {
identity := t.keychain.GetIdentity(id)
encKey := t.keychain.EncryptionKeys[id]
rawEncKey, _ := encKey.GetPublic().Raw()
encReadKey, err := encKey.GetPublic().Encrypt(newReadKey.Key.Bytes())
if err != nil {
panic(err)
}
replaces = append(replaces, &aclpb.ACLChangeReadKeyReplace{
Identity: identity,
EncryptionKey: rawEncKey,
EncryptedReadKey: encReadKey,
})
}
convCh = &aclpb.ACLChangeACLContentValue{
Value: &aclpb.ACLChangeACLContentValueValueOfUserRemove{
UserRemove: &aclpb.ACLChangeUserRemove{
Identity: t.keychain.GetIdentity(remove.RemovedIdentity),
ReadKeyReplaces: replaces,
},
},
}
}
if convCh == nil {
panic("cannot have empty acl change")
}
return convCh
}
func (t *ACLListStorageBuilder) encryptReadKeys(keys []string, encKey encryptionkey.PrivKey) (enc [][]byte) {
for _, k := range keys {
realKey := t.keychain.GetKey(k).(*SymKey).Key.Bytes()
res, err := encKey.GetPublic().Encrypt(realKey)
if err != nil {
panic(err)
}
enc = append(enc, res)
}
return
}
func (t *ACLListStorageBuilder) convertPermission(perm string) aclpb.ACLChangeUserPermissions {
switch perm {
case "admin":
return aclpb.ACLChange_Admin
case "writer":
return aclpb.ACLChange_Writer
case "reader":
return aclpb.ACLChange_Reader
default:
panic(fmt.Sprintf("incorrect permission: %s", perm))
}
}
func (t *ACLListStorageBuilder) traverseFromHead(f func(rec *aclpb.Record, id string) error) (err error) {
for i := len(t.records) - 1; i >= 0; i-- {
err = f(t.records[i], t.rawRecords[i].Id)
if err != nil {
return err
}
}
return nil
}
func (t *ACLListStorageBuilder) createHeaderAndId() {
t.header = &aclpb.Header{
FirstId: t.rawRecords[0].Id,
AclListId: "",
WorkspaceId: "",
DocType: aclpb.Header_ACL,
}
bytes, _ := t.header.Marshal()
id, _ := cid.NewCIDFromBytes(bytes)
t.id = id
}

4
pkg/acl/testutils/treestoragebuilder/treestoragebuildergraph.go → pkg/acl/testutils/acllistbuilder/liststoragebuildergraph.go
View File

@ -2,10 +2,10 @@
// +build !linux,!darwin android ios nographviz
// +build !amd64
package treestoragebuilder
package acllistbuilder
import "fmt"
func (t *TreeStorageBuilder) Graph() (string, error) {
func (t *ACLListStorageBuilder) Graph() (string, error) {
return "", fmt.Errorf("building graphs is not supported")
}

121
pkg/acl/testutils/acllistbuilder/liststoragebuildergraph_nix.go Normal file
View File

@ -0,0 +1,121 @@
//go:build (linux || darwin) && !android && !ios && !nographviz && (amd64 || arm64)
// +build linux darwin
// +build !android
// +build !ios
// +build !nographviz
// +build amd64 arm64
package acllistbuilder
import (
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/gogo/protobuf/proto"
"strings"
"unicode"
"github.com/awalterschulze/gographviz"
)
// To quickly look at visualized string you can use https://dreampuf.github.io/GraphvizOnline
type EdgeParameters struct {
style string
color string
label string
}
func (t *ACLListStorageBuilder) Graph() (string, error) {
// TODO: check updates on https://github.com/goccy/go-graphviz/issues/52 or make a fix yourself to use better library here
graph := gographviz.NewGraph()
graph.SetName("G")
graph.SetDir(true)
var nodes = make(map[string]struct{})
var addNodes = func(r *aclpb.Record, id string) error {
style := "solid"
var chSymbs []string
aclData := &aclpb.ACLChangeACLData{}
err := proto.Unmarshal(r.GetData(), aclData)
if err != nil {
return err
}
for _, chc := range aclData.AclContent {
tp := fmt.Sprintf("%T", chc.Value)
tp = strings.Replace(tp, "ACLChangeACLContentValueValueOf", "", 1)
res := ""
for _, ts := range tp {
if unicode.IsUpper(ts) {
res += string(ts)
}
}
chSymbs = append(chSymbs, res)
}
shortId := id
label := fmt.Sprintf("Id: %s\nChanges: %s\n",
shortId,
strings.Join(chSymbs, ","),
)
e := graph.AddNode("G", "\""+id+"\"", map[string]string{
"label": "\"" + label + "\"",
"style": "\"" + style + "\"",
})
if e != nil {
return e
}
nodes[id] = struct{}{}
return nil
}
var createEdge = func(firstId, secondId string, params EdgeParameters) error {
_, exists := nodes[firstId]
if !exists {
return fmt.Errorf("no such node")
}
_, exists = nodes[secondId]
if !exists {
return fmt.Errorf("no previous node")
}
err := graph.AddEdge("\""+firstId+"\"", "\""+secondId+"\"", true, map[string]string{
"color": params.color,
"style": params.style,
})
if err != nil {
return err
}
return nil
}
var addLinks = func(r *aclpb.Record, id string) error {
if r.PrevId == "" {
return nil
}
err := createEdge(id, r.PrevId, EdgeParameters{
style: "dashed",
color: "red",
})
if err != nil {
return err
}
return nil
}
err := t.traverseFromHead(addNodes)
if err != nil {
return "", err
}
err = t.traverseFromHead(addLinks)
if err != nil {
return "", err
}
return graph.String(), nil
}

70
pkg/acl/testutils/acllistbuilder/ymlentities.go Normal file
View File

@ -0,0 +1,70 @@
package acllistbuilder
type Key struct {
Name string `yaml:"name"`
Value string `yaml:"value"`
}
type Keys struct {
Enc []*Key `yaml:"Enc"`
Sign []*Key `yaml:"Sign"`
Read []*Key `yaml:"Read"`
}
type ACLChange struct {
UserAdd *struct {
Identity string `yaml:"identity"`
EncryptionKey string `yaml:"encryptionKey"`
EncryptedReadKeys []string `yaml:"encryptedReadKeys"`
Permission string `yaml:"permission"`
} `yaml:"userAdd"`
UserJoin *struct {
Identity string `yaml:"identity"`
EncryptionKey string `yaml:"encryptionKey"`
AcceptSignature string `yaml:"acceptSignature"`
InviteId string `yaml:"inviteId"`
EncryptedReadKeys []string `yaml:"encryptedReadKeys"`
} `yaml:"userJoin"`
UserInvite *struct {
AcceptKey string `yaml:"acceptKey"`
EncryptionKey string `yaml:"encryptionKey"`
EncryptedReadKeys []string `yaml:"encryptedReadKeys"`
Permissions string `yaml:"permissions"`
InviteId string `yaml:"inviteId"`
} `yaml:"userInvite"`
UserConfirm *struct {
Identity string `yaml:"identity"`
UserAddId string `yaml:"UserAddId"`
} `yaml:"userConfirm"`
UserRemove *struct {
RemovedIdentity string `yaml:"removedIdentity"`
NewReadKey string `yaml:"newReadKey"`
IdentitiesLeft []string `yaml:"identitiesLeft"`
} `yaml:"userRemove"`
UserPermissionChange *struct {
Identity string `yaml:"identity"`
Permission string `yaml:"permission"`
}
}
type Record struct {
Identity string `yaml:"identity"`
AclChanges []*ACLChange `yaml:"aclChanges"`
ReadKey string `yaml:"readKey"`
}
type Header struct {
FirstChangeId string `yaml:"firstChangeId"`
IsWorkspace bool `yaml:"isWorkspace"`
}
type YMLList struct {
Records []*Record `yaml:"records"`
Keys Keys `yaml:"keys"`
}

163
pkg/acl/testutils/testchanges/testchangepb/testdocumentchanges.pb.go → pkg/acl/testutils/testchanges/proto/test.pb.go
View File

@ -1,7 +1,7 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: pkg/acl/testutils/testchanges/testchangepb/protos/testdocumentchanges.proto
// source: pkg/acl/testutils/testchanges/proto/test.proto
package testchangepb
package testchanges
import (
fmt "fmt"
@ -29,7 +29,7 @@ func (m *PlainTextChange) Reset() { *m = PlainTextChange{} }
func (m *PlainTextChange) String() string { return proto.CompactTextString(m) }
func (*PlainTextChange) ProtoMessage() {}
func (*PlainTextChange) Descriptor() ([]byte, []int) {
return fileDescriptor_c07268f9f08f2beb, []int{0}
return fileDescriptor_37f33c266ada4318, []int{0}
}
func (m *PlainTextChange) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -68,7 +68,7 @@ func (m *PlainTextChangeContent) Reset() { *m = PlainTextChangeContent{}
func (m *PlainTextChangeContent) String() string { return proto.CompactTextString(m) }
func (*PlainTextChangeContent) ProtoMessage() {}
func (*PlainTextChangeContent) Descriptor() ([]byte, []int) {
return fileDescriptor_c07268f9f08f2beb, []int{0, 0}
return fileDescriptor_37f33c266ada4318, []int{0, 0}
}
func (m *PlainTextChangeContent) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -138,7 +138,7 @@ func (m *PlainTextChangeTextAppend) Reset() { *m = PlainTextChangeTextAp
func (m *PlainTextChangeTextAppend) String() string { return proto.CompactTextString(m) }
func (*PlainTextChangeTextAppend) ProtoMessage() {}
func (*PlainTextChangeTextAppend) Descriptor() ([]byte, []int) {
return fileDescriptor_c07268f9f08f2beb, []int{0, 1}
return fileDescriptor_37f33c266ada4318, []int{0, 1}
}
func (m *PlainTextChangeTextAppend) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -182,7 +182,7 @@ func (m *PlainTextChangeSnapshot) Reset() { *m = PlainTextChangeSnapshot
func (m *PlainTextChangeSnapshot) String() string { return proto.CompactTextString(m) }
func (*PlainTextChangeSnapshot) ProtoMessage() {}
func (*PlainTextChangeSnapshot) Descriptor() ([]byte, []int) {
return fileDescriptor_c07268f9f08f2beb, []int{0, 2}
return fileDescriptor_37f33c266ada4318, []int{0, 2}
}
func (m *PlainTextChangeSnapshot) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -227,7 +227,7 @@ func (m *PlainTextChangeData) Reset() { *m = PlainTextChangeData{} }
func (m *PlainTextChangeData) String() string { return proto.CompactTextString(m) }
func (*PlainTextChangeData) ProtoMessage() {}
func (*PlainTextChangeData) Descriptor() ([]byte, []int) {
return fileDescriptor_c07268f9f08f2beb, []int{0, 3}
return fileDescriptor_37f33c266ada4318, []int{0, 3}
}
func (m *PlainTextChangeData) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -279,29 +279,28 @@ func init() {
}
func init() {
proto.RegisterFile("pkg/acl/testutils/testchanges/testchangepb/protos/testdocumentchanges.proto", fileDescriptor_c07268f9f08f2beb)
proto.RegisterFile("pkg/acl/testutils/testchanges/proto/test.proto", fileDescriptor_37f33c266ada4318)
}
var fileDescriptor_c07268f9f08f2beb = []byte{
// 278 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xf2, 0x2e, 0xc8, 0x4e, 0xd7,
var fileDescriptor_37f33c266ada4318 = []byte{
// 266 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0x2b, 0xc8, 0x4e, 0xd7,
0x4f, 0x4c, 0xce, 0xd1, 0x2f, 0x49, 0x2d, 0x2e, 0x29, 0x2d, 0xc9, 0xcc, 0x29, 0x06, 0xb3, 0x92,
0x33, 0x12, 0xf3, 0xd2, 0x53, 0x91, 0xd9, 0x05, 0x49, 0xfa, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x10,
0xb1, 0x94, 0xfc, 0xe4, 0xd2, 0xdc, 0xd4, 0x3c, 0x98, 0x3a, 0x3d, 0xb0, 0x94, 0x10, 0x7b, 0x62,
0x5e, 0x65, 0x49, 0x65, 0x41, 0xaa, 0xd2, 0x26, 0x26, 0x2e, 0xfe, 0x80, 0x9c, 0xc4, 0xcc, 0xbc,
0x90, 0xd4, 0x8a, 0x12, 0x67, 0xb0, 0x1a, 0xa9, 0x48, 0x2e, 0x76, 0xe7, 0xfc, 0xbc, 0x92, 0xd4,
0xbc, 0x12, 0x21, 0x57, 0x2e, 0xae, 0x92, 0xd4, 0x8a, 0x12, 0xc7, 0x82, 0x82, 0xd4, 0xbc, 0x14,
0x09, 0x46, 0x05, 0x46, 0x0d, 0x6e, 0x23, 0x65, 0x3d, 0xa8, 0x66, 0x3d, 0x34, 0x8d, 0x7a, 0x21,
0x70, 0xa5, 0x1e, 0x0c, 0x41, 0x48, 0x1a, 0x9d, 0xd8, 0xb9, 0x58, 0xcb, 0x12, 0x73, 0x4a, 0x53,
0xa5, 0x14, 0xb8, 0xb8, 0x10, 0x8a, 0x84, 0x84, 0xb8, 0x58, 0x40, 0x8a, 0xc0, 0xe6, 0x72, 0x06,
0x81, 0xd9, 0x52, 0x72, 0x5c, 0x1c, 0xc1, 0x79, 0x89, 0x05, 0xc5, 0x19, 0xf9, 0x25, 0x58, 0xe5,
0x1b, 0x19, 0xb9, 0x58, 0x5c, 0x12, 0x4b, 0x12, 0x85, 0xac, 0xb8, 0xd8, 0x93, 0x21, 0xae, 0x94,
0x60, 0x54, 0x60, 0xd6, 0xe0, 0x36, 0x52, 0xc0, 0xe9, 0x2e, 0xa8, 0x6f, 0x82, 0x60, 0x1a, 0x84,
0x6c, 0xb9, 0x38, 0x8a, 0xa1, 0x96, 0x48, 0x30, 0x81, 0x3d, 0xa5, 0x88, 0x53, 0x33, 0xcc, 0x35,
0x41, 0x70, 0x2d, 0x4e, 0x6a, 0x27, 0x1e, 0xc9, 0x31, 0x5e, 0x78, 0x24, 0xc7, 0xf8, 0xe0, 0x91,
0x1c, 0xe3, 0x84, 0xc7, 0x72, 0x0c, 0x17, 0x1e, 0xcb, 0x31, 0xdc, 0x78, 0x2c, 0xc7, 0x10, 0xc5,
0x83, 0x1c, 0x0d, 0x49, 0x6c, 0xe0, 0xc0, 0x36, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0xfd, 0x73,
0xe1, 0xf2, 0xbb, 0x01, 0x00, 0x00,
0x33, 0x12, 0xf3, 0xd2, 0x53, 0x8b, 0xf5, 0x0b, 0x8a, 0xf2, 0x4b, 0xf2, 0xc1, 0x22, 0x7a, 0x60,
0xa6, 0x10, 0x7b, 0x62, 0x5e, 0x65, 0x49, 0x65, 0x41, 0xaa, 0xd2, 0x26, 0x26, 0x2e, 0xfe, 0x80,
0x9c, 0xc4, 0xcc, 0xbc, 0x90, 0xd4, 0x8a, 0x12, 0x67, 0xb0, 0x72, 0xa9, 0x48, 0x2e, 0x76, 0xe7,
0xfc, 0xbc, 0x92, 0xd4, 0xbc, 0x12, 0x21, 0x57, 0x2e, 0xae, 0x92, 0xd4, 0x8a, 0x12, 0xc7, 0x82,
0x82, 0xd4, 0xbc, 0x14, 0x09, 0x46, 0x05, 0x46, 0x0d, 0x6e, 0x23, 0x65, 0x3d, 0xa8, 0x66, 0x3d,
0x34, 0x8d, 0x7a, 0x21, 0x70, 0xa5, 0x1e, 0x0c, 0x41, 0x48, 0x1a, 0x9d, 0xd8, 0xb9, 0x58, 0xcb,
0x12, 0x73, 0x4a, 0x53, 0xa5, 0x14, 0xb8, 0xb8, 0x10, 0x8a, 0x84, 0x84, 0xb8, 0x58, 0x40, 0x8a,
0xc0, 0xe6, 0x72, 0x06, 0x81, 0xd9, 0x52, 0x72, 0x5c, 0x1c, 0xc1, 0x79, 0x89, 0x05, 0xc5, 0x19,
0xf9, 0x25, 0x58, 0xe5, 0x1b, 0x19, 0xb9, 0x58, 0x5c, 0x12, 0x4b, 0x12, 0x85, 0xac, 0xb8, 0xd8,
0x93, 0x21, 0xae, 0x94, 0x60, 0x54, 0x60, 0xd6, 0xe0, 0x36, 0x52, 0xc0, 0xe9, 0x2e, 0xa8, 0x6f,
0x82, 0x60, 0x1a, 0x84, 0x6c, 0xb9, 0x38, 0x8a, 0xa1, 0x96, 0x48, 0x30, 0x81, 0x3d, 0xa5, 0x88,
0x53, 0x33, 0xcc, 0x35, 0x41, 0x70, 0x2d, 0x4e, 0xaa, 0x27, 0x1e, 0xc9, 0x31, 0x5e, 0x78, 0x24,
0xc7, 0xf8, 0xe0, 0x91, 0x1c, 0xe3, 0x84, 0xc7, 0x72, 0x0c, 0x17, 0x1e, 0xcb, 0x31, 0xdc, 0x78,
0x2c, 0xc7, 0x10, 0xc5, 0x8d, 0x14, 0xea, 0x49, 0x6c, 0xe0, 0xb0, 0x36, 0x06, 0x04, 0x00, 0x00,
0xff, 0xff, 0xf8, 0x8c, 0x6a, 0x1d, 0x9d, 0x01, 0x00, 0x00,
}
func (m *PlainTextChange) Marshal() (dAtA []byte, err error) {
@ -373,7 +372,7 @@ func (m *PlainTextChangeContentValueOfTextAppend) MarshalToSizedBuffer(dAtA []by
return 0, err
}
i -= size
i = encodeVarintTestdocumentchanges(dAtA, i, uint64(size))
i = encodeVarintTest(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0xa
@ -403,7 +402,7 @@ func (m *PlainTextChangeTextAppend) MarshalToSizedBuffer(dAtA []byte) (int, erro
if len(m.Text) > 0 {
i -= len(m.Text)
copy(dAtA[i:], m.Text)
i = encodeVarintTestdocumentchanges(dAtA, i, uint64(len(m.Text)))
i = encodeVarintTest(dAtA, i, uint64(len(m.Text)))
i--
dAtA[i] = 0xa
}
@ -433,7 +432,7 @@ func (m *PlainTextChangeSnapshot) MarshalToSizedBuffer(dAtA []byte) (int, error)
if len(m.Text) > 0 {
i -= len(m.Text)
copy(dAtA[i:], m.Text)
i = encodeVarintTestdocumentchanges(dAtA, i, uint64(len(m.Text)))
i = encodeVarintTest(dAtA, i, uint64(len(m.Text)))
i--
dAtA[i] = 0xa
}
@ -467,7 +466,7 @@ func (m *PlainTextChangeData) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return 0, err
}
i -= size
i = encodeVarintTestdocumentchanges(dAtA, i, uint64(size))
i = encodeVarintTest(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0x12
@ -480,7 +479,7 @@ func (m *PlainTextChangeData) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return 0, err
}
i -= size
i = encodeVarintTestdocumentchanges(dAtA, i, uint64(size))
i = encodeVarintTest(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0xa
@ -489,8 +488,8 @@ func (m *PlainTextChangeData) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil
}
func encodeVarintTestdocumentchanges(dAtA []byte, offset int, v uint64) int {
offset -= sovTestdocumentchanges(v)
func encodeVarintTest(dAtA []byte, offset int, v uint64) int {
offset -= sovTest(v)
base := offset
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
@ -529,7 +528,7 @@ func (m *PlainTextChangeContentValueOfTextAppend) Size() (n int) {
_ = l
if m.TextAppend != nil {
l = m.TextAppend.Size()
n += 1 + l + sovTestdocumentchanges(uint64(l))
n += 1 + l + sovTest(uint64(l))
}
return n
}
@ -541,7 +540,7 @@ func (m *PlainTextChangeTextAppend) Size() (n int) {
_ = l
l = len(m.Text)
if l > 0 {
n += 1 + l + sovTestdocumentchanges(uint64(l))
n += 1 + l + sovTest(uint64(l))
}
return n
}
@ -554,7 +553,7 @@ func (m *PlainTextChangeSnapshot) Size() (n int) {
_ = l
l = len(m.Text)
if l > 0 {
n += 1 + l + sovTestdocumentchanges(uint64(l))
n += 1 + l + sovTest(uint64(l))
}
return n
}
@ -568,21 +567,21 @@ func (m *PlainTextChangeData) Size() (n int) {
if len(m.Content) > 0 {
for _, e := range m.Content {
l = e.Size()
n += 1 + l + sovTestdocumentchanges(uint64(l))
n += 1 + l + sovTest(uint64(l))
}
}
if m.Snapshot != nil {
l = m.Snapshot.Size()
n += 1 + l + sovTestdocumentchanges(uint64(l))
n += 1 + l + sovTest(uint64(l))
}
return n
}
func sovTestdocumentchanges(x uint64) (n int) {
func sovTest(x uint64) (n int) {
return (math_bits.Len64(x|1) + 6) / 7
}
func sozTestdocumentchanges(x uint64) (n int) {
return sovTestdocumentchanges(uint64((x << 1) ^ uint64((int64(x) >> 63))))
func sozTest(x uint64) (n int) {
return sovTest(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *PlainTextChange) Unmarshal(dAtA []byte) error {
l := len(dAtA)
@ -592,7 +591,7 @@ func (m *PlainTextChange) Unmarshal(dAtA []byte) error {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTestdocumentchanges
return ErrIntOverflowTest
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
@ -615,12 +614,12 @@ func (m *PlainTextChange) Unmarshal(dAtA []byte) error {
switch fieldNum {
default:
iNdEx = preIndex
skippy, err := skipTestdocumentchanges(dAtA[iNdEx:])
skippy, err := skipTest(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
@ -642,7 +641,7 @@ func (m *PlainTextChangeContent) Unmarshal(dAtA []byte) error {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTestdocumentchanges
return ErrIntOverflowTest
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
@ -670,7 +669,7 @@ func (m *PlainTextChangeContent) Unmarshal(dAtA []byte) error {
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTestdocumentchanges
return ErrIntOverflowTest
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
@ -683,11 +682,11 @@ func (m *PlainTextChangeContent) Unmarshal(dAtA []byte) error {
}
}
if msglen < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
if postIndex > l {
return io.ErrUnexpectedEOF
@ -700,12 +699,12 @@ func (m *PlainTextChangeContent) Unmarshal(dAtA []byte) error {
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipTestdocumentchanges(dAtA[iNdEx:])
skippy, err := skipTest(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
@ -727,7 +726,7 @@ func (m *PlainTextChangeTextAppend) Unmarshal(dAtA []byte) error {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTestdocumentchanges
return ErrIntOverflowTest
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
@ -755,7 +754,7 @@ func (m *PlainTextChangeTextAppend) Unmarshal(dAtA []byte) error {
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTestdocumentchanges
return ErrIntOverflowTest
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
@ -769,11 +768,11 @@ func (m *PlainTextChangeTextAppend) Unmarshal(dAtA []byte) error {
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
if postIndex > l {
return io.ErrUnexpectedEOF
@ -782,12 +781,12 @@ func (m *PlainTextChangeTextAppend) Unmarshal(dAtA []byte) error {
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipTestdocumentchanges(dAtA[iNdEx:])
skippy, err := skipTest(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
@ -809,7 +808,7 @@ func (m *PlainTextChangeSnapshot) Unmarshal(dAtA []byte) error {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTestdocumentchanges
return ErrIntOverflowTest
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
@ -837,7 +836,7 @@ func (m *PlainTextChangeSnapshot) Unmarshal(dAtA []byte) error {
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTestdocumentchanges
return ErrIntOverflowTest
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
@ -851,11 +850,11 @@ func (m *PlainTextChangeSnapshot) Unmarshal(dAtA []byte) error {
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
if postIndex > l {
return io.ErrUnexpectedEOF
@ -864,12 +863,12 @@ func (m *PlainTextChangeSnapshot) Unmarshal(dAtA []byte) error {
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipTestdocumentchanges(dAtA[iNdEx:])
skippy, err := skipTest(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
@ -891,7 +890,7 @@ func (m *PlainTextChangeData) Unmarshal(dAtA []byte) error {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTestdocumentchanges
return ErrIntOverflowTest
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
@ -919,7 +918,7 @@ func (m *PlainTextChangeData) Unmarshal(dAtA []byte) error {
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTestdocumentchanges
return ErrIntOverflowTest
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
@ -932,11 +931,11 @@ func (m *PlainTextChangeData) Unmarshal(dAtA []byte) error {
}
}
if msglen < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
if postIndex > l {
return io.ErrUnexpectedEOF
@ -953,7 +952,7 @@ func (m *PlainTextChangeData) Unmarshal(dAtA []byte) error {
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTestdocumentchanges
return ErrIntOverflowTest
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
@ -966,11 +965,11 @@ func (m *PlainTextChangeData) Unmarshal(dAtA []byte) error {
}
}
if msglen < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
if postIndex > l {
return io.ErrUnexpectedEOF
@ -984,12 +983,12 @@ func (m *PlainTextChangeData) Unmarshal(dAtA []byte) error {
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipTestdocumentchanges(dAtA[iNdEx:])
skippy, err := skipTest(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthTestdocumentchanges
return ErrInvalidLengthTest
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
@ -1003,7 +1002,7 @@ func (m *PlainTextChangeData) Unmarshal(dAtA []byte) error {
}
return nil
}
func skipTestdocumentchanges(dAtA []byte) (n int, err error) {
func skipTest(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
depth := 0
@ -1011,7 +1010,7 @@ func skipTestdocumentchanges(dAtA []byte) (n int, err error) {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowTestdocumentchanges
return 0, ErrIntOverflowTest
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
@ -1028,7 +1027,7 @@ func skipTestdocumentchanges(dAtA []byte) (n int, err error) {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowTestdocumentchanges
return 0, ErrIntOverflowTest
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
@ -1044,7 +1043,7 @@ func skipTestdocumentchanges(dAtA []byte) (n int, err error) {
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowTestdocumentchanges
return 0, ErrIntOverflowTest
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
@ -1057,14 +1056,14 @@ func skipTestdocumentchanges(dAtA []byte) (n int, err error) {
}
}
if length < 0 {
return 0, ErrInvalidLengthTestdocumentchanges
return 0, ErrInvalidLengthTest
}
iNdEx += length
case 3:
depth++
case 4:
if depth == 0 {
return 0, ErrUnexpectedEndOfGroupTestdocumentchanges
return 0, ErrUnexpectedEndOfGroupTest
}
depth--
case 5:
@ -1073,7 +1072,7 @@ func skipTestdocumentchanges(dAtA []byte) (n int, err error) {
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
if iNdEx < 0 {
return 0, ErrInvalidLengthTestdocumentchanges
return 0, ErrInvalidLengthTest
}
if depth == 0 {
return iNdEx, nil
@ -1083,7 +1082,7 @@ func skipTestdocumentchanges(dAtA []byte) (n int, err error) {
}
var (
ErrInvalidLengthTestdocumentchanges = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowTestdocumentchanges = fmt.Errorf("proto: integer overflow")
ErrUnexpectedEndOfGroupTestdocumentchanges = fmt.Errorf("proto: unexpected end of group")
ErrInvalidLengthTest = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowTest = fmt.Errorf("proto: integer overflow")
ErrUnexpectedEndOfGroupTest = fmt.Errorf("proto: unexpected end of group")
)

6
pkg/acl/testutils/testchanges/testchangepb/protos/testdocumentchanges.proto → pkg/acl/testutils/testchanges/proto/test.proto
View File

@ -1,8 +1,6 @@
syntax = "proto3";
package anytype;
option go_package = "testchangepb";
// TODO: move to separate package
option go_package = "testchanges";
message PlainTextChange {
message Content {
@ -23,4 +21,4 @@ message PlainTextChange {
repeated Content content = 1;
Snapshot snapshot = 2;
}
}
}

539
pkg/acl/testutils/treestoragebuilder/treestoragebuilder.go
View File

@ -1,539 +0,0 @@
package treestoragebuilder
import (
"context"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
testpb "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/testchanges/testchangepb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/yamltests"
storagepb "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/encryptionkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice"
"io/ioutil"
"path"
"github.com/gogo/protobuf/proto"
"gopkg.in/yaml.v3"
)
const plainTextDocType uint16 = 1
type treeChange struct {
*aclpb.ACLChange
id string
readKey *SymKey
signKey signingkey.PrivKey
changesDataDecrypted []byte
}
type updateUseCase struct {
changes map[string]*treeChange
}
type TreeStorageBuilder struct {
treeId string
allChanges map[string]*treeChange
updates map[string]*updateUseCase
heads []string
orphans []string
keychain *Keychain
header *storagepb.TreeHeader
}
func NewTreeStorageBuilder(keychain *Keychain) *TreeStorageBuilder {
return &TreeStorageBuilder{
allChanges: make(map[string]*treeChange),
updates: make(map[string]*updateUseCase),
keychain: keychain,
}
}
func NewTreeStorageBuilderWithTestName(name string) (*TreeStorageBuilder, error) {
filePath := path.Join(yamltests.Path(), name)
return NewTreeStorageBuilderFromFile(filePath)
}
func NewTreeStorageBuilderFromFile(file string) (*TreeStorageBuilder, error) {
content, err := ioutil.ReadFile(file)
if err != nil {
return nil, err
}
ymlTree := YMLTree{}
err = yaml.Unmarshal(content, &ymlTree)
if err != nil {
return nil, err
}
tb := NewTreeStorageBuilder(NewKeychain())
tb.Parse(&ymlTree)
return tb, nil
}
func (t *TreeStorageBuilder) TreeID() (string, error) {
return t.treeId, nil
}
func (t *TreeStorageBuilder) GetKeychain() *Keychain {
return t.keychain
}
func (t *TreeStorageBuilder) Heads() ([]string, error) {
return t.heads, nil
}
func (t *TreeStorageBuilder) AddRawChange(change *aclpb.RawChange) error {
aclChange := new(aclpb.ACLChange)
var err error
if err = proto.Unmarshal(change.Payload, aclChange); err != nil {
return fmt.Errorf("could not unmarshall changes")
}
var changesData []byte
// get correct readkey
readKey := t.keychain.ReadKeysByHash[aclChange.CurrentReadKeyHash]
if aclChange.ChangesData != nil {
changesData, err = readKey.Key.Decrypt(aclChange.ChangesData)
if err != nil {
return fmt.Errorf("failed to decrypt changes data: %w", err)
}
}
// get correct signing key
signKey := t.keychain.SigningKeysByIdentity[aclChange.Identity]
t.allChanges[change.Id] = &treeChange{
ACLChange: aclChange,
id: change.Id,
readKey: readKey,
signKey: signKey,
changesDataDecrypted: changesData,
}
return nil
}
func (t *TreeStorageBuilder) AddOrphans(orphans ...string) error {
t.orphans = append(t.orphans, orphans...)
return nil
}
func (t *TreeStorageBuilder) AddChange(change aclchanges.Change) error {
aclChange := change.ProtoChange()
var err error
var changesData []byte
// get correct readkey
readKey := t.keychain.ReadKeysByHash[aclChange.CurrentReadKeyHash]
if aclChange.ChangesData != nil {
changesData, err = readKey.Key.Decrypt(aclChange.ChangesData)
if err != nil {
return fmt.Errorf("failed to decrypt changes data: %w", err)
}
}
// get correct signing key
signKey := t.keychain.SigningKeysByIdentity[aclChange.Identity]
t.allChanges[change.CID()] = &treeChange{
ACLChange: aclChange,
id: change.CID(),
readKey: readKey,
signKey: signKey,
changesDataDecrypted: changesData,
}
return nil
}
func (t *TreeStorageBuilder) Orphans() ([]string, error) {
return t.orphans, nil
}
func (t *TreeStorageBuilder) SetHeads(heads []string) error {
// we should copy here instead of just setting the value
t.heads = heads
return nil
}
func (t *TreeStorageBuilder) RemoveOrphans(orphans ...string) error {
t.orphans = slice.Difference(t.orphans, orphans)
return nil
}
func (t *TreeStorageBuilder) GetChange(ctx context.Context, recordID string) (*aclpb.RawChange, error) {
return t.getChange(recordID, t.allChanges), nil
}
func (t *TreeStorageBuilder) GetUpdates(useCase string) []*aclpb.RawChange {
var res []*aclpb.RawChange
update := t.updates[useCase]
for _, ch := range update.changes {
rawCh := t.getChange(ch.id, update.changes)
res = append(res, rawCh)
}
return res
}
func (t *TreeStorageBuilder) Header() (*storagepb.TreeHeader, error) {
return t.header, nil
}
func (t *TreeStorageBuilder) getChange(changeId string, m map[string]*treeChange) *aclpb.RawChange {
rec := m[changeId]
if rec.changesDataDecrypted != nil {
encrypted, err := rec.readKey.Key.Encrypt(rec.changesDataDecrypted)
if err != nil {
panic("should be able to encrypt data with read key!")
}
rec.ChangesData = encrypted
}
aclMarshaled, err := proto.Marshal(rec.ACLChange)
if err != nil {
panic("should be able to marshal final acl message!")
}
signature, err := rec.signKey.Sign(aclMarshaled)
if err != nil {
panic("should be able to sign final acl message!")
}
transformedRec := &aclpb.RawChange{
Payload: aclMarshaled,
Signature: signature,
Id: changeId,
}
return transformedRec
}
func (t *TreeStorageBuilder) Parse(tree *YMLTree) {
// Just to clarify - we are generating new identities for the ones that
// are specified in the yml file, because our identities should be Ed25519
// the same thing is happening for the encryption keys
t.keychain.ParseKeys(&tree.Keys)
t.treeId = t.parseTreeId(tree.Description)
for _, ch := range tree.Changes {
newChange := t.parseChange(ch)
t.allChanges[newChange.id] = newChange
}
t.parseGraph(tree)
t.parseOrphans(tree)
t.parseHeader(tree)
t.parseUpdates(tree.Updates)
}
func (t *TreeStorageBuilder) parseChange(ch *Change) *treeChange {
newChange := &treeChange{
id: ch.Id,
}
k := t.keychain.GetKey(ch.ReadKey).(*SymKey)
newChange.readKey = k
newChange.signKey = t.keychain.SigningKeys[ch.Identity]
aclChange := &aclpb.ACLChange{}
aclChange.Identity = t.keychain.GetIdentity(ch.Identity)
if len(ch.AclChanges) > 0 || ch.AclSnapshot != nil {
aclChange.AclData = &aclpb.ACLChangeACLData{}
if ch.AclSnapshot != nil {
aclChange.AclData.AclSnapshot = t.parseACLSnapshot(ch.AclSnapshot)
}
if ch.AclChanges != nil {
var aclChangeContents []*aclpb.ACLChangeACLContentValue
for _, ch := range ch.AclChanges {
aclChangeContent := t.parseACLChange(ch)
aclChangeContents = append(aclChangeContents, aclChangeContent)
}
aclChange.AclData.AclContent = aclChangeContents
}
}
if len(ch.Changes) > 0 || ch.Snapshot != nil {
changesData := &testpb.PlainTextChangeData{}
if ch.Snapshot != nil {
changesData.Snapshot = t.parseChangeSnapshot(ch.Snapshot)
}
if len(ch.Changes) > 0 {
var changeContents []*testpb.PlainTextChangeContent
for _, ch := range ch.Changes {
aclChangeContent := t.parseDocumentChange(ch)
changeContents = append(changeContents, aclChangeContent)
}
changesData.Content = changeContents
}
m, err := proto.Marshal(changesData)
if err != nil {
return nil
}
newChange.changesDataDecrypted = m
}
aclChange.CurrentReadKeyHash = k.Hash
newChange.ACLChange = aclChange
return newChange
}
func (t *TreeStorageBuilder) parseTreeId(description *TreeDescription) string {
if description == nil {
panic("no author in tree")
}
return description.Author + ".tree.id"
}
func (t *TreeStorageBuilder) parseChangeSnapshot(s *PlainTextSnapshot) *testpb.PlainTextChangeSnapshot {
return &testpb.PlainTextChangeSnapshot{
Text: s.Text,
}
}
func (t *TreeStorageBuilder) parseACLSnapshot(s *ACLSnapshot) *aclpb.ACLChangeACLSnapshot {
newState := &aclpb.ACLChangeACLState{}
for _, state := range s.UserStates {
aclUserState := &aclpb.ACLChangeUserState{}
aclUserState.Identity = t.keychain.GetIdentity(state.Identity)
encKey := t.keychain.
GetKey(state.EncryptionKey).(encryptionkey.PrivKey)
rawKey, _ := encKey.GetPublic().Raw()
aclUserState.EncryptionKey = rawKey
aclUserState.EncryptedReadKeys = t.encryptReadKeys(state.EncryptedReadKeys, encKey)
aclUserState.Permissions = t.convertPermission(state.Permissions)
newState.UserStates = append(newState.UserStates, aclUserState)
}
return &aclpb.ACLChangeACLSnapshot{
AclState: newState,
}
}
func (t *TreeStorageBuilder) parseDocumentChange(ch *PlainTextChange) (convCh *testpb.PlainTextChangeContent) {
switch {
case ch.TextAppend != nil:
convCh = &testpb.PlainTextChangeContent{
Value: &testpb.PlainTextChangeContentValueOfTextAppend{
TextAppend: &testpb.PlainTextChangeTextAppend{
Text: ch.TextAppend.Text,
},
},
}
}
if convCh == nil {
panic("cannot have empty document change")
}
return convCh
}
func (t *TreeStorageBuilder) parseACLChange(ch *ACLChange) (convCh *aclpb.ACLChangeACLContentValue) {
switch {
case ch.UserAdd != nil:
add := ch.UserAdd
encKey := t.keychain.
GetKey(add.EncryptionKey).(encryptionkey.PrivKey)
rawKey, _ := encKey.GetPublic().Raw()
convCh = &aclpb.ACLChangeACLContentValue{
Value: &aclpb.ACLChangeACLContentValueValueOfUserAdd{
UserAdd: &aclpb.ACLChangeUserAdd{
Identity: t.keychain.GetIdentity(add.Identity),
EncryptionKey: rawKey,
EncryptedReadKeys: t.encryptReadKeys(add.EncryptedReadKeys, encKey),
Permissions: t.convertPermission(add.Permission),
},
},
}
case ch.UserJoin != nil:
join := ch.UserJoin
encKey := t.keychain.
GetKey(join.EncryptionKey).(encryptionkey.PrivKey)
rawKey, _ := encKey.GetPublic().Raw()
idKey, _ := t.keychain.SigningKeys[join.Identity].GetPublic().Raw()
signKey := t.keychain.GetKey(join.AcceptSignature).(signingkey.PrivKey)
signature, err := signKey.Sign(idKey)
if err != nil {
panic(err)
}
convCh = &aclpb.ACLChangeACLContentValue{
Value: &aclpb.ACLChangeACLContentValueValueOfUserJoin{
UserJoin: &aclpb.ACLChangeUserJoin{
Identity: t.keychain.GetIdentity(join.Identity),
EncryptionKey: rawKey,
AcceptSignature: signature,
UserInviteId: join.InviteId,
EncryptedReadKeys: t.encryptReadKeys(join.EncryptedReadKeys, encKey),
},
},
}
case ch.UserInvite != nil:
invite := ch.UserInvite
rawAcceptKey, _ := t.keychain.GetKey(invite.AcceptKey).(signingkey.PrivKey).GetPublic().Raw()
encKey := t.keychain.
GetKey(invite.EncryptionKey).(encryptionkey.PrivKey)
rawEncKey, _ := encKey.GetPublic().Raw()
convCh = &aclpb.ACLChangeACLContentValue{
Value: &aclpb.ACLChangeACLContentValueValueOfUserInvite{
UserInvite: &aclpb.ACLChangeUserInvite{
AcceptPublicKey: rawAcceptKey,
EncryptPublicKey: rawEncKey,
EncryptedReadKeys: t.encryptReadKeys(invite.EncryptedReadKeys, encKey),
Permissions: t.convertPermission(invite.Permissions),
InviteId: invite.InviteId,
},
},
}
case ch.UserConfirm != nil:
confirm := ch.UserConfirm
convCh = &aclpb.ACLChangeACLContentValue{
Value: &aclpb.ACLChangeACLContentValueValueOfUserConfirm{
UserConfirm: &aclpb.ACLChangeUserConfirm{
Identity: t.keychain.GetIdentity(confirm.Identity),
UserAddId: confirm.UserAddId,
},
},
}
case ch.UserPermissionChange != nil:
permissionChange := ch.UserPermissionChange
convCh = &aclpb.ACLChangeACLContentValue{
Value: &aclpb.ACLChangeACLContentValueValueOfUserPermissionChange{
UserPermissionChange: &aclpb.ACLChangeUserPermissionChange{
Identity: t.keychain.GetIdentity(permissionChange.Identity),
Permissions: t.convertPermission(permissionChange.Permission),
},
},
}
case ch.UserRemove != nil:
remove := ch.UserRemove
newReadKey := t.keychain.GetKey(remove.NewReadKey).(*SymKey)
var replaces []*aclpb.ACLChangeReadKeyReplace
for _, id := range remove.IdentitiesLeft {
identity := t.keychain.GetIdentity(id)
encKey := t.keychain.EncryptionKeys[id]
rawEncKey, _ := encKey.GetPublic().Raw()
encReadKey, err := encKey.GetPublic().Encrypt(newReadKey.Key.Bytes())
if err != nil {
panic(err)
}
replaces = append(replaces, &aclpb.ACLChangeReadKeyReplace{
Identity: identity,
EncryptionKey: rawEncKey,
EncryptedReadKey: encReadKey,
})
}
convCh = &aclpb.ACLChangeACLContentValue{
Value: &aclpb.ACLChangeACLContentValueValueOfUserRemove{
UserRemove: &aclpb.ACLChangeUserRemove{
Identity: t.keychain.GetIdentity(remove.RemovedIdentity),
ReadKeyReplaces: replaces,
},
},
}
}
if convCh == nil {
panic("cannot have empty acl change")
}
return convCh
}
func (t *TreeStorageBuilder) encryptReadKeys(keys []string, encKey encryptionkey.PrivKey) (enc [][]byte) {
for _, k := range keys {
realKey := t.keychain.GetKey(k).(*SymKey).Key.Bytes()
res, err := encKey.GetPublic().Encrypt(realKey)
if err != nil {
panic(err)
}
enc = append(enc, res)
}
return
}
func (t *TreeStorageBuilder) convertPermission(perm string) aclpb.ACLChangeUserPermissions {
switch perm {
case "admin":
return aclpb.ACLChange_Admin
case "writer":
return aclpb.ACLChange_Writer
case "reader":
return aclpb.ACLChange_Reader
default:
panic(fmt.Sprintf("incorrect permission: %s", perm))
}
}
func (t *TreeStorageBuilder) traverseFromHeads(f func(t *treeChange) error) error {
uniqMap := map[string]struct{}{}
stack := make([]string, len(t.orphans), 10)
copy(stack, t.orphans)
for len(stack) > 0 {
id := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if _, exists := uniqMap[id]; exists {
continue
}
ch := t.allChanges[id]
uniqMap[id] = struct{}{}
if err := f(ch); err != nil {
return err
}
for _, prev := range ch.ACLChange.TreeHeadIds {
stack = append(stack, prev)
}
}
return nil
}
func (t *TreeStorageBuilder) parseUpdates(updates []*Update) {
for _, update := range updates {
useCase := &updateUseCase{
changes: map[string]*treeChange{},
}
for _, ch := range update.Changes {
newChange := t.parseChange(ch)
useCase.changes[newChange.id] = newChange
}
for _, node := range update.Graph {
rec := useCase.changes[node.Id]
rec.AclHeadIds = node.ACLHeads
rec.TreeHeadIds = node.TreeHeads
rec.SnapshotBaseId = node.BaseSnapshot
}
t.updates[update.UseCase] = useCase
}
}
func (t *TreeStorageBuilder) parseGraph(tree *YMLTree) {
for _, node := range tree.Graph {
rec := t.allChanges[node.Id]
rec.AclHeadIds = node.ACLHeads
rec.TreeHeadIds = node.TreeHeads
rec.SnapshotBaseId = node.BaseSnapshot
}
}
func (t *TreeStorageBuilder) parseOrphans(tree *YMLTree) {
t.orphans = tree.Orphans
}
func (t *TreeStorageBuilder) parseHeader(tree *YMLTree) {
t.header = &storagepb.TreeHeader{
FirstChangeId: tree.Header.FirstChangeId,
IsWorkspace: tree.Header.IsWorkspace,
}
}

162
pkg/acl/testutils/treestoragebuilder/treestoragebuildergraph_nix.go
View File

@ -1,162 +0,0 @@
//go:build (linux || darwin) && !android && !ios && !nographviz && (amd64 || arm64)
// +build linux darwin
// +build !android
// +build !ios
// +build !nographviz
// +build amd64 arm64
package treestoragebuilder
import (
"fmt"
testpb "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/testchanges/testchangepb"
"github.com/gogo/protobuf/proto"
"strings"
"unicode"
"github.com/awalterschulze/gographviz"
)
// To quickly look at visualized string you can use https://dreampuf.github.io/GraphvizOnline
type EdgeParameters struct {
style string
color string
label string
}
func (t *TreeStorageBuilder) Graph() (string, error) {
// TODO: check updates on https://github.com/goccy/go-graphviz/issues/52 or make a fix yourself to use better library here
graph := gographviz.NewGraph()
graph.SetName("G")
graph.SetDir(true)
var nodes = make(map[string]struct{})
var addNodes = func(r *treeChange) error {
// TODO: revisit function after checking
style := "solid"
if r.GetAclData() != nil {
style = "filled"
} else if r.changesDataDecrypted != nil {
style = "dashed"
}
var chSymbs []string
if r.changesDataDecrypted != nil {
res := &testpb.PlainTextChangeData{}
err := proto.Unmarshal(r.changesDataDecrypted, res)
if err != nil {
return err
}
for _, chc := range res.Content {
tp := fmt.Sprintf("%T", chc.Value)
tp = strings.Replace(tp, "ChangeContentValueOf", "", 1)
res := ""
for _, ts := range tp {
if unicode.IsUpper(ts) {
res += string(ts)
}
}
chSymbs = append(chSymbs, res)
}
}
if r.GetAclData() != nil {
for _, chc := range r.GetAclData().AclContent {
tp := fmt.Sprintf("%T", chc.Value)
tp = strings.Replace(tp, "ACLChangeACLContentValueValueOf", "", 1)
res := ""
for _, ts := range tp {
if unicode.IsUpper(ts) {
res += string(ts)
}
}
chSymbs = append(chSymbs, res)
}
}
shortId := r.id
label := fmt.Sprintf("Id: %s\nChanges: %s\n",
shortId,
strings.Join(chSymbs, ","),
)
e := graph.AddNode("G", "\""+r.id+"\"", map[string]string{
"label": "\"" + label + "\"",
"style": "\"" + style + "\"",
})
if e != nil {
return e
}
nodes[r.id] = struct{}{}
return nil
}
var createEdge = func(firstId, secondId string, params EdgeParameters) error {
_, exists := nodes[firstId]
if !exists {
return fmt.Errorf("no such node")
}
_, exists = nodes[secondId]
if !exists {
return fmt.Errorf("no previous node")
}
err := graph.AddEdge("\""+firstId+"\"", "\""+secondId+"\"", true, map[string]string{
"color": params.color,
"style": params.style,
})
if err != nil {
return err
}
return nil
}
var addLinks = func(t *treeChange) error {
for _, prevId := range t.AclHeadIds {
err := createEdge(t.id, prevId, EdgeParameters{
style: "dashed",
color: "red",
})
if err != nil {
return err
}
}
for _, prevId := range t.TreeHeadIds {
err := createEdge(t.id, prevId, EdgeParameters{
style: "dashed",
color: "blue",
})
if err != nil {
return err
}
}
if t.SnapshotBaseId != "" {
err := createEdge(t.id, t.SnapshotBaseId, EdgeParameters{
style: "bold",
color: "blue",
})
if err != nil {
return err
}
}
return nil
}
err := t.traverseFromHeads(addNodes)
if err != nil {
return "", err
}
err = t.traverseFromHeads(addLinks)
if err != nil {
return "", err
}
return graph.String(), nil
}

117
pkg/acl/testutils/treestoragebuilder/ymlentities.go
View File

@ -1,117 +0,0 @@
package treestoragebuilder
type TreeDescription struct {
Author string `yaml:"author"`
}
type Keys struct {
Enc []string `yaml:"Enc"`
Sign []string `yaml:"Sign"`
Read []string `yaml:"Read"`
}
type ACLSnapshot struct {
UserStates []struct {
Identity string `yaml:"identity"`
EncryptionKey string `yaml:"encryptionKey"`
EncryptedReadKeys []string `yaml:"encryptedReadKeys"`
Permissions string `yaml:"permission"`
IsConfirmed bool `yaml:"isConfirmed"`
} `yaml:"userStates"`
}
type PlainTextSnapshot struct {
Text string `yaml:"text"`
}
type ACLChange struct {
UserAdd *struct {
Identity string `yaml:"identity"`
EncryptionKey string `yaml:"encryptionKey"`
EncryptedReadKeys []string `yaml:"encryptedReadKeys"`
Permission string `yaml:"permission"`
} `yaml:"userAdd"`
UserJoin *struct {
Identity string `yaml:"identity"`
EncryptionKey string `yaml:"encryptionKey"`
AcceptSignature string `yaml:"acceptSignature"`
InviteId string `yaml:"inviteId"`
EncryptedReadKeys []string `yaml:"encryptedReadKeys"`
} `yaml:"userJoin"`
UserInvite *struct {
AcceptKey string `yaml:"acceptKey"`
EncryptionKey string `yaml:"encryptionKey"`
EncryptedReadKeys []string `yaml:"encryptedReadKeys"`
Permissions string `yaml:"permissions"`
InviteId string `yaml:"inviteId"`
} `yaml:"userInvite"`
UserConfirm *struct {
Identity string `yaml:"identity"`
UserAddId string `yaml:"UserAddId"`
} `yaml:"userConfirm"`
UserRemove *struct {
RemovedIdentity string `yaml:"removedIdentity"`
NewReadKey string `yaml:"newReadKey"`
IdentitiesLeft []string `yaml:"identitiesLeft"`
} `yaml:"userRemove"`
UserPermissionChange *struct {
Identity string `yaml:"identity"`
Permission string `yaml:"permission"`
}
}
type PlainTextChange struct {
TextAppend *struct {
Text string `yaml:"text"`
} `yaml:"textAppend"`
}
type GraphNode struct {
Id string `yaml:"id"`
BaseSnapshot string `yaml:"baseSnapshot"`
AclSnapshot string `yaml:"aclSnapshot"`
ACLHeads []string `yaml:"aclHeads"`
TreeHeads []string `yaml:"treeHeads"`
}
type Change struct {
Id string `yaml:"id"`
Identity string `yaml:"identity"`
AclSnapshot *ACLSnapshot `yaml:"aclSnapshot"`
Snapshot *PlainTextSnapshot `yaml:"snapshot"`
AclChanges []*ACLChange `yaml:"aclChanges"`
Changes []*PlainTextChange `yaml:"changes"`
ReadKey string `yaml:"readKey"`
}
type Header struct {
FirstChangeId string `yaml:"firstChangeId"`
IsWorkspace bool `yaml:"isWorkspace"`
}
type Update struct {
UseCase string `yaml:"useCase"`
Changes []*Change `yaml:"changes"`
Graph []*GraphNode `yaml:"graph"`
}
type YMLTree struct {
Description *TreeDescription `yaml:"tree"`
Changes []*Change `yaml:"changes"`
Updates []*Update `yaml:"updates"`
Keys Keys `yaml:"keys"`
Graph []*GraphNode `yaml:"graph"`
Heads []string `yaml:"heads"`
Orphans []string `yaml:"orphans"`
Header *Header `yaml:"header"`
}

12
pkg/acl/testutils/treestoragebuilder/ymlentities_test.go
View File

@ -1,12 +0,0 @@
package treestoragebuilder
import (
"fmt"
"testing"
)
func Test_YamlParse(t *testing.T) {
tb, _ := NewTreeStorageBuilderWithTestName("userjoinexampleupdate.yml")
gr, _ := tb.Graph()
fmt.Println(gr)
}

126
pkg/acl/testutils/yamltests/invalidsnapshotexample.yml
View File

@ -1,126 +0,0 @@
tree:
author: A
changes:
- id: A.1.1
identity: A
aclSnapshot:
userStates:
- identity: A
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
permission: admin
- identity: B
encryptionKey: key.Enc.B
encryptedReadKeys: [key.Read.1]
permission: admin
snapshot:
text: "some text"
aclChanges:
- userAdd:
identity: A
permission: admin
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
- userAdd:
identity: B
permission: admin
encryptionKey: key.Enc.B
encryptedReadKeys: [key.Read.1]
readKey: key.Read.1
- id: A.1.2
identity: A
aclSnapshot:
userStates:
- identity: A
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
permission: admin
- identity: B
encryptionKey: key.Enc.B
encryptedReadKeys: [key.Read.1]
permission: admin
- identity: D
encryptionKey: key.Enc.D
encryptedReadKeys: [ key.Read.1 ]
permission: admin
snapshot:
text: "some text"
aclChanges:
- userAdd:
identity: D
permission: admin
encryptionKey: key.Enc.D
encryptedReadKeys: [key.Read.1]
readKey: key.Read.1
- id: A.1.3
identity: A
aclChanges:
- userAdd:
identity: E
permission: admin
encryptionKey: key.Enc.E
encryptedReadKeys: [key.Read.1]
readKey: key.Read.1
- id: B.1.1
identity: B
aclChanges:
- userAdd:
identity: C
permission: admin
encryptionKey: key.Enc.C
encryptedReadKeys: [ key.Read.1 ]
readKey: key.Read.1
- id: B.1.2
identity: B
aclChanges:
- userAdd:
identity: F
permission: admin
encryptionKey: key.Enc.F
encryptedReadKeys: [ key.Read.1 ]
readKey: key.Read.1
keys:
Enc:
- A
- B
- C
- D
- E
- F
Sign:
- A
- B
- C
- D
- E
- F
Read:
- 1
- 2
graph:
- id: A.1.1
baseSnapshot: A.1.1
aclSnapshot: A.1.1
- id: A.1.2
baseSnapshot: A.1.1
aclHeads: [B.1.1]
treeHeads: [B.1.1]
- id: B.1.1
baseSnapshot: A.1.1
aclHeads: [A.1.1]
treeHeads: [A.1.1]
- id: B.1.2
baseSnapshot: A.1.2
aclHeads: [A.1.2]
treeHeads: [A.1.2]
- id: A.1.3
baseSnapshot: A.1.2
aclHeads: [A.1.2]
treeHeads: [A.1.2]
header:
firstChangeId: A.1.1
isWorkspace: false
orphans:
- A.1.3
- B.1.2

99
pkg/acl/testutils/yamltests/userjoinexample.yml
View File

@ -1,28 +1,13 @@
tree:
author: A
changes:
- id: A.1.1
identity: A
aclSnapshot:
userStates:
- identity: A
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
permission: admin
snapshot:
text: "some text"
records:
- identity: A
aclChanges:
- userAdd:
identity: A
permission: admin
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
changes:
- textAppend:
text: "some text"
readKey: key.Read.1
- id: A.1.2
identity: A
- identity: A
aclChanges:
- userInvite:
acceptKey: key.Sign.Onetime1
@ -34,16 +19,9 @@ changes:
identity: C
permission: reader
encryptionKey: key.Enc.C
encryptedReadKeys: [ key.Read.1 ]
encryptedReadKeys: [key.Read.1]
readKey: key.Read.1
- id: A.1.3
identity: A
changes:
- textAppend:
text: "second"
readKey: key.Read.1
- id: B.1.1
identity: B
- identity: B
aclChanges:
- userJoin:
identity: B
@ -52,56 +30,25 @@ changes:
inviteId: A.1.2
encryptedReadKeys: [key.Read.1]
readKey: key.Read.1
- id: B.1.2
identity: B
changes:
- textAppend:
text: "first"
readKey: key.Read.1
- id: C.1.1
identity: C
changes:
- textAppend:
text: "third"
readKey: key.Read.1
keys:
Enc:
- A
- B
- C
- Onetime1
- name: A
value: generated
- name: B
value: generated
- name: C
value: generated
- name: Onetime1
value: generated
Sign:
- A
- B
- C
- Onetime1
- name: A
value: generated
- name: B
value: generated
- name: C
value: generated
- name: Onetime1
value: generated
Read:
- 1
graph:
- id: A.1.1
baseSnapshot: A.1.1
- id: A.1.2
baseSnapshot: A.1.1
aclHeads: [A.1.1]
treeHeads: [A.1.1]
- id: B.1.1
baseSnapshot: A.1.1
aclHeads: [A.1.2]
treeHeads: [A.1.2]
- id: B.1.2
baseSnapshot: A.1.1
aclHeads: [B.1.1]
treeHeads: [B.1.1]
- id: A.1.3 # this should be invalid, because it is based on one of the invalid changes
baseSnapshot: A.1.1
aclHeads: [B.1.1]
treeHeads: [B.1.2, C.1.1]
- id: C.1.1 # this should be invalid, because C is a reader
baseSnapshot: A.1.1
aclHeads: [B.1.1]
treeHeads: [B.1.1]
header:
firstChangeId: A.1.1
isWorkspace: false
orphans:
- "A.1.3"
- name: 1
value: generated

152
pkg/acl/testutils/yamltests/userjoinexampleupdate.yml
View File

@ -1,152 +0,0 @@
tree:
author: A
changes:
- id: A.1.1
identity: A
aclSnapshot:
userStates:
- identity: A
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
permission: admin
snapshot:
text: "some text"
aclChanges:
- userAdd:
identity: A
permission: admin
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
changes:
- textAppend:
text: "some text"
readKey: key.Read.1
- id: A.1.2
identity: A
aclChanges:
- userInvite:
acceptKey: key.Sign.Onetime1
encryptionKey: key.Enc.Onetime1
encryptedReadKeys: [key.Read.1]
permissions: writer
inviteId: A.1.2
- userAdd:
identity: C
permission: reader
encryptionKey: key.Enc.C
encryptedReadKeys: [ key.Read.1 ]
readKey: key.Read.1
- id: A.1.3
identity: A
changes:
- textAppend:
text: "second"
readKey: key.Read.1
- id: B.1.1
identity: B
aclChanges:
- userJoin:
identity: B
encryptionKey: key.Enc.B
acceptSignature: key.Sign.Onetime1
inviteId: A.1.2
encryptedReadKeys: [key.Read.1]
readKey: key.Read.1
- id: B.1.2
identity: B
changes:
- textAppend:
text: "first"
readKey: key.Read.1
- id: C.1.1
identity: C
changes:
- textAppend:
text: "third"
readKey: key.Read.1
keys:
Enc:
- A
- B
- C
- D
- Onetime1
Sign:
- A
- B
- C
- D
- Onetime1
Read:
- 1
graph:
- id: A.1.1
baseSnapshot: A.1.1
- id: A.1.2
baseSnapshot: A.1.1
aclHeads: [A.1.1]
treeHeads: [A.1.1]
- id: B.1.1
baseSnapshot: A.1.1
aclHeads: [A.1.2]
treeHeads: [A.1.2]
- id: B.1.2
baseSnapshot: A.1.1
aclHeads: [B.1.1]
treeHeads: [B.1.1]
- id: A.1.3 # this should be invalid, because it is based on one of the invalid changes
baseSnapshot: A.1.1
aclHeads: [B.1.1]
treeHeads: [B.1.2, C.1.1]
- id: C.1.1 # this should be invalid, because C is a reader
baseSnapshot: A.1.1
aclHeads: [B.1.1]
treeHeads: [B.1.1]
header:
firstChangeId: A.1.1
isWorkspace: false
orphans:
- "A.1.3"
updates:
- useCase: append
changes:
- id: B.1.3
identity: B
changes:
- textAppend:
text: "second"
readKey: key.Read.1
- id: A.1.4
identity: A
aclChanges:
- userAdd:
identity: D
permission: writer
encryptionKey: key.Enc.D
encryptedReadKeys: [ key.Read.1 ]
readKey: key.Read.1
graph:
- id: B.1.3
baseSnapshot: A.1.1
aclHeads: [ B.1.1 ]
treeHeads: [ B.1.2 ]
- id: A.1.4
baseSnapshot: A.1.1
aclHeads: [ B.1.1 ]
treeHeads: [ B.1.3 ]
- useCase: rebuild
changes:
- id: A.1.4
identity: A
aclChanges:
- userAdd:
identity: D
permission: writer
encryptionKey: key.Enc.D
encryptedReadKeys: [ key.Read.1 ]
readKey: key.Read.1
graph:
- id: A.1.4
baseSnapshot: A.1.1
aclHeads: [ A.1.1 ]
treeHeads: [ A.1.1 ]

109
pkg/acl/testutils/yamltests/userremovebeforeexample.yml
View File

@ -1,109 +0,0 @@
tree:
author: A
changes:
- id: A.1.1
identity: A
aclSnapshot:
userStates:
- identity: A
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
permission: admin
- identity: B
encryptionKey: key.Enc.B
encryptedReadKeys: [key.Read.1]
permission: admin
snapshot:
text: "some text"
aclChanges:
- userAdd:
identity: A
permission: admin
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
- userAdd:
identity: B
permission: admin
encryptionKey: key.Enc.B
encryptedReadKeys: [key.Read.1]
changes:
- textAppend:
text: "some text"
readKey: key.Read.1
- id: A.1.2
identity: A
aclChanges:
- userRemove:
removedIdentity: B
newReadKey: key.Read.2
identitiesLeft: [A, C]
readKey: key.Read.2
- id: A.1.3
identity: A
aclChanges:
- userAdd:
identity: E
permission: admin
encryptionKey: key.Enc.E
encryptedReadKeys: [key.Read.1, key.Read.2]
readKey: key.Read.2
- id: B.1.1
identity: B
aclChanges:
- userAdd:
identity: C
permission: admin
encryptionKey: key.Enc.C
encryptedReadKeys: [ key.Read.1 ]
readKey: key.Read.1
- id: B.1.2
identity: B
aclChanges:
- userAdd:
identity: D
permission: admin
encryptionKey: key.Enc.D
encryptedReadKeys: [ key.Read.1 ]
readKey: key.Read.1
keys:
Enc:
- A
- B
- C
- D
- E
Sign:
- A
- B
- C
- D
- E
Read:
- 1
- 2
graph:
- id: A.1.1
baseSnapshot: A.1.1
aclSnapshot: A.1.1
- id: A.1.2
baseSnapshot: A.1.1
aclHeads: [B.1.1]
treeHeads: [B.1.1]
- id: B.1.1
baseSnapshot: A.1.1
aclHeads: [A.1.1]
treeHeads: [A.1.1]
- id: B.1.2
baseSnapshot: A.1.1
aclHeads: [B.1.1]
treeHeads: [B.1.1]
- id: A.1.3
baseSnapshot: A.1.1
aclHeads: [A.1.2]
treeHeads: [A.1.2]
orphans:
- "A.1.3"
- "B.1.2"
header:
firstChangeId: A.1.1
isWorkspace: false

119
pkg/acl/testutils/yamltests/userremoveexample.yml
View File

@ -1,28 +1,13 @@
tree:
author: A
changes:
- id: A.1.1
identity: A
aclSnapshot:
userStates:
- identity: A
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
permission: admin
snapshot:
text: "some text"
records:
- identity: A
aclChanges:
- userAdd:
identity: A
permission: admin
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
changes:
- textAppend:
text: "some text"
readKey: key.Read.1
- id: A.1.2
identity: A
- identity: A
aclChanges:
- userInvite:
acceptKey: key.Sign.Onetime1
@ -30,23 +15,13 @@ changes:
encryptedReadKeys: [key.Read.1]
permissions: writer
inviteId: A.1.2
- userAdd:
identity: C
permission: reader
encryptionKey: key.Enc.C
encryptedReadKeys: [key.Read.1]
readKey: key.Read.1
- id: A.1.3
identity: A
aclChanges:
- userRemove:
removedIdentity: B
newReadKey: key.Read.2
identitiesLeft: [A]
readKey: key.Read.2
- id: A.1.4
identity: A
changes:
- textAppend:
text: "first"
readKey: key.Read.2
- id: B.1.1
identity: B
- identity: B
aclChanges:
- userJoin:
identity: B
@ -55,56 +30,34 @@ changes:
inviteId: A.1.2
encryptedReadKeys: [key.Read.1]
readKey: key.Read.1
- id: B.1.2
identity: B
changes:
- textAppend:
text: "second"
readKey: key.Read.1
- identity: A
aclChanges:
- userRemove:
removedIdentity: B
newReadKey: key.Read.2
identitiesLeft: [A, C]
readKey: key.Read.2
keys:
Enc:
- A
- B
- Onetime1
- name: A
value: generated
- name: B
value: generated
- name: C
value: generated
- name: Onetime1
value: generated
Sign:
- A
- B
- Onetime1
- name: A
value: generated
- name: B
value: generated
- name: C
value: generated
- name: Onetime1
value: generated
Read:
- 1
- 2
graph:
- id: A.1.1
baseSnapshot: A.1.1
aclSnapshot: A.1.1
- id: A.1.2
baseSnapshot: A.1.1
aclSnapshot: A.1.1
aclHeads: [A.1.1]
treeHeads: [A.1.1]
- id: B.1.1
baseSnapshot: A.1.1
aclSnapshot: A.1.1
aclHeads: [A.1.2]
treeHeads: [A.1.2]
- id: B.1.2
baseSnapshot: A.1.1
aclSnapshot: A.1.1
aclHeads: [B.1.1]
treeHeads: [B.1.1]
- id: A.1.3
baseSnapshot: A.1.1
aclSnapshot: A.1.1
aclHeads: [B.1.1]
treeHeads: [B.1.1]
- id: A.1.4
baseSnapshot: A.1.1
aclSnapshot: A.1.1
aclHeads: [A.1.3]
treeHeads: [A.1.3]
orphans:
- "A.1.4"
- "B.1.2"
header:
firstChangeId: A.1.1
isWorkspace: false
- name: 1
value: generated
- name: 2
value: generated

133
pkg/acl/testutils/yamltests/validsnapshotexample.yml
View File

@ -1,133 +0,0 @@
tree:
author: A
changes:
- id: A.1.1
identity: A
aclSnapshot:
userStates:
- identity: A
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
permission: admin
- identity: B
encryptionKey: key.Enc.B
encryptedReadKeys: [key.Read.1]
permission: admin
snapshot:
text: "some text"
aclChanges:
- userAdd:
identity: A
permission: admin
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
- userAdd:
identity: B
permission: admin
encryptionKey: key.Enc.B
encryptedReadKeys: [key.Read.1]
readKey: key.Read.1
changes:
- textAppend:
text: "some text"
- id: A.1.2
identity: A
aclSnapshot:
userStates:
- identity: A
encryptionKey: key.Enc.A
encryptedReadKeys: [key.Read.1]
permission: admin
- identity: B
encryptionKey: key.Enc.B
encryptedReadKeys: [key.Read.1]
permission: admin
- identity: C
encryptionKey: key.Enc.C
encryptedReadKeys: [ key.Read.1 ]
permission: admin
- identity: D
encryptionKey: key.Enc.D
encryptedReadKeys: [ key.Read.1 ]
permission: admin
snapshot:
text: "some text"
aclChanges:
- userAdd:
identity: D
permission: admin
encryptionKey: key.Enc.D
encryptedReadKeys: [key.Read.1]
readKey: key.Read.1
- id: A.1.3
identity: A
aclChanges:
- userAdd:
identity: E
permission: admin
encryptionKey: key.Enc.E
encryptedReadKeys: [key.Read.1]
readKey: key.Read.1
- id: B.1.1
identity: B
aclChanges:
- userAdd:
identity: C
permission: admin
encryptionKey: key.Enc.C
encryptedReadKeys: [ key.Read.1 ]
readKey: key.Read.1
- id: B.1.2
identity: B
aclChanges:
- userAdd:
identity: F
permission: admin
encryptionKey: key.Enc.F
encryptedReadKeys: [ key.Read.1 ]
readKey: key.Read.1
keys:
Enc:
- A
- B
- C
- D
- E
- F
Sign:
- A
- B
- C
- D
- E
- F
Read:
- 1
- 2
graph:
- id: A.1.1
baseSnapshot: A.1.1
aclSnapshot: A.1.1
- id: A.1.2
baseSnapshot: A.1.1
aclHeads: [B.1.1]
treeHeads: [B.1.1]
- id: B.1.1
baseSnapshot: A.1.1
aclHeads: [A.1.1]
treeHeads: [A.1.1]
- id: B.1.2
baseSnapshot: A.1.2
aclHeads: [A.1.2]
treeHeads: [A.1.2]
- id: A.1.3
baseSnapshot: A.1.2
aclHeads: [A.1.2]
treeHeads: [A.1.2]
orphans:
- "A.1.3"
- "B.1.2"
header:
firstChangeId: A.1.1
isWorkspace: false

117
pkg/acl/tree/change.go Normal file
View File

@ -0,0 +1,117 @@
package tree
import (
"errors"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/gogo/protobuf/proto"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/symmetric"
)
var (
ErrIncorrectSignature = errors.New("change has incorrect signature")
ErrIncorrectCID = errors.New("change has incorrect CID")
)
type ChangeContent struct {
ChangesData proto.Marshaler
ACLData *aclpb.ACLChangeACLData
Id string // TODO: this is just for testing, because id should be created automatically from content
}
// Change is an abstract type for all types of changes
type Change struct {
Next []*Change
PreviousIds []string
Id string
SnapshotId string
IsSnapshot bool
DecryptedChange []byte // TODO: check if we need it
ParsedModel interface{} // TODO: check if we need it
// iterator helpers
visited bool
branchesFinished bool
Content *aclpb.Change
Sign []byte
}
func (ch *Change) ProtoChange() proto.Marshaler {
return ch.Content
}
func (ch *Change) DecryptContents(key *symmetric.Key) error {
// if the document is already decrypted
if ch.Content.CurrentReadKeyHash == 0 {
return nil
}
decrypted, err := key.Decrypt(ch.Content.ChangesData)
if err != nil {
return fmt.Errorf("failed to decrypt changes data: %w", err)
}
ch.DecryptedChange = decrypted
return nil
}
func NewChangeFromRaw(rawChange *aclpb.RawChange) (*Change, error) {
unmarshalled := &aclpb.Change{}
err := proto.Unmarshal(rawChange.Payload, unmarshalled)
if err != nil {
return nil, err
}
ch := NewChange(rawChange.Id, unmarshalled, rawChange.Signature)
return ch, nil
}
func newVerifiedChangeFromRaw(
rawChange *aclpb.RawChange,
kch *keychain) (*Change, error) {
unmarshalled := &aclpb.Change{}
ch, err := NewChangeFromRaw(rawChange)
if err != nil {
return nil, err
}
identityKey, err := kch.getOrAdd(unmarshalled.Identity)
if err != nil {
return nil, err
}
res, err := identityKey.Verify(rawChange.Payload, rawChange.Signature)
if err != nil {
return nil, err
}
if !res {
return nil, ErrIncorrectSignature
}
return ch, nil
}
func NewChange(id string, ch *aclpb.Change, signature []byte) *Change {
return &Change{
Next: nil,
PreviousIds: ch.TreeHeadIds,
Id: id,
Content: ch,
SnapshotId: ch.SnapshotBaseId,
IsSnapshot: ch.IsSnapshot,
Sign: signature,
}
}
func (ch *Change) DecryptedChangeContent() []byte {
return ch.DecryptedChange
}
func (ch *Change) Signature() []byte {
return ch.Sign
}
func (ch *Change) CID() string {
return ch.Id
}

126
pkg/acl/tree/changebuilder.go Normal file
View File

@ -0,0 +1,126 @@
package tree
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/cid"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/symmetric"
"github.com/gogo/protobuf/proto"
"time"
)
const componentBuilder = "tree.changebuilder"
type BuilderContent struct {
treeHeadIds []string
aclHeadId string
snapshotBaseId string
currentReadKeyHash uint64
identity string
isSnapshot bool
signingKey signingkey.PrivKey
readKey *symmetric.Key
content proto.Marshaler
}
type ChangeBuilder interface {
ConvertFromRaw(rawChange *aclpb.RawChange) (ch *Change, err error)
ConvertFromRawAndVerify(rawChange *aclpb.RawChange) (ch *Change, err error)
BuildContent(payload BuilderContent) (ch *Change, raw *aclpb.RawChange, err error)
}
type changeBuilder struct {
keys *keychain
}
func newChangeBuilder(keys *keychain) *changeBuilder {
return &changeBuilder{keys: keys}
}
func (c *changeBuilder) ConvertFromRaw(rawChange *aclpb.RawChange) (ch *Change, err error) {
unmarshalled := &aclpb.Change{}
err = proto.Unmarshal(rawChange.Payload, unmarshalled)
if err != nil {
return nil, err
}
ch = NewChange(rawChange.Id, unmarshalled, rawChange.Signature)
return
}
func (c *changeBuilder) ConvertFromRawAndVerify(rawChange *aclpb.RawChange) (ch *Change, err error) {
unmarshalled := &aclpb.Change{}
ch, err = c.ConvertFromRaw(rawChange)
if err != nil {
return nil, err
}
identityKey, err := c.keys.getOrAdd(unmarshalled.Identity)
if err != nil {
return
}
// verifying signature
res, err := identityKey.Verify(rawChange.Payload, rawChange.Signature)
if err != nil {
return
}
if !res {
err = ErrIncorrectSignature
return
}
// verifying ID
if !cid.VerifyCID(rawChange.Payload, rawChange.Id) {
err = ErrIncorrectCID
}
return
}
func (c *changeBuilder) BuildContent(payload BuilderContent) (ch *Change, raw *aclpb.RawChange, err error) {
aclChange := &aclpb.Change{
TreeHeadIds: payload.treeHeadIds,
AclHeadId: payload.aclHeadId,
SnapshotBaseId: payload.snapshotBaseId,
CurrentReadKeyHash: payload.currentReadKeyHash,
Timestamp: int64(time.Now().Nanosecond()),
Identity: payload.identity,
IsSnapshot: payload.isSnapshot,
}
marshalledData, err := payload.content.Marshal()
if err != nil {
return
}
encrypted, err := payload.readKey.Encrypt(marshalledData)
if err != nil {
return
}
aclChange.ChangesData = encrypted
fullMarshalledChange, err := proto.Marshal(aclChange)
if err != nil {
return
}
signature, err := payload.signingKey.Sign(fullMarshalledChange)
if err != nil {
return
}
id, err := cid.NewCIDFromBytes(fullMarshalledChange)
if err != nil {
return
}
ch = NewChange(id, aclChange, signature)
ch.ParsedModel = payload.content
raw = &aclpb.RawChange{
Payload: fullMarshalledChange,
Signature: signature,
Id: id,
}
return
}

58
pkg/acl/tree/changevalidator.go Normal file
View File

@ -0,0 +1,58 @@
package tree
import (
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/list"
)
type ObjectTreeValidator interface {
// ValidateTree should always be entered while holding a read lock on ACLList
ValidateTree(tree *Tree, aclList list.ACLList) error
}
type objectTreeValidator struct{}
func newTreeValidator() ObjectTreeValidator {
return &objectTreeValidator{}
}
func (v *objectTreeValidator) ValidateTree(tree *Tree, aclList list.ACLList) (err error) {
var (
perm list.UserPermissionPair
state = aclList.ACLState()
)
tree.Iterate(tree.RootId(), func(c *Change) (isContinue bool) {
// checking if the user could write
perm, err = state.PermissionsAtRecord(c.Content.AclHeadId, c.Content.Identity)
if err != nil {
return false
}
if perm.Permission != aclpb.ACLChange_Writer && perm.Permission != aclpb.ACLChange_Admin {
err = list.ErrInsufficientPermissions
return false
}
// checking if the change refers to later acl heads than its previous ids
for _, id := range c.PreviousIds {
prevChange := tree.attached[id]
if prevChange.Content.AclHeadId == c.Content.AclHeadId {
continue
}
var after bool
after, err = aclList.IsAfter(c.Content.AclHeadId, prevChange.Content.AclHeadId)
if err != nil {
return false
}
if !after {
err = fmt.Errorf("current acl head id (%s) should be after each of the previous ones (%s)", c.Content.AclHeadId, prevChange.Content.AclHeadId)
return false
}
}
return true
})
return err
}

13
pkg/acl/tree/descriptionparser.go Normal file
View File

@ -0,0 +1,13 @@
package tree
type DescriptionParser interface {
ParseChange(*Change) ([]string, error)
}
var NoOpDescriptionParser = noopDescriptionParser{}
type noopDescriptionParser struct{}
func (n noopDescriptionParser) ParseChange(change *Change) ([]string, error) {
return []string{"DOC"}, nil
}

31
pkg/acl/tree/keychain.go Normal file
View File

@ -0,0 +1,31 @@
package tree
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
)
type keychain struct {
decoder keys.Decoder
keys map[string]signingkey.PubKey
}
func newKeychain() *keychain {
return &keychain{
decoder: signingkey.NewEDPubKeyDecoder(),
keys: make(map[string]signingkey.PubKey),
}
}
func (k *keychain) getOrAdd(identity string) (signingkey.PubKey, error) {
if key, exists := k.keys[identity]; exists {
return key, nil
}
res, err := k.decoder.DecodeFromString(identity)
if err != nil {
return nil, err
}
k.keys[identity] = res.(signingkey.PubKey)
return res.(signingkey.PubKey), nil
}

571
pkg/acl/tree/objecttree.go Normal file
View File

@ -0,0 +1,571 @@
package tree
import (
"context"
"errors"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/list"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/symmetric"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice"
"go.uber.org/zap"
"sync"
)
type ObjectTreeUpdateListener interface {
Update(tree ObjectTree)
Rebuild(tree ObjectTree)
}
type RWLocker interface {
sync.Locker
RLock()
RUnlock()
}
var (
ErrHasInvalidChanges = errors.New("the change is invalid")
ErrNoCommonSnapshot = errors.New("trees doesn't have a common snapshot")
)
type AddResultSummary int
const (
AddResultSummaryNothing AddResultSummary = iota
AddResultSummaryAppend
AddResultSummaryRebuild
)
type AddResult struct {
OldHeads []string
Heads []string
Added []*aclpb.RawChange
Summary AddResultSummary
}
type ChangeIterateFunc = func(change *Change) bool
type ChangeConvertFunc = func(decrypted []byte) (any, error)
type ObjectTree interface {
RWLocker
ID() string
Header() *aclpb.Header
Heads() []string
Root() *Change
HasChange(string) bool
Iterate(convert ChangeConvertFunc, iterate ChangeIterateFunc) error
IterateFrom(id string, convert ChangeConvertFunc, iterate ChangeIterateFunc) error
SnapshotPath() []string
ChangesAfterCommonSnapshot(snapshotPath, heads []string) ([]*aclpb.RawChange, error)
Storage() storage.TreeStorage
DebugDump() (string, error)
AddContent(ctx context.Context, content SignableChangeContent) (*aclpb.RawChange, error)
AddRawChanges(ctx context.Context, changes ...*aclpb.RawChange) (AddResult, error)
Close() error
}
type objectTree struct {
treeStorage storage.TreeStorage
changeBuilder ChangeBuilder
updateListener ObjectTreeUpdateListener
validator ObjectTreeValidator
rawChangeLoader *rawChangeLoader
treeBuilder *treeBuilder
aclList list.ACLList
id string
header *aclpb.Header
tree *Tree
keys map[uint64]*symmetric.Key
// buffers
difSnapshotBuf []*aclpb.RawChange
tmpChangesBuf []*Change
newSnapshotsBuf []*Change
notSeenIdxBuf []int
snapshotPath []string
sync.RWMutex
}
type objectTreeDeps struct {
changeBuilder ChangeBuilder
treeBuilder *treeBuilder
treeStorage storage.TreeStorage
updateListener ObjectTreeUpdateListener
validator ObjectTreeValidator
rawChangeLoader *rawChangeLoader
aclList list.ACLList
}
func defaultObjectTreeDeps(
treeStorage storage.TreeStorage,
listener ObjectTreeUpdateListener,
aclList list.ACLList) objectTreeDeps {
keychain := newKeychain()
changeBuilder := newChangeBuilder(keychain)
treeBuilder := newTreeBuilder(treeStorage, changeBuilder)
return objectTreeDeps{
changeBuilder: changeBuilder,
treeBuilder: treeBuilder,
treeStorage: treeStorage,
updateListener: listener,
validator: newTreeValidator(),
rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder),
aclList: aclList,
}
}
func buildObjectTree(deps objectTreeDeps) (ObjectTree, error) {
objTree := &objectTree{
treeStorage: deps.treeStorage,
updateListener: deps.updateListener,
treeBuilder: deps.treeBuilder,
validator: deps.validator,
aclList: deps.aclList,
changeBuilder: deps.changeBuilder,
rawChangeLoader: deps.rawChangeLoader,
tree: nil,
keys: make(map[uint64]*symmetric.Key),
tmpChangesBuf: make([]*Change, 0, 10),
difSnapshotBuf: make([]*aclpb.RawChange, 0, 10),
notSeenIdxBuf: make([]int, 0, 10),
newSnapshotsBuf: make([]*Change, 0, 10),
}
err := objTree.rebuildFromStorage(nil)
if err != nil {
return nil, err
}
storageHeads, err := objTree.treeStorage.Heads()
if err != nil {
return nil, err
}
// comparing rebuilt heads with heads in storage
// in theory it can happen that we didn't set heads because the process has crashed
// therefore we want to set them later
if !slice.UnsortedEquals(storageHeads, objTree.tree.Heads()) {
log.With(zap.Strings("storage", storageHeads), zap.Strings("rebuilt", objTree.tree.Heads())).
Errorf("the heads in storage and objTree are different")
err = objTree.treeStorage.SetHeads(objTree.tree.Heads())
if err != nil {
return nil, err
}
}
objTree.id, err = objTree.treeStorage.ID()
if err != nil {
return nil, err
}
objTree.header, err = objTree.treeStorage.Header()
if err != nil {
return nil, err
}
if objTree.updateListener != nil {
objTree.updateListener.Rebuild(objTree)
}
return objTree, nil
}
func BuildObjectTree(treeStorage storage.TreeStorage, listener ObjectTreeUpdateListener, aclList list.ACLList) (ObjectTree, error) {
deps := defaultObjectTreeDeps(treeStorage, listener, aclList)
return buildObjectTree(deps)
}
func (ot *objectTree) rebuildFromStorage(newChanges []*Change) (err error) {
ot.treeBuilder.Reset()
ot.tree, err = ot.treeBuilder.Build(newChanges)
if err != nil {
return
}
// during building the tree we may have marked some changes as possible roots,
// but obviously they are not roots, because of the way how we construct the tree
ot.tree.clearPossibleRoots()
return ot.validateTree()
}
func (ot *objectTree) ID() string {
return ot.id
}
func (ot *objectTree) Header() *aclpb.Header {
return ot.header
}
func (ot *objectTree) Storage() storage.TreeStorage {
return ot.treeStorage
}
func (ot *objectTree) AddContent(ctx context.Context, content SignableChangeContent) (rawChange *aclpb.RawChange, err error) {
defer func() {
if err == nil && ot.updateListener != nil {
ot.updateListener.Update(ot)
}
}()
payload, err := ot.prepareBuilderContent(content)
if err != nil {
return
}
objChange, rawChange, err := ot.changeBuilder.BuildContent(payload)
if content.IsSnapshot {
// clearing tree, because we already fixed everything in the last snapshot
ot.tree = &Tree{}
}
err = ot.tree.AddMergedHead(objChange)
if err != nil {
panic(err)
}
err = ot.treeStorage.AddRawChange(rawChange)
if err != nil {
return
}
err = ot.treeStorage.SetHeads([]string{objChange.Id})
return
}
func (ot *objectTree) prepareBuilderContent(content SignableChangeContent) (cnt BuilderContent, err error) {
ot.aclList.RLock()
defer ot.aclList.RUnlock()
state := ot.aclList.ACLState() // special method for own keys
readKey, err := state.CurrentReadKey()
if err != nil {
return
}
cnt = BuilderContent{
treeHeadIds: ot.tree.Heads(),
aclHeadId: ot.aclList.Head().Id,
snapshotBaseId: ot.tree.RootId(),
currentReadKeyHash: state.CurrentReadKeyHash(),
identity: content.Identity,
isSnapshot: content.IsSnapshot,
signingKey: content.Key,
readKey: readKey,
content: content.Proto,
}
return
}
func (ot *objectTree) AddRawChanges(ctx context.Context, rawChanges ...*aclpb.RawChange) (addResult AddResult, err error) {
var mode Mode
mode, addResult, err = ot.addRawChanges(ctx, rawChanges...)
if err != nil {
return
}
// reducing tree if we have new roots
ot.tree.reduceTree()
// adding to database all the added changes only after they are good
for _, ch := range addResult.Added {
err = ot.treeStorage.AddRawChange(ch)
if err != nil {
return
}
}
// setting heads
err = ot.treeStorage.SetHeads(ot.tree.Heads())
if err != nil {
return
}
if ot.updateListener == nil {
return
}
switch mode {
case Append:
ot.updateListener.Update(ot)
case Rebuild:
ot.updateListener.Rebuild(ot)
default:
break
}
return
}
func (ot *objectTree) addRawChanges(ctx context.Context, rawChanges ...*aclpb.RawChange) (mode Mode, addResult AddResult, err error) {
// resetting buffers
ot.tmpChangesBuf = ot.tmpChangesBuf[:0]
ot.notSeenIdxBuf = ot.notSeenIdxBuf[:0]
ot.difSnapshotBuf = ot.difSnapshotBuf[:0]
ot.newSnapshotsBuf = ot.newSnapshotsBuf[:0]
headsCopy := func() []string {
newHeads := make([]string, 0, len(ot.tree.Heads()))
newHeads = append(newHeads, ot.tree.Heads()...)
return newHeads
}
// this will be returned to client, so we shouldn't use buffer here
prevHeadsCopy := headsCopy()
// filtering changes, verifying and unmarshalling them
for idx, ch := range rawChanges {
if ot.HasChange(ch.Id) {
continue
}
var change *Change
change, err = ot.changeBuilder.ConvertFromRawAndVerify(ch)
if err != nil {
return
}
if change.IsSnapshot {
ot.newSnapshotsBuf = append(ot.newSnapshotsBuf, change)
}
ot.tmpChangesBuf = append(ot.tmpChangesBuf, change)
ot.notSeenIdxBuf = append(ot.notSeenIdxBuf, idx)
}
// if no new changes, then returning
if len(ot.notSeenIdxBuf) == 0 {
addResult = AddResult{
OldHeads: prevHeadsCopy,
Heads: prevHeadsCopy,
Summary: AddResultSummaryNothing,
}
return
}
// returns changes that we added to the tree
getAddedChanges := func() []*aclpb.RawChange {
var added []*aclpb.RawChange
for _, idx := range ot.notSeenIdxBuf {
rawChange := rawChanges[idx]
if _, exists := ot.tree.attached[rawChange.Id]; exists {
added = append(added, rawChange)
}
}
return added
}
rollback := func() {
for _, ch := range ot.tmpChangesBuf {
if _, exists := ot.tree.attached[ch.Id]; exists {
delete(ot.tree.attached, ch.Id)
} else if _, exists := ot.tree.unAttached[ch.Id]; exists {
delete(ot.tree.unAttached, ch.Id)
}
}
}
// checks if we need to go to database
isOldSnapshot := func(ch *Change) bool {
if ch.SnapshotId == ot.tree.RootId() {
return false
}
for _, sn := range ot.newSnapshotsBuf {
// if change refers to newly received snapshot
if ch.SnapshotId == sn.Id {
return false
}
}
return true
}
// checking if we have some changes with different snapshot and then rebuilding
for _, ch := range ot.tmpChangesBuf {
if isOldSnapshot(ch) {
err = ot.rebuildFromStorage(ot.tmpChangesBuf)
if err != nil {
// rebuilding without new changes
ot.rebuildFromStorage(nil)
return
}
addResult = AddResult{
OldHeads: prevHeadsCopy,
Heads: headsCopy(),
Added: getAddedChanges(),
Summary: AddResultSummaryRebuild,
}
return
}
}
// normal mode of operation, where we don't need to rebuild from database
mode = ot.tree.Add(ot.tmpChangesBuf...)
switch mode {
case Nothing:
addResult = AddResult{
OldHeads: prevHeadsCopy,
Heads: prevHeadsCopy,
Summary: AddResultSummaryNothing,
}
return
default:
// just rebuilding the state from start without reloading everything from tree storage
// as an optimization we could've started from current heads, but I didn't implement that
err = ot.validateTree()
if err != nil {
rollback()
err = ErrHasInvalidChanges
return
}
addResult = AddResult{
OldHeads: prevHeadsCopy,
Heads: headsCopy(),
Added: getAddedChanges(),
Summary: AddResultSummaryAppend,
}
}
return
}
func (ot *objectTree) Iterate(convert ChangeConvertFunc, iterate ChangeIterateFunc) (err error) {
return ot.IterateFrom(ot.tree.RootId(), convert, iterate)
}
func (ot *objectTree) IterateFrom(id string, convert ChangeConvertFunc, iterate ChangeIterateFunc) (err error) {
if convert == nil {
ot.tree.Iterate(id, iterate)
return
}
ot.tree.Iterate(ot.tree.RootId(), func(c *Change) (isContinue bool) {
var model any
if c.ParsedModel != nil {
return iterate(c)
}
readKey, exists := ot.keys[c.Content.CurrentReadKeyHash]
if !exists {
err = list.ErrNoReadKey
return false
}
var decrypted []byte
decrypted, err = readKey.Decrypt(c.Content.GetChangesData())
if err != nil {
return false
}
model, err = convert(decrypted)
if err != nil {
return false
}
c.ParsedModel = model
return iterate(c)
})
return
}
func (ot *objectTree) HasChange(s string) bool {
_, attachedExists := ot.tree.attached[s]
_, unattachedExists := ot.tree.unAttached[s]
return attachedExists || unattachedExists
}
func (ot *objectTree) Heads() []string {
return ot.tree.Heads()
}
func (ot *objectTree) Root() *Change {
return ot.tree.Root()
}
func (ot *objectTree) Close() error {
return nil
}
func (ot *objectTree) SnapshotPath() []string {
// TODO: Add error as return parameter
if ot.snapshotPathIsActual() {
return ot.snapshotPath
}
var path []string
// TODO: think that the user may have not all of the snapshots locally
currentSnapshotId := ot.tree.RootId()
for currentSnapshotId != "" {
sn, err := ot.treeBuilder.loadChange(currentSnapshotId)
if err != nil {
break
}
path = append(path, currentSnapshotId)
currentSnapshotId = sn.SnapshotId
}
ot.snapshotPath = path
return path
}
func (ot *objectTree) ChangesAfterCommonSnapshot(theirPath, theirHeads []string) ([]*aclpb.RawChange, error) {
var (
needFullDocument = len(theirPath) == 0
ourPath = ot.SnapshotPath()
// by default returning everything we have from start
commonSnapshot = ourPath[len(ourPath)-1]
err error
)
// if this is non-empty request
if !needFullDocument {
commonSnapshot, err = commonSnapshotForTwoPaths(ourPath, theirPath)
if err != nil {
return nil, err
}
}
if commonSnapshot == ot.tree.RootId() {
return ot.getChangesFromTree(theirHeads)
} else {
return ot.getChangesFromDB(commonSnapshot, theirHeads)
}
}
func (ot *objectTree) getChangesFromTree(theirHeads []string) (rawChanges []*aclpb.RawChange, err error) {
return ot.rawChangeLoader.LoadFromTree(ot.tree, theirHeads)
}
func (ot *objectTree) getChangesFromDB(commonSnapshot string, theirHeads []string) (rawChanges []*aclpb.RawChange, err error) {
return ot.rawChangeLoader.LoadFromStorage(commonSnapshot, ot.tree.headIds, theirHeads)
}
func (ot *objectTree) snapshotPathIsActual() bool {
return len(ot.snapshotPath) != 0 && ot.snapshotPath[0] == ot.tree.RootId()
}
func (ot *objectTree) validateTree() error {
ot.aclList.RLock()
defer ot.aclList.RUnlock()
state := ot.aclList.ACLState()
// just not to take lock many times, updating the key map from aclList
if len(ot.keys) != len(state.UserReadKeys()) {
for key, value := range state.UserReadKeys() {
ot.keys[key] = value
}
}
return ot.validator.ValidateTree(ot.tree, ot.aclList)
}
func (ot *objectTree) DebugDump() (string, error) {
return ot.tree.Graph(NoOpDescriptionParser)
}

476
pkg/acl/tree/objecttree_test.go Normal file
View File

@ -0,0 +1,476 @@
package tree
import (
"context"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/list"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/acllistbuilder"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"github.com/gogo/protobuf/proto"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing"
)
type mockChangeCreator struct{}
func (c *mockChangeCreator) createRaw(id, aclId, snapshotId string, isSnapshot bool, prevIds ...string) *aclpb.RawChange {
aclChange := &aclpb.Change{
TreeHeadIds: prevIds,
AclHeadId: aclId,
SnapshotBaseId: snapshotId,
ChangesData: nil,
IsSnapshot: isSnapshot,
}
res, _ := aclChange.Marshal()
return &aclpb.RawChange{
Payload: res,
Signature: nil,
Id: id,
}
}
func (c *mockChangeCreator) createNewTreeStorage(treeId, aclListId, aclHeadId, firstChangeId string) storage.TreeStorage {
firstChange := c.createRaw(firstChangeId, aclHeadId, "", true)
header := &aclpb.Header{
FirstId: firstChangeId,
AclListId: aclListId,
WorkspaceId: "",
DocType: aclpb.Header_DocTree,
}
treeStorage, _ := storage.NewInMemoryTreeStorage(treeId, header, []string{firstChangeId}, []*aclpb.RawChange{firstChange})
return treeStorage
}
type mockChangeBuilder struct{}
func (c *mockChangeBuilder) ConvertFromRaw(rawChange *aclpb.RawChange) (ch *Change, err error) {
unmarshalled := &aclpb.Change{}
err = proto.Unmarshal(rawChange.Payload, unmarshalled)
if err != nil {
return nil, err
}
ch = NewChange(rawChange.Id, unmarshalled, rawChange.Signature)
return
}
func (c *mockChangeBuilder) ConvertFromRawAndVerify(rawChange *aclpb.RawChange) (ch *Change, err error) {
return c.ConvertFromRaw(rawChange)
}
func (c *mockChangeBuilder) BuildContent(payload BuilderContent) (ch *Change, raw *aclpb.RawChange, err error) {
panic("implement me")
}
type mockChangeValidator struct{}
func (m *mockChangeValidator) ValidateTree(tree *Tree, aclList list.ACLList) error {
return nil
}
type testTreeContext struct {
aclList list.ACLList
treeStorage storage.TreeStorage
changeBuilder *mockChangeBuilder
changeCreator *mockChangeCreator
objTree ObjectTree
}
func prepareACLList(t *testing.T) list.ACLList {
st, err := acllistbuilder.NewListStorageWithTestName("userjoinexample.yml")
require.NoError(t, err, "building storage should not result in error")
aclList, err := list.BuildACLList(signingkey.NewEDPubKeyDecoder(), st)
require.NoError(t, err, "building acl list should be without error")
return aclList
}
func prepareTreeContext(t *testing.T, aclList list.ACLList) testTreeContext {
changeCreator := &mockChangeCreator{}
treeStorage := changeCreator.createNewTreeStorage("treeId", aclList.ID(), aclList.Head().Id, "0")
changeBuilder := &mockChangeBuilder{}
deps := objectTreeDeps{
changeBuilder: changeBuilder,
treeBuilder: newTreeBuilder(treeStorage, changeBuilder),
treeStorage: treeStorage,
updateListener: nil,
rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder),
validator: &mockChangeValidator{},
aclList: aclList,
}
// check build
objTree, err := buildObjectTree(deps)
require.NoError(t, err, "building tree should be without error")
// check tree iterate
var iterChangesId []string
err = objTree.Iterate(nil, func(change *Change) bool {
iterChangesId = append(iterChangesId, change.Id)
return true
})
require.NoError(t, err, "iterate should be without error")
assert.Equal(t, []string{"0"}, iterChangesId)
return testTreeContext{
aclList: aclList,
treeStorage: treeStorage,
changeBuilder: changeBuilder,
changeCreator: changeCreator,
objTree: objTree,
}
}
func TestObjectTree(t *testing.T) {
aclList := prepareACLList(t)
t.Run("add simple", func(t *testing.T) {
ctx := prepareTreeContext(t, aclList)
treeStorage := ctx.treeStorage
changeCreator := ctx.changeCreator
objTree := ctx.objTree
rawChanges := []*aclpb.RawChange{
changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"),
changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"),
}
res, err := objTree.AddRawChanges(context.Background(), rawChanges...)
require.NoError(t, err, "adding changes should be without error")
// check result
assert.Equal(t, []string{"0"}, res.OldHeads)
assert.Equal(t, []string{"2"}, res.Heads)
assert.Equal(t, len(rawChanges), len(res.Added))
assert.Equal(t, AddResultSummaryAppend, res.Summary)
// check tree heads
assert.Equal(t, []string{"2"}, objTree.Heads())
// check tree iterate
var iterChangesId []string
err = objTree.Iterate(nil, func(change *Change) bool {
iterChangesId = append(iterChangesId, change.Id)
return true
})
require.NoError(t, err, "iterate should be without error")
assert.Equal(t, []string{"0", "1", "2"}, iterChangesId)
// check storage
heads, _ := treeStorage.Heads()
assert.Equal(t, []string{"2"}, heads)
for _, ch := range rawChanges {
raw, err := treeStorage.GetRawChange(context.Background(), ch.Id)
assert.NoError(t, err, "storage should have all the changes")
assert.Equal(t, ch, raw, "the changes in the storage should be the same")
}
})
t.Run("add no new changes", func(t *testing.T) {
ctx := prepareTreeContext(t, aclList)
changeCreator := ctx.changeCreator
objTree := ctx.objTree
rawChanges := []*aclpb.RawChange{
changeCreator.createRaw("0", aclList.Head().Id, "", true, ""),
}
res, err := objTree.AddRawChanges(context.Background(), rawChanges...)
require.NoError(t, err, "adding changes should be without error")
// check result
assert.Equal(t, []string{"0"}, res.OldHeads)
assert.Equal(t, []string{"0"}, res.Heads)
assert.Equal(t, 0, len(res.Added))
// check tree heads
assert.Equal(t, []string{"0"}, objTree.Heads())
})
t.Run("add unattachable changes", func(t *testing.T) {
ctx := prepareTreeContext(t, aclList)
changeCreator := ctx.changeCreator
objTree := ctx.objTree
rawChanges := []*aclpb.RawChange{
changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"),
}
res, err := objTree.AddRawChanges(context.Background(), rawChanges...)
require.NoError(t, err, "adding changes should be without error")
// check result
assert.Equal(t, []string{"0"}, res.OldHeads)
assert.Equal(t, []string{"0"}, res.Heads)
assert.Equal(t, 0, len(res.Added))
assert.Equal(t, AddResultSummaryNothing, res.Summary)
// check tree heads
assert.Equal(t, []string{"0"}, objTree.Heads())
})
t.Run("add new snapshot simple", func(t *testing.T) {
ctx := prepareTreeContext(t, aclList)
treeStorage := ctx.treeStorage
changeCreator := ctx.changeCreator
objTree := ctx.objTree
rawChanges := []*aclpb.RawChange{
changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"),
changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"),
changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"),
changeCreator.createRaw("4", aclList.Head().Id, "3", false, "3"),
}
res, err := objTree.AddRawChanges(context.Background(), rawChanges...)
require.NoError(t, err, "adding changes should be without error")
// check result
assert.Equal(t, []string{"0"}, res.OldHeads)
assert.Equal(t, []string{"4"}, res.Heads)
assert.Equal(t, len(rawChanges), len(res.Added))
assert.Equal(t, AddResultSummaryAppend, res.Summary)
// check tree heads
assert.Equal(t, []string{"4"}, objTree.Heads())
// check tree iterate
var iterChangesId []string
err = objTree.Iterate(nil, func(change *Change) bool {
iterChangesId = append(iterChangesId, change.Id)
return true
})
require.NoError(t, err, "iterate should be without error")
assert.Equal(t, []string{"3", "4"}, iterChangesId)
assert.Equal(t, "3", objTree.Root().Id)
// check storage
heads, _ := treeStorage.Heads()
assert.Equal(t, []string{"4"}, heads)
for _, ch := range rawChanges {
raw, err := treeStorage.GetRawChange(context.Background(), ch.Id)
assert.NoError(t, err, "storage should have all the changes")
assert.Equal(t, ch, raw, "the changes in the storage should be the same")
}
})
t.Run("snapshot path", func(t *testing.T) {
ctx := prepareTreeContext(t, aclList)
changeCreator := ctx.changeCreator
objTree := ctx.objTree
rawChanges := []*aclpb.RawChange{
changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"),
changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"),
changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"),
}
_, err := objTree.AddRawChanges(context.Background(), rawChanges...)
require.NoError(t, err, "adding changes should be without error")
snapshotPath := objTree.SnapshotPath()
assert.Equal(t, []string{"3", "0"}, snapshotPath)
assert.Equal(t, true, objTree.(*objectTree).snapshotPathIsActual())
})
t.Run("changes from tree after common snapshot complex", func(t *testing.T) {
ctx := prepareTreeContext(t, aclList)
changeCreator := ctx.changeCreator
objTree := ctx.objTree
rawChanges := []*aclpb.RawChange{
changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"),
changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"),
changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"),
changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"),
changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"),
changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"),
}
_, err := objTree.AddRawChanges(context.Background(), rawChanges...)
require.NoError(t, err, "adding changes should be without error")
require.Equal(t, "0", objTree.Root().Id)
t.Run("all changes from tree", func(t *testing.T) {
changes, err := objTree.ChangesAfterCommonSnapshot([]string{"3", "0"}, []string{})
require.NoError(t, err, "changes after common snapshot should be without error")
changeIds := make(map[string]struct{})
for _, ch := range changes {
changeIds[ch.Id] = struct{}{}
}
for _, raw := range rawChanges {
_, ok := changeIds[raw.Id]
assert.Equal(t, true, ok)
}
_, ok := changeIds["0"]
assert.Equal(t, true, ok)
})
t.Run("changes from tree after 1", func(t *testing.T) {
changes, err := objTree.ChangesAfterCommonSnapshot([]string{"3", "0"}, []string{"1"})
require.NoError(t, err, "changes after common snapshot should be without error")
changeIds := make(map[string]struct{})
for _, ch := range changes {
changeIds[ch.Id] = struct{}{}
}
for _, id := range []string{"2", "3", "4", "5", "6"} {
_, ok := changeIds[id]
assert.Equal(t, true, ok)
}
for _, id := range []string{"0", "1"} {
_, ok := changeIds[id]
assert.Equal(t, false, ok)
}
})
t.Run("changes from tree after 5", func(t *testing.T) {
changes, err := objTree.ChangesAfterCommonSnapshot([]string{"3", "0"}, []string{"5"})
require.NoError(t, err, "changes after common snapshot should be without error")
changeIds := make(map[string]struct{})
for _, ch := range changes {
changeIds[ch.Id] = struct{}{}
}
for _, id := range []string{"2", "3", "4", "6"} {
_, ok := changeIds[id]
assert.Equal(t, true, ok)
}
for _, id := range []string{"0", "1", "5"} {
_, ok := changeIds[id]
assert.Equal(t, false, ok)
}
})
})
t.Run("changes after common snapshot db complex", func(t *testing.T) {
ctx := prepareTreeContext(t, aclList)
changeCreator := ctx.changeCreator
objTree := ctx.objTree
rawChanges := []*aclpb.RawChange{
changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"),
changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"),
changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"),
changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"),
changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"),
// main difference from tree example
changeCreator.createRaw("6", aclList.Head().Id, "0", true, "3", "4", "5"),
}
_, err := objTree.AddRawChanges(context.Background(), rawChanges...)
require.NoError(t, err, "adding changes should be without error")
require.Equal(t, "6", objTree.Root().Id)
t.Run("all changes from db", func(t *testing.T) {
changes, err := objTree.ChangesAfterCommonSnapshot([]string{"3", "0"}, []string{})
require.NoError(t, err, "changes after common snapshot should be without error")
changeIds := make(map[string]struct{})
for _, ch := range changes {
changeIds[ch.Id] = struct{}{}
}
for _, raw := range rawChanges {
_, ok := changeIds[raw.Id]
assert.Equal(t, true, ok)
}
_, ok := changeIds["0"]
assert.Equal(t, true, ok)
})
t.Run("changes from tree db 1", func(t *testing.T) {
changes, err := objTree.ChangesAfterCommonSnapshot([]string{"3", "0"}, []string{"1"})
require.NoError(t, err, "changes after common snapshot should be without error")
changeIds := make(map[string]struct{})
for _, ch := range changes {
changeIds[ch.Id] = struct{}{}
}
for _, id := range []string{"2", "3", "4", "5", "6"} {
_, ok := changeIds[id]
assert.Equal(t, true, ok)
}
for _, id := range []string{"0", "1"} {
_, ok := changeIds[id]
assert.Equal(t, false, ok)
}
})
t.Run("changes from tree db 5", func(t *testing.T) {
changes, err := objTree.ChangesAfterCommonSnapshot([]string{"3", "0"}, []string{"5"})
require.NoError(t, err, "changes after common snapshot should be without error")
changeIds := make(map[string]struct{})
for _, ch := range changes {
changeIds[ch.Id] = struct{}{}
}
for _, id := range []string{"2", "3", "4", "6"} {
_, ok := changeIds[id]
assert.Equal(t, true, ok)
}
for _, id := range []string{"0", "1", "5"} {
_, ok := changeIds[id]
assert.Equal(t, false, ok)
}
})
})
t.Run("add new changes related to previous snapshot", func(t *testing.T) {
ctx := prepareTreeContext(t, aclList)
treeStorage := ctx.treeStorage
changeCreator := ctx.changeCreator
objTree := ctx.objTree
rawChanges := []*aclpb.RawChange{
changeCreator.createRaw("1", aclList.Head().Id, "0", false, "0"),
changeCreator.createRaw("2", aclList.Head().Id, "0", false, "1"),
changeCreator.createRaw("3", aclList.Head().Id, "0", true, "2"),
}
res, err := objTree.AddRawChanges(context.Background(), rawChanges...)
require.NoError(t, err, "adding changes should be without error")
require.Equal(t, "3", objTree.Root().Id)
rawChanges = []*aclpb.RawChange{
changeCreator.createRaw("4", aclList.Head().Id, "0", false, "2"),
changeCreator.createRaw("5", aclList.Head().Id, "0", false, "1"),
changeCreator.createRaw("6", aclList.Head().Id, "0", false, "3", "4", "5"),
}
res, err = objTree.AddRawChanges(context.Background(), rawChanges...)
require.NoError(t, err, "adding changes should be without error")
// check result
assert.Equal(t, []string{"3"}, res.OldHeads)
assert.Equal(t, []string{"6"}, res.Heads)
assert.Equal(t, len(rawChanges), len(res.Added))
assert.Equal(t, AddResultSummaryRebuild, res.Summary)
// check tree heads
assert.Equal(t, []string{"6"}, objTree.Heads())
// check tree iterate
var iterChangesId []string
err = objTree.Iterate(nil, func(change *Change) bool {
iterChangesId = append(iterChangesId, change.Id)
return true
})
require.NoError(t, err, "iterate should be without error")
assert.Equal(t, []string{"0", "1", "2", "3", "4", "5", "6"}, iterChangesId)
assert.Equal(t, "0", objTree.Root().Id)
// check storage
heads, _ := treeStorage.Heads()
assert.Equal(t, []string{"6"}, heads)
for _, ch := range rawChanges {
raw, err := treeStorage.GetRawChange(context.Background(), ch.Id)
assert.NoError(t, err, "storage should have all the changes")
assert.Equal(t, ch, raw, "the changes in the storage should be the same")
}
})
}

255
pkg/acl/tree/rawloader.go Normal file
View File

@ -0,0 +1,255 @@
package tree
import (
"context"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
"time"
)
type rawChangeLoader struct {
treeStorage storage.TreeStorage
changeBuilder ChangeBuilder
// buffers
idStack []string
cache map[string]rawCacheEntry
}
type rawCacheEntry struct {
change *Change
rawChange *aclpb.RawChange
position int
}
func newRawChangeLoader(treeStorage storage.TreeStorage, changeBuilder ChangeBuilder) *rawChangeLoader {
return &rawChangeLoader{
treeStorage: treeStorage,
changeBuilder: changeBuilder,
}
}
func (r *rawChangeLoader) LoadFromTree(t *Tree, breakpoints []string) ([]*aclpb.RawChange, error) {
var stack []*Change
for _, h := range t.headIds {
stack = append(stack, t.attached[h])
}
convert := func(chs []*Change) (rawChanges []*aclpb.RawChange, err error) {
for _, ch := range chs {
var marshalled []byte
marshalled, err = ch.Content.Marshal()
if err != nil {
return
}
raw := &aclpb.RawChange{
Payload: marshalled,
Signature: ch.Signature(),
Id: ch.Id,
}
rawChanges = append(rawChanges, raw)
}
return
}
// getting all changes that we visit
var results []*Change
rootVisited := false
t.dfsPrev(
stack,
breakpoints,
func(ch *Change) bool {
results = append(results, ch)
return true
},
func(visited []*Change) {
if t.root.visited {
rootVisited = true
}
},
)
// if we stopped at breakpoints or there are no breakpoints
if !rootVisited || len(breakpoints) == 0 {
// in this case we will add root if there are no breakpoints
return convert(results)
}
// now starting from breakpoints
stack = stack[:0]
for _, h := range breakpoints {
stack = append(stack, t.attached[h])
}
// doing another dfs to get all changes before breakpoints, we need to exclude them from results
// if we don't have some breakpoints we will just ignore them
t.dfsPrev(
stack,
[]string{},
func(ch *Change) bool {
return true
},
func(visited []*Change) {
results = discardFromSlice(results, func(change *Change) bool {
return change.visited
})
},
)
// otherwise we want to exclude everything that wasn't in breakpoints
return convert(results)
}
func (r *rawChangeLoader) LoadFromStorage(commonSnapshot string, heads, breakpoints []string) ([]*aclpb.RawChange, error) {
// resetting cache
r.cache = make(map[string]rawCacheEntry)
defer func() {
r.cache = nil
}()
existingBreakpoints := make([]string, 0, len(breakpoints))
for _, b := range breakpoints {
entry, err := r.loadEntry(b)
if err != nil {
continue
}
entry.position = -1
r.cache[b] = entry
existingBreakpoints = append(existingBreakpoints, b)
}
r.cache[commonSnapshot] = rawCacheEntry{position: -1}
dfs := func(
commonSnapshot string,
heads []string,
startCounter int,
shouldVisit func(counter int, mapExists bool) bool,
visit func(entry rawCacheEntry) rawCacheEntry) bool {
// resetting stack
r.idStack = r.idStack[:0]
r.idStack = append(r.idStack, heads...)
commonSnapshotVisited := false
var err error
for len(r.idStack) > 0 {
id := r.idStack[len(r.idStack)-1]
r.idStack = r.idStack[:len(r.idStack)-1]
entry, exists := r.cache[id]
if !shouldVisit(entry.position, exists) {
continue
}
if !exists {
entry, err = r.loadEntry(id)
if err != nil {
continue
}
}
// setting the counter when we visit
r.cache[id] = visit(entry)
for _, prev := range entry.change.PreviousIds {
if prev == commonSnapshot {
commonSnapshotVisited = true
break
}
entry, exists = r.cache[prev]
if !shouldVisit(entry.position, exists) {
continue
}
r.idStack = append(r.idStack, prev)
}
}
return commonSnapshotVisited
}
// preparing first pass
r.idStack = append(r.idStack, heads...)
var buffer []*aclpb.RawChange
rootVisited := dfs(commonSnapshot, heads, 0,
func(counter int, mapExists bool) bool {
return !mapExists
},
func(entry rawCacheEntry) rawCacheEntry {
buffer = append(buffer, entry.rawChange)
entry.position = len(buffer) - 1
return entry
})
// checking if we stopped at breakpoints
if !rootVisited {
return buffer, nil
}
// if there are no breakpoints then we should load root also
if len(breakpoints) == 0 {
common, err := r.loadEntry(commonSnapshot)
if err != nil {
return nil, err
}
buffer = append(buffer, common.rawChange)
return buffer, nil
}
// marking all visited as nil
dfs(commonSnapshot, existingBreakpoints, len(buffer),
func(counter int, mapExists bool) bool {
return !mapExists || counter < len(buffer)
},
func(entry rawCacheEntry) rawCacheEntry {
if entry.position != -1 {
buffer[entry.position] = nil
}
entry.position = len(buffer) + 1
return entry
})
// discarding visited
buffer = discardFromSlice(buffer, func(change *aclpb.RawChange) bool {
return change == nil
})
return buffer, nil
}
func (r *rawChangeLoader) loadEntry(id string) (entry rawCacheEntry, err error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
rawChange, err := r.treeStorage.GetRawChange(ctx, id)
if err != nil {
return
}
change, err := r.changeBuilder.ConvertFromRaw(rawChange)
if err != nil {
return
}
entry = rawCacheEntry{
change: change,
rawChange: rawChange,
}
return
}
func discardFromSlice[T any](elements []T, isDiscarded func(T) bool) []T {
var (
finishedIdx = 0
currentIdx = 0
)
for currentIdx < len(elements) {
if !isDiscarded(elements[currentIdx]) {
if finishedIdx != currentIdx {
elements[finishedIdx] = elements[currentIdx]
}
finishedIdx++
}
currentIdx++
}
elements = elements[:finishedIdx]
return elements
}

13
pkg/acl/tree/signablecontent.go Normal file
View File

@ -0,0 +1,13 @@
package tree
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"github.com/gogo/protobuf/proto"
)
type SignableChangeContent struct {
Proto proto.Marshaler
Key signingkey.PrivKey
Identity string
IsSnapshot bool
}

186
pkg/acl/acltree/tree.go → pkg/acl/tree/tree.go
View File

@ -1,10 +1,9 @@
package acltree
package tree
import (
"bytes"
"crypto/md5"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice"
"sort"
)
@ -16,7 +15,6 @@ const (
Nothing
)
// TODO: consider abstracting into separate package with iterator, remove
type Tree struct {
root *Change
headIds []string
@ -26,18 +24,15 @@ type Tree struct {
// missed id -> list of dependency ids
waitList map[string][]string
invalidChanges map[string]struct{}
possibleRoots []*Change
// bufs
iterCompBuf []*Change
iterQueue []*Change
visitedBuf []*Change
stackBuf []*Change
duplicateEvents int
}
func (t *Tree) GetUnattachedChanges(changes ...*Change) []*Change {
return nil
}
func (t *Tree) RootId() string {
if t.root != nil {
return t.root.Id
@ -62,6 +57,30 @@ func (t *Tree) AddFast(changes ...*Change) {
t.updateHeads()
}
func (t *Tree) AddMergedHead(c *Change) error {
// check that it was not inserted previously
if _, ok := t.attached[c.Id]; ok {
return fmt.Errorf("change already exists") // TODO: named error
} else if _, ok := t.unAttached[c.Id]; ok {
return fmt.Errorf("change already exists")
}
// check that it was attached after adding
if !t.add(c) {
return fmt.Errorf("change is not attached")
}
// check that previous heads have new change as next
for _, prevHead := range t.headIds {
head := t.attached[prevHead]
if len(head.Next) != 1 || head.Next[0].Id != c.Id {
return fmt.Errorf("this is not a new head")
}
}
t.headIds = []string{c.Id}
return nil
}
func (t *Tree) Add(changes ...*Change) (mode Mode) {
var beforeHeadIds = t.headIds
var attached bool
@ -96,6 +115,7 @@ func (t *Tree) Add(changes ...*Change) (mode Mode) {
return Append
}
// RemoveInvalidChange removes all the changes that are descendants of id
func (t *Tree) RemoveInvalidChange(id string) {
stack := []string{id}
// removing all children of this id (either next or unattached)
@ -112,6 +132,7 @@ func (t *Tree) RemoveInvalidChange(id string) {
t.invalidChanges[top] = struct{}{}
if rem, exists = t.unAttached[top]; exists {
delete(t.unAttached, top)
// TODO: delete waitlist, this can only help for memory/performance
} else if rem, exists = t.attached[top]; exists {
// remove from all prev changes
for _, id := range rem.PreviousIds {
@ -129,9 +150,6 @@ func (t *Tree) RemoveInvalidChange(id string) {
}
delete(t.attached, top)
}
for _, el := range rem.Unattached {
stack = append(stack, el.Id)
}
for _, el := range rem.Next {
stack = append(stack, el.Id)
}
@ -155,6 +173,7 @@ func (t *Tree) add(c *Change) (attached bool) {
t.unAttached = make(map[string]*Change)
t.waitList = make(map[string][]string)
t.invalidChanges = make(map[string]struct{})
t.possibleRoots = make([]*Change, 0, 10)
return true
}
if len(c.PreviousIds) > 1 {
@ -163,15 +182,11 @@ func (t *Tree) add(c *Change) (attached bool) {
// attaching only if all prev ids are attached
attached = true
for _, pid := range c.PreviousIds {
if prev, ok := t.attached[pid]; ok {
prev.Unattached = append(prev.Unattached, c)
if _, ok := t.attached[pid]; ok {
continue
}
attached = false
if prev, ok := t.unAttached[pid]; ok {
prev.Unattached = append(prev.Unattached, c)
continue
}
// updating wait list for either unseen or unAttached changes
wl := t.waitList[pid]
wl = append(wl, c.Id)
t.waitList[pid] = wl
@ -179,11 +194,6 @@ func (t *Tree) add(c *Change) (attached bool) {
if attached {
t.attach(c, true)
} else {
// clearing wait list
for _, wid := range t.waitList[c.Id] {
c.Unattached = append(c.Unattached, t.unAttached[wid])
}
delete(t.waitList, c.Id)
t.unAttached[c.Id] = c
}
return
@ -197,6 +207,7 @@ func (t *Tree) canAttach(c *Change) (attach bool) {
for _, id := range c.PreviousIds {
if _, exists := t.attached[id]; !exists {
attach = false
break
}
}
return
@ -207,40 +218,46 @@ func (t *Tree) attach(c *Change, newEl bool) {
if !newEl {
delete(t.unAttached, c.Id)
}
if c.IsSnapshot {
t.possibleRoots = append(t.possibleRoots, c)
}
// add next to all prev changes
for _, id := range c.PreviousIds {
// prev id must be attached if we attach this id
// prev id must already be attached if we attach this id, so we don't need to check if it exists
prev := t.attached[id]
prev.Next = append(prev.Next, c)
if len(prev.Next) > 1 {
sort.Sort(sortChanges(prev.Next))
}
for i, next := range prev.Unattached {
if next.Id == c.Id {
prev.Unattached[i] = nil
prev.Unattached = append(prev.Unattached[:i], prev.Unattached[i+1:]...)
break
// appending c to next changes of all previous changes
if len(prev.Next) == 0 || prev.Next[len(prev.Next)-1].Id <= c.Id {
prev.Next = append(prev.Next, c)
} else {
// inserting in correct position, before the change which is greater or equal
insertIdx := 0
for idx, el := range prev.Next {
if el.Id >= c.Id {
insertIdx = idx
break
}
}
prev.Next = append(prev.Next[:insertIdx+1], prev.Next[insertIdx:]...)
prev.Next[insertIdx] = c
}
}
// TODO: as a future optimization we can actually sort next later after we finished building the tree
// clearing wait list
if waitIds, ok := t.waitList[c.Id]; ok {
for _, wid := range waitIds {
// next can only be in unAttached, because if next is attached then previous (we) are attached
// which is obviously not true, because we are attaching previous only now
next := t.unAttached[wid]
if t.canAttach(next) {
t.attach(next, false)
}
// if we can't attach next that means that some other change will trigger attachment later,
// so we don't care about those changes
}
delete(t.waitList, c.Id)
}
for _, next := range c.Unattached {
if t.canAttach(next) {
t.attach(next, false)
}
}
}
func (t *Tree) after(id1, id2 string) (found bool) {
@ -254,29 +271,48 @@ func (t *Tree) after(id1, id2 string) (found bool) {
return
}
func (t *Tree) dfs(startChange string) (uniqMap map[string]*Change) {
stack := make([]*Change, 0, 10)
stack = append(stack, t.attached[startChange])
uniqMap = map[string]*Change{}
func (t *Tree) dfsPrev(stack []*Change, breakpoints []string, visit func(ch *Change) (isContinue bool), afterVisit func([]*Change)) {
t.visitedBuf = t.visitedBuf[:0]
// setting breakpoints as visited
for _, breakpoint := range breakpoints {
if ch, ok := t.attached[breakpoint]; ok {
ch.visited = true
t.visitedBuf = append(t.visitedBuf, ch)
}
}
defer func() {
afterVisit(t.visitedBuf)
for _, ch := range t.visitedBuf {
ch.visited = false
}
}()
for len(stack) > 0 {
ch := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if _, exists := uniqMap[ch.Id]; exists {
if ch.visited {
continue
}
uniqMap[ch.Id] = ch
ch.visited = true
t.visitedBuf = append(t.visitedBuf, ch)
for _, prev := range ch.PreviousIds {
stack = append(stack, t.attached[prev])
for _, prevId := range ch.PreviousIds {
prevCh := t.attached[prevId]
if !prevCh.visited {
stack = append(stack, prevCh)
}
}
if !visit(ch) {
return
}
}
return uniqMap
}
func (t *Tree) updateHeads() {
var newHeadIds, newMetaHeadIds []string
var newHeadIds []string
t.iterate(t.root, func(c *Change) (isContinue bool) {
if len(c.Next) == 0 {
newHeadIds = append(newHeadIds, c.Id)
@ -284,37 +320,7 @@ func (t *Tree) updateHeads() {
return true
})
t.headIds = newHeadIds
t.metaHeadIds = newMetaHeadIds
sort.Strings(t.headIds)
sort.Strings(t.metaHeadIds)
}
func (t *Tree) ACLHeads() []string {
var aclTreeHeads []string
for _, head := range t.Heads() {
if slice.FindPos(aclTreeHeads, head) != -1 { // do not scan known heads
continue
}
precedingHeads := t.getPrecedingACLHeads(head)
for _, aclHead := range precedingHeads {
if slice.FindPos(aclTreeHeads, aclHead) != -1 {
continue
}
aclTreeHeads = append(aclTreeHeads, aclHead)
}
}
return aclTreeHeads
}
func (t *Tree) getPrecedingACLHeads(head string) []string {
headChange := t.attached[head]
if headChange.Content.GetAclData() != nil {
return []string{head}
} else {
return headChange.Content.AclHeadIds
}
}
func (t *Tree) iterate(start *Change, f func(c *Change) (isContinue bool)) {
@ -381,6 +387,14 @@ func (t *Tree) Heads() []string {
return t.headIds
}
func (t *Tree) HeadsChanges() []*Change {
var heads []*Change
for _, head := range t.headIds {
heads = append(heads, t.attached[head])
}
return heads
}
func (t *Tree) String() string {
var buf = bytes.NewBuffer(nil)
t.Iterate(t.RootId(), func(c *Change) (isContinue bool) {
@ -400,17 +414,3 @@ func (t *Tree) String() string {
func (t *Tree) Get(id string) *Change {
return t.attached[id]
}
type sortChanges []*Change
func (s sortChanges) Len() int {
return len(s)
}
func (s sortChanges) Less(i, j int) bool {
return s[i].Id < s[j].Id
}
func (s sortChanges) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}

325
pkg/acl/tree/tree_test.go Normal file
View File

@ -0,0 +1,325 @@
package tree
import (
"fmt"
"github.com/stretchr/testify/assert"
"math/rand"
"testing"
"time"
)
func newChange(id string, snapshotId string, prevIds ...string) *Change {
return &Change{
PreviousIds: prevIds,
Id: id,
SnapshotId: snapshotId,
IsSnapshot: false,
}
}
func newSnapshot(id, snapshotId string, prevIds ...string) *Change {
return &Change{
PreviousIds: prevIds,
Id: id,
SnapshotId: snapshotId,
IsSnapshot: true,
}
}
func TestTree_Add(t *testing.T) {
t.Run("add first el", func(t *testing.T) {
tr := new(Tree)
assert.Equal(t, Rebuild, tr.Add(newSnapshot("root", "")))
assert.Equal(t, tr.root.Id, "root")
assert.Equal(t, []string{"root"}, tr.Heads())
})
t.Run("linear add", func(t *testing.T) {
tr := new(Tree)
assert.Equal(t, Rebuild, tr.Add(
newSnapshot("root", ""),
newChange("one", "root", "root"),
newChange("two", "root", "one"),
))
assert.Equal(t, []string{"two"}, tr.Heads())
assert.Equal(t, Append, tr.Add(newChange("three", "root", "two")))
el := tr.root
var ids []string
for el != nil {
ids = append(ids, el.Id)
if len(el.Next) > 0 {
el = el.Next[0]
} else {
el = nil
}
}
assert.Equal(t, []string{"root", "one", "two", "three"}, ids)
assert.Equal(t, []string{"three"}, tr.Heads())
})
t.Run("branch", func(t *testing.T) {
tr := new(Tree)
assert.Equal(t, Rebuild, tr.Add(
newSnapshot("root", ""),
newChange("1", "root", "root"),
newChange("2", "root", "1"),
))
assert.Equal(t, []string{"2"}, tr.Heads())
assert.Equal(t, Rebuild, tr.Add(
newChange("1.2", "root", "1.1"),
newChange("1.3", "root", "1.2"),
newChange("1.1", "root", "1"),
))
assert.Len(t, tr.attached["1"].Next, 2)
assert.Len(t, tr.unAttached, 0)
assert.Len(t, tr.attached, 6)
assert.Equal(t, []string{"1.3", "2"}, tr.Heads())
})
t.Run("branch union", func(t *testing.T) {
tr := new(Tree)
assert.Equal(t, Rebuild, tr.Add(
newSnapshot("root", ""),
newChange("1", "root", "root"),
newChange("2", "root", "1"),
newChange("1.2", "root", "1.1"),
newChange("1.3", "root", "1.2"),
newChange("1.1", "root", "1"),
newChange("3", "root", "2", "1.3"),
newChange("4", "root", "3"),
))
assert.Len(t, tr.unAttached, 0)
assert.Len(t, tr.attached, 8)
assert.Equal(t, []string{"4"}, tr.Heads())
})
t.Run("big set", func(t *testing.T) {
tr := new(Tree)
tr.Add(newSnapshot("root", ""))
var changes []*Change
for i := 0; i < 10000; i++ {
if i == 0 {
changes = append(changes, newChange(fmt.Sprint(i), "root", "root"))
} else {
changes = append(changes, newChange(fmt.Sprint(i), "root", fmt.Sprint(i-1)))
}
}
st := time.Now()
tr.AddFast(changes...)
t.Log(time.Since(st))
assert.Equal(t, []string{"9999"}, tr.Heads())
})
// TODO: add my tests
}
func TestTree_Hash(t *testing.T) {
tr := new(Tree)
tr.Add(newSnapshot("root", ""))
hash1 := tr.Hash()
assert.Equal(t, tr.Hash(), hash1)
tr.Add(newChange("1", "root", "root"))
assert.NotEqual(t, tr.Hash(), hash1)
assert.Equal(t, tr.Hash(), tr.Hash())
}
func TestTree_AddFuzzy(t *testing.T) {
rand.Seed(time.Now().UnixNano())
getChanges := func() []*Change {
changes := []*Change{
newChange("1", "root", "root"),
newChange("2", "root", "1"),
newChange("1.2", "root", "1.1"),
newChange("1.3", "root", "1.2"),
newChange("1.1", "root", "1"),
newChange("3", "root", "2", "1.3"),
}
rand.Shuffle(len(changes), func(i, j int) {
changes[i], changes[j] = changes[j], changes[i]
})
return changes
}
var phash string
for i := 0; i < 100; i++ {
tr := new(Tree)
tr.Add(newSnapshot("root", ""))
tr.Add(getChanges()...)
assert.Len(t, tr.unAttached, 0)
assert.Len(t, tr.attached, 7)
hash := tr.Hash()
if phash != "" {
assert.Equal(t, phash, hash)
}
phash = hash
assert.Equal(t, []string{"3"}, tr.Heads())
}
}
func TestTree_CheckRootReduce(t *testing.T) {
t.Run("check root once", func(t *testing.T) {
tr := new(Tree)
tr.Add(
newSnapshot("0", ""),
newChange("1", "0", "0"),
newChange("1.1", "0", "1"),
newChange("1.2", "0", "1"),
newChange("1.4", "0", "1.2"),
newChange("1.3", "0", "1"),
newChange("1.3.1", "0", "1.3"),
newChange("1.2+3", "0", "1.4", "1.3.1"),
newChange("1.2+3.1", "0", "1.2+3"),
newSnapshot("10", "0", "1.2+3.1", "1.1"),
newChange("last", "10", "10"),
)
t.Run("check root", func(t *testing.T) {
total := tr.checkRoot(tr.attached["10"])
assert.Equal(t, 1, total)
})
t.Run("reduce", func(t *testing.T) {
tr.reduceTree()
assert.Equal(t, "10", tr.RootId())
var res []string
tr.Iterate(tr.RootId(), func(c *Change) (isContinue bool) {
res = append(res, c.Id)
return true
})
assert.Equal(t, []string{"10", "last"}, res)
})
})
t.Run("check root many", func(t *testing.T) {
tr := new(Tree)
tr.Add(
newSnapshot("0", ""),
newSnapshot("1", "0", "0"),
newChange("1.2", "0", "1"),
newChange("1.3", "0", "1"),
newChange("1.3.1", "0", "1.3"),
newSnapshot("1.2+3", "1", "1.2", "1.3.1"),
newChange("1.2+3.1", "1", "1.2+3"),
newChange("1.2+3.2", "1", "1.2+3"),
newSnapshot("10", "1.2+3", "1.2+3.1", "1.2+3.2"),
newChange("last", "10", "10"),
)
t.Run("check root", func(t *testing.T) {
total := tr.checkRoot(tr.attached["10"])
assert.Equal(t, 1, total)
total = tr.checkRoot(tr.attached["1.2+3"])
assert.Equal(t, 4, total)
total = tr.checkRoot(tr.attached["1"])
assert.Equal(t, 8, total)
})
t.Run("reduce", func(t *testing.T) {
tr.reduceTree()
assert.Equal(t, "10", tr.RootId())
var res []string
tr.Iterate(tr.RootId(), func(c *Change) (isContinue bool) {
res = append(res, c.Id)
return true
})
assert.Equal(t, []string{"10", "last"}, res)
})
})
t.Run("check root incorrect", func(t *testing.T) {
tr := new(Tree)
tr.Add(
newSnapshot("0", ""),
newChange("1", "0", "0"),
newChange("1.1", "0", "1"),
newChange("1.2", "0", "1"),
newChange("1.4", "0", "1.2"),
newChange("1.3", "0", "1"),
newSnapshot("1.3.1", "0", "1.3"),
newChange("1.2+3", "0", "1.4", "1.3.1"),
newChange("1.2+3.1", "0", "1.2+3"),
newChange("10", "0", "1.2+3.1", "1.1"),
newChange("last", "10", "10"),
)
t.Run("check root", func(t *testing.T) {
total := tr.checkRoot(tr.attached["1.3.1"])
assert.Equal(t, -1, total)
})
t.Run("reduce", func(t *testing.T) {
tr.reduceTree()
assert.Equal(t, "0", tr.RootId())
assert.Equal(t, 0, len(tr.possibleRoots))
})
})
}
func TestTree_Iterate(t *testing.T) {
t.Run("complex tree", func(t *testing.T) {
tr := new(Tree)
tr.Add(
newSnapshot("0", ""),
newChange("1", "0", "0"),
newChange("1.1", "0", "1"),
newChange("1.2", "0", "1"),
newChange("1.4", "0", "1.2"),
newChange("1.3", "0", "1"),
newChange("1.3.1", "0", "1.3"),
newChange("1.2+3", "0", "1.4", "1.3.1"),
newChange("1.2+3.1", "0", "1.2+3"),
newChange("10", "0", "1.2+3.1", "1.1"),
newChange("last", "0", "10"),
)
var res []string
tr.Iterate("0", func(c *Change) (isContinue bool) {
res = append(res, c.Id)
return true
})
res = res[:0]
tr.Iterate("0", func(c *Change) (isContinue bool) {
res = append(res, c.Id)
return true
})
assert.Equal(t, []string{"0", "1", "1.1", "1.2", "1.4", "1.3", "1.3.1", "1.2+3", "1.2+3.1", "10", "last"}, res)
})
}
func BenchmarkTree_Add(b *testing.B) {
getChanges := func() []*Change {
return []*Change{
newChange("1", "root", "root"),
newChange("2", "root", "1"),
newChange("1.2", "root", "1.1"),
newChange("1.3", "root", "1.2"),
newChange("1.1", "root", "1"),
newChange("3", "root", "2", "1.3"),
}
}
b.Run("by one", func(b *testing.B) {
tr := new(Tree)
tr.Add(newSnapshot("root", ""))
tr.Add(getChanges()...)
for i := 0; i < b.N; i++ {
tr.Add(newChange(fmt.Sprint(i+4), "root", fmt.Sprint(i+3)))
}
})
b.Run("add", func(b *testing.B) {
for i := 0; i < b.N; i++ {
tr := new(Tree)
tr.Add(newSnapshot("root", ""))
tr.Add(getChanges()...)
}
})
b.Run("add fast", func(b *testing.B) {
for i := 0; i < b.N; i++ {
tr := new(Tree)
tr.AddFast(newSnapshot("root", ""))
tr.AddFast(getChanges()...)
}
})
}
func BenchmarkTree_IterateLinear(b *testing.B) {
// prepare linear tree
tr := new(Tree)
tr.AddFast(newSnapshot("0", ""))
for j := 0; j < 10000; j++ {
tr.Add(newChange(fmt.Sprint(j+1), "0", fmt.Sprint(j)))
}
b.Run("add linear", func(b *testing.B) {
for i := 0; i < b.N; i++ {
tr.Iterate("0", func(c *Change) (isContinue bool) {
return true
})
}
})
}

184
pkg/acl/acltree/treebuilder.go → pkg/acl/tree/treebuilder.go
View File

@ -1,12 +1,14 @@
package acltree
package tree
import (
"context"
"errors"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app/logger"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice"
"go.uber.org/zap"
"time"
)
var (
@ -15,87 +17,83 @@ var (
)
type treeBuilder struct {
cache map[string]*Change
identityKeys map[string]signingkey.PubKey
signingPubKeyDecoder signingkey.PubKeyDecoder
tree *Tree
treeStorage treestorage.TreeStorage
treeStorage storage.TreeStorage
builder ChangeBuilder
*changeLoader
cache map[string]*Change
tree *Tree
// buffers
idStack []string
loadBuffer []*Change
}
func newTreeBuilder(t treestorage.TreeStorage, decoder signingkey.PubKeyDecoder) *treeBuilder {
func newTreeBuilder(storage storage.TreeStorage, builder ChangeBuilder) *treeBuilder {
return &treeBuilder{
signingPubKeyDecoder: decoder,
treeStorage: t,
changeLoader: newChangeLoader(
t,
decoder,
NewChange),
treeStorage: storage,
builder: builder,
}
}
func (tb *treeBuilder) Init() {
func (tb *treeBuilder) Reset() {
tb.cache = make(map[string]*Change)
tb.identityKeys = make(map[string]signingkey.PubKey)
tb.tree = &Tree{}
tb.changeLoader.Init(tb.cache, tb.identityKeys)
}
func (tb *treeBuilder) Build(fromStart bool) (*Tree, error) {
var headsAndOrphans []string
orphans, err := tb.treeStorage.Orphans()
if err != nil {
return nil, err
}
func (tb *treeBuilder) Build(newChanges []*Change) (*Tree, error) {
var headsAndNewChanges []string
heads, err := tb.treeStorage.Heads()
if err != nil {
return nil, err
}
headsAndOrphans = append(headsAndOrphans, orphans...)
headsAndOrphans = append(headsAndOrphans, heads...)
if fromStart {
if err := tb.buildTreeFromStart(headsAndOrphans); err != nil {
return nil, fmt.Errorf("buildTree error: %v", err)
}
} else {
breakpoint, err := tb.findBreakpoint(headsAndOrphans)
if err != nil {
return nil, fmt.Errorf("findBreakpoint error: %v", err)
}
if err = tb.buildTree(headsAndOrphans, breakpoint); err != nil {
return nil, fmt.Errorf("buildTree error: %v", err)
}
headsAndNewChanges = append(headsAndNewChanges, heads...)
tb.cache = make(map[string]*Change)
for _, ch := range newChanges {
headsAndNewChanges = append(headsAndNewChanges, ch.Id)
tb.cache[ch.Id] = ch
}
tb.cache = nil
log.With(zap.Strings("heads", heads)).Debug("building tree")
breakpoint, err := tb.findBreakpoint(headsAndNewChanges)
if err != nil {
return nil, fmt.Errorf("findBreakpoint error: %v", err)
}
if err = tb.buildTree(headsAndNewChanges, breakpoint); err != nil {
return nil, fmt.Errorf("buildTree error: %v", err)
}
return tb.tree, nil
}
func (tb *treeBuilder) buildTreeFromStart(heads []string) (err error) {
changes, root, err := tb.dfsFromStart(heads)
func (tb *treeBuilder) buildTree(heads []string, breakpoint string) (err error) {
ch, err := tb.loadChange(breakpoint)
if err != nil {
return err
return
}
tb.tree.AddFast(ch)
changes, err := tb.dfs(heads, breakpoint)
tb.tree.AddFast(root)
tb.tree.AddFast(changes...)
return
}
func (tb *treeBuilder) dfsFromStart(heads []string) (buf []*Change, root *Change, err error) {
var possibleRoots []*Change
stack := make([]string, len(heads), len(heads)*2)
copy(stack, heads)
func (tb *treeBuilder) dfs(heads []string, breakpoint string) (buf []*Change, err error) {
// initializing buffers
tb.idStack = tb.idStack[:0]
tb.loadBuffer = tb.loadBuffer[:0]
buf = make([]*Change, 0, len(stack)*2)
uniqMap := make(map[string]struct{})
for len(stack) > 0 {
id := stack[len(stack)-1]
stack = stack[:len(stack)-1]
// updating map
uniqMap := map[string]struct{}{breakpoint: {}}
// preparing dfs
tb.idStack = append(tb.idStack, heads...)
// dfs
for len(tb.idStack) > 0 {
id := tb.idStack[len(tb.idStack)-1]
tb.idStack = tb.idStack[:len(tb.idStack)-1]
if _, exists := uniqMap[id]; exists {
continue
}
@ -106,69 +104,38 @@ func (tb *treeBuilder) dfsFromStart(heads []string) (buf []*Change, root *Change
}
uniqMap[id] = struct{}{}
buf = append(buf, ch)
tb.loadBuffer = append(tb.loadBuffer, ch)
for _, prev := range ch.PreviousIds {
stack = append(stack, prev)
}
if len(ch.PreviousIds) == 0 {
possibleRoots = append(possibleRoots, ch)
if _, exists := uniqMap[prev]; exists {
continue
}
tb.idStack = append(tb.idStack, prev)
}
}
header, err := tb.treeStorage.Header()
if err != nil {
return nil, nil, err
}
for _, r := range possibleRoots {
if r.Id == header.FirstChangeId {
return buf, r, nil
}
}
return nil, nil, fmt.Errorf("could not find root change")
return tb.loadBuffer, nil
}
func (tb *treeBuilder) buildTree(heads []string, breakpoint string) (err error) {
ch, err := tb.loadChange(breakpoint)
func (tb *treeBuilder) loadChange(id string) (ch *Change, err error) {
if ch, ok := tb.cache[id]; ok {
return ch, nil
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
change, err := tb.treeStorage.GetRawChange(ctx, id)
if err != nil {
return
return nil, err
}
tb.tree.AddFast(ch)
changes, err := tb.dfs(heads, breakpoint, tb.loadChange)
tb.tree.AddFast(changes...)
return
}
func (tb *treeBuilder) dfs(
heads []string,
breakpoint string,
load func(string) (*Change, error)) (buf []*Change, err error) {
stack := make([]string, len(heads), len(heads)*2)
copy(stack, heads)
buf = make([]*Change, 0, len(stack)*2)
uniqMap := map[string]struct{}{breakpoint: {}}
for len(stack) > 0 {
id := stack[len(stack)-1]
stack = stack[:len(stack)-1]
if _, exists := uniqMap[id]; exists {
continue
}
ch, err := load(id)
if err != nil {
continue
}
uniqMap[id] = struct{}{}
buf = append(buf, ch)
for _, prev := range ch.PreviousIds {
stack = append(stack, prev)
}
ch, err = tb.builder.ConvertFromRawAndVerify(change)
if err != nil {
return nil, err
}
return buf, nil
tb.cache[id] = ch
return ch, nil
}
func (tb *treeBuilder) findBreakpoint(heads []string) (breakpoint string, err error) {
@ -281,11 +248,9 @@ func (tb *treeBuilder) findCommonForTwoSnapshots(s1, s2 string) (s string, err e
}
isEmptySnapshot := func(ch *Change) bool {
// TODO: add more sophisticated checks in Change for snapshots
return !ch.IsSnapshot
}
// TODO: can we even have empty snapshots?
// prefer not empty snapshot
if isEmptySnapshot(ch1) && !isEmptySnapshot(ch2) {
log.Warnf("changes build Tree: prefer %s(not empty) over %s(empty)", s2, s1)
@ -295,7 +260,6 @@ func (tb *treeBuilder) findCommonForTwoSnapshots(s1, s2 string) (s string, err e
return s1, nil
}
// TODO: add virtual change mechanics
// unexpected behavior - just return lesser id
if s1 < s2 {
log.Warnf("changes build Tree: prefer %s (%s<%s)", s1, s1, s2)

4
pkg/acl/acltree/treegraph.go → pkg/acl/tree/treegraph.go
View File

@ -2,10 +2,10 @@
// +build !linux,!darwin android ios nographviz
// +build !amd64
package acltree
package tree
import "fmt"
func (t *Tree) Graph() (data string, err error) {
func (t *Tree) Graph(parser DescriptionParser) (data []string, err error) {
return "", fmt.Errorf("not supported")
}

47
pkg/acl/acltree/treegraph_nix.go → pkg/acl/tree/treegraph_nix.go
View File

@ -5,20 +5,18 @@
// +build !nographviz
// +build amd64 arm64
package acltree
package tree
import (
"bytes"
"fmt"
"strings"
"time"
"unicode"
"github.com/goccy/go-graphviz"
"github.com/goccy/go-graphviz/cgraph"
"strings"
"time"
)
func (t *Tree) Graph() (data string, err error) {
func (t *Tree) Graph(parser DescriptionParser) (data string, err error) {
var order = make(map[string]string)
var seq = 0
t.Iterate(t.RootId(), func(c *Change) (isContinue bool) {
@ -46,44 +44,15 @@ func (t *Tree) Graph() (data string, err error) {
if e != nil {
return e
}
if c.Content.GetAclData() != nil {
n.SetStyle(cgraph.FilledNodeStyle)
} else if c.IsSnapshot {
n.SetStyle(cgraph.DashedNodeStyle)
}
n.SetStyle(cgraph.FilledNodeStyle)
nodes[c.Id] = n
ord := order[c.Id]
if ord == "" {
ord = "miss"
}
var chSymbs []string
if c.Content.AclData != nil {
for _, chc := range c.Content.AclData.AclContent {
tp := fmt.Sprintf("%T", chc.Value)
tp = strings.Replace(tp, "ACLChangeACLContentValueValueOf", "", 1)
res := ""
for _, ts := range tp {
if unicode.IsUpper(ts) {
res += string(ts)
}
}
chSymbs = append(chSymbs, res)
}
}
if c.DecryptedDocumentChange != nil {
// TODO: add some parser to provide custom unmarshalling for the document change
//for _, chc := range c.DecryptedDocumentChange.Content {
// tp := fmt.Sprintf("%T", chc.Value)
// tp = strings.Replace(tp, "ChangeContentValueOf", "", 1)
// res := ""
// for _, ts := range tp {
// if unicode.IsUpper(ts) {
// res += string(ts)
// }
// }
// chSymbs = append(chSymbs, res)
//}
chSymbs = append(chSymbs, "DEC")
chSymbs, err := parser.ParseChange(c)
if err != nil {
return err
}
shortId := c.Id

93
pkg/acl/tree/treeiterator.go Normal file
View File

@ -0,0 +1,93 @@
package tree
import (
"sync"
)
var itPool = &sync.Pool{
New: func() interface{} {
return &iterator{
stack: make([]*Change, 0, 100),
resBuf: make([]*Change, 0, 100),
}
},
}
func newIterator() *iterator {
return itPool.Get().(*iterator)
}
func freeIterator(i *iterator) {
itPool.Put(i)
}
type iterator struct {
resBuf []*Change
stack []*Change
f func(c *Change) bool
}
func (i *iterator) iterateSkip(start *Change, skipBefore *Change, f func(c *Change) (isContinue bool)) {
skipping := true
i.iterate(start, func(c *Change) (isContinue bool) {
if skipping && c != skipBefore {
return true
}
skipping = false
return f(c)
})
}
func (i *iterator) topSort(start *Change) {
stack := i.stack
stack = append(stack, start)
for len(stack) > 0 {
ch := stack[len(stack)-1]
stack = stack[:len(stack)-1]
// this looks a bit clumsy, but the idea is that we will go through the change again as soon as we finished
// going through its branches
if ch.branchesFinished {
i.resBuf = append(i.resBuf, ch)
ch.branchesFinished = false
continue
}
// in theory, it may be the case that we add the change two times
// but probably due to the way how we build the tree, we won't need it
if ch.visited {
continue
}
stack = append(stack, ch)
ch.visited = true
ch.branchesFinished = true
for j := 0; j < len(ch.Next); j++ {
if !ch.Next[j].visited {
stack = append(stack, ch.Next[j])
}
}
}
for _, ch := range i.resBuf {
ch.visited = false
}
}
func (i *iterator) iterate(start *Change, f func(c *Change) (isContinue bool)) {
if start == nil {
return
}
// reset
i.resBuf = i.resBuf[:0]
i.stack = i.stack[:0]
i.f = f
i.topSort(start)
for idx := len(i.resBuf) - 1; idx >= 0; idx-- {
if !f(i.resBuf[idx]) {
return
}
}
}

92
pkg/acl/tree/treereduce.go Normal file
View File

@ -0,0 +1,92 @@
package tree
import "math"
// clearPossibleRoots force removes any snapshots which can further be deemed as roots
func (t *Tree) clearPossibleRoots() {
t.possibleRoots = t.possibleRoots[:0]
}
// checkRoot checks if a change can be a new root for the tree
// it returns total changes which were discovered during dfsPrev from heads
func (t *Tree) checkRoot(change *Change) (total int) {
t.stackBuf = t.stackBuf[:0]
stack := t.stackBuf
// starting with heads
for _, h := range t.headIds {
stack = append(stack, t.attached[h])
}
t.dfsPrev(
stack,
[]string{change.Id},
func(ch *Change) bool {
total += 1
return true
},
func(changes []*Change) {
if t.root.visited {
total = -1
}
},
)
return
}
// makeRootAndRemove removes all changes before start and makes start the root
func (t *Tree) makeRootAndRemove(start *Change) {
t.stackBuf = t.stackBuf[:0]
stack := t.stackBuf
for _, prev := range start.PreviousIds {
stack = append(stack, t.attached[prev])
}
t.dfsPrev(
stack,
[]string{},
func(ch *Change) bool {
return true
},
func(changes []*Change) {
for _, ch := range changes {
delete(t.unAttached, ch.Id)
}
},
)
// removing unattached because they may refer to previous root
t.unAttached = make(map[string]*Change)
t.root = start
}
// reduceTree tries to reduce the tree to one of possible tree roots
func (t *Tree) reduceTree() (res bool) {
if len(t.possibleRoots) == 0 {
return
}
var (
minRoot *Change
minTotal = math.MaxInt
)
// checking if we can reduce tree to other root
for _, root := range t.possibleRoots {
totalChanges := t.checkRoot(root)
// we prefer new root with min amount of total changes
if totalChanges != -1 && totalChanges < minTotal {
minRoot = root
minTotal = totalChanges
}
}
t.clearPossibleRoots()
if minRoot == nil {
return
}
t.makeRootAndRemove(minRoot)
res = true
return
}

91
pkg/acl/tree/treestorage.go Normal file
View File

@ -0,0 +1,91 @@
package tree
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/list"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/cid"
"github.com/gogo/protobuf/proto"
"time"
)
func CreateNewTreeStorage(
acc *account.AccountData,
aclList list.ACLList,
content proto.Marshaler,
create storage.TreeStorageCreatorFunc) (thr storage.TreeStorage, err error) {
state := aclList.ACLState()
change := &aclpb.Change{
AclHeadId: aclList.Head().Id,
CurrentReadKeyHash: state.CurrentReadKeyHash(),
Timestamp: int64(time.Now().Nanosecond()),
Identity: acc.Identity,
IsSnapshot: true,
}
marshalledData, err := content.Marshal()
if err != nil {
return
}
readKey, err := state.CurrentReadKey()
if err != nil {
return
}
encrypted, err := readKey.Encrypt(marshalledData)
if err != nil {
return
}
change.ChangesData = encrypted
fullMarshalledChange, err := proto.Marshal(change)
if err != nil {
return
}
signature, err := acc.SignKey.Sign(fullMarshalledChange)
if err != nil {
return
}
changeId, err := cid.NewCIDFromBytes(fullMarshalledChange)
if err != nil {
return
}
rawChange := &aclpb.RawChange{
Payload: fullMarshalledChange,
Signature: signature,
Id: changeId,
}
header, treeId, err := createTreeHeaderAndId(rawChange, aclpb.Header_DocTree, aclList.ID())
if err != nil {
return
}
return create(storage.TreeStorageCreatePayload{
TreeId: treeId,
Header: header,
Changes: []*aclpb.RawChange{rawChange},
Heads: []string{rawChange.Id},
})
}
func createTreeHeaderAndId(change *aclpb.RawChange, treeType aclpb.HeaderDocType, aclListId string) (header *aclpb.Header, treeId string, err error) {
header = &aclpb.Header{
FirstId: change.Id,
DocType: treeType,
AclListId: aclListId,
}
marshalledHeader, err := proto.Marshal(header)
if err != nil {
return
}
treeId, err = cid.NewCIDFromBytes(marshalledHeader)
return
}

29
pkg/acl/tree/util.go Normal file
View File

@ -0,0 +1,29 @@
package tree
func commonSnapshotForTwoPaths(ourPath []string, theirPath []string) (string, error) {
var i int
var j int
OuterLoop:
// find starting point from the right
for i = len(ourPath) - 1; i >= 0; i-- {
for j = len(theirPath) - 1; j >= 0; j-- {
// most likely there would be only one comparison, because mostly the snapshot path will start from the root for nodes
if ourPath[i] == theirPath[j] {
break OuterLoop
}
}
}
if i < 0 || j < 0 {
return "", ErrNoCommonSnapshot
}
// find last common element of the sequence moving from right to left
for i >= 0 && j >= 0 {
if ourPath[i] == theirPath[j] {
i--
j--
} else {
break
}
}
return ourPath[i+1], nil
}

163
pkg/acl/treestorage/inmemory.go
View File

@ -1,163 +0,0 @@
package treestorage
import (
"context"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice"
"github.com/gogo/protobuf/proto"
"sync"
)
type inMemoryTreeStorage struct {
id string
header *treepb.TreeHeader
heads []string
orphans []string
changes map[string]*aclpb.RawChange
sync.RWMutex
}
type CreatorFunc = func(string, *treepb.TreeHeader, []*aclpb.RawChange) (TreeStorage, error)
func NewInMemoryTreeStorage(
treeId string,
header *treepb.TreeHeader,
changes []*aclpb.RawChange) (TreeStorage, error) {
allChanges := make(map[string]*aclpb.RawChange)
var orphans []string
for _, ch := range changes {
allChanges[ch.Id] = ch
orphans = append(orphans, ch.Id)
}
return &inMemoryTreeStorage{
id: treeId,
header: header,
heads: nil,
orphans: orphans,
changes: allChanges,
RWMutex: sync.RWMutex{},
}, nil
}
func (t *inMemoryTreeStorage) TreeID() (string, error) {
t.RLock()
defer t.RUnlock()
return t.id, nil
}
func (t *inMemoryTreeStorage) Header() (*treepb.TreeHeader, error) {
t.RLock()
defer t.RUnlock()
return t.header, nil
}
func (t *inMemoryTreeStorage) Heads() ([]string, error) {
t.RLock()
defer t.RUnlock()
return t.heads, nil
}
func (t *inMemoryTreeStorage) Orphans() ([]string, error) {
t.RLock()
defer t.RUnlock()
return t.orphans, nil
}
func (t *inMemoryTreeStorage) SetHeads(heads []string) error {
t.Lock()
defer t.Unlock()
t.heads = t.heads[:0]
for _, h := range heads {
t.heads = append(t.heads, h)
}
return nil
}
func (t *inMemoryTreeStorage) RemoveOrphans(orphans ...string) error {
t.Lock()
defer t.Unlock()
t.orphans = slice.Difference(t.orphans, orphans)
return nil
}
func (t *inMemoryTreeStorage) AddOrphans(orphans ...string) error {
t.Lock()
defer t.Unlock()
t.orphans = append(t.orphans, orphans...)
return nil
}
func (t *inMemoryTreeStorage) AddRawChange(change *aclpb.RawChange) error {
t.Lock()
defer t.Unlock()
// TODO: better to do deep copy
t.changes[change.Id] = change
return nil
}
func (t *inMemoryTreeStorage) AddChange(change aclchanges.Change) error {
t.Lock()
defer t.Unlock()
signature := change.Signature()
id := change.CID()
aclChange := change.ProtoChange()
fullMarshalledChange, err := proto.Marshal(aclChange)
if err != nil {
return err
}
rawChange := &aclpb.RawChange{
Payload: fullMarshalledChange,
Signature: signature,
Id: id,
}
t.changes[id] = rawChange
return nil
}
func (t *inMemoryTreeStorage) GetChange(ctx context.Context, changeId string) (*aclpb.RawChange, error) {
t.RLock()
defer t.RUnlock()
if res, exists := t.changes[changeId]; exists {
return res, nil
}
return nil, fmt.Errorf("could not get change with id: %s", changeId)
}
type inMemoryTreeStorageProvider struct {
trees map[string]TreeStorage
sync.RWMutex
}
func (i *inMemoryTreeStorageProvider) TreeStorage(treeId string) (TreeStorage, error) {
i.RLock()
defer i.RUnlock()
if tree, exists := i.trees[treeId]; exists {
return tree, nil
}
return nil, ErrUnknownTreeId
}
func (i *inMemoryTreeStorageProvider) CreateTreeStorage(treeId string, header *treepb.TreeHeader, changes []*aclpb.RawChange) (TreeStorage, error) {
i.Lock()
defer i.Unlock()
res, err := NewInMemoryTreeStorage(treeId, header, changes)
if err != nil {
return nil, err
}
i.trees[treeId] = res
return res, nil
}
func NewInMemoryTreeStorageProvider() Provider {
return &inMemoryTreeStorageProvider{
trees: make(map[string]TreeStorage),
}
}

14
pkg/acl/treestorage/provider.go
View File

@ -1,14 +0,0 @@
package treestorage
import (
"errors"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb"
)
var ErrUnknownTreeId = errors.New("tree does not exist")
type Provider interface {
TreeStorage(treeId string) (TreeStorage, error)
CreateTreeStorage(treeId string, header *treepb.TreeHeader, changes []*aclpb.RawChange) (TreeStorage, error)
}

25
pkg/acl/treestorage/storage.go
View File

@ -1,25 +0,0 @@
package treestorage
import (
"context"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb"
)
type TreeStorage interface {
TreeID() (string, error)
Header() (*treepb.TreeHeader, error)
Heads() ([]string, error)
Orphans() ([]string, error)
SetHeads(heads []string) error
RemoveOrphans(orphan ...string) error
AddOrphans(orphan ...string) error
AddRawChange(change *aclpb.RawChange) error
AddChange(change aclchanges.Change) error
// TODO: have methods with raw changes also
GetChange(ctx context.Context, recordID string) (*aclpb.RawChange, error)
}

9
pkg/acl/treestorage/treepb/protos/tree.proto
View File

@ -1,9 +0,0 @@
syntax = "proto3";
package tree;
option go_package = "treepb";
message TreeHeader {
string firstChangeId = 1;
bool isWorkspace = 2;
// TODO: add user identity, signature and nano timestamp
}

358
pkg/acl/treestorage/treepb/tree.pb.go
View File

@ -1,358 +0,0 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: pkg/acl/treestorage/treepb/protos/tree.proto
package treepb
import (
fmt "fmt"
proto "github.com/gogo/protobuf/proto"
io "io"
math "math"
math_bits "math/bits"
)
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
type TreeHeader struct {
FirstChangeId string `protobuf:"bytes,1,opt,name=firstChangeId,proto3" json:"firstChangeId,omitempty"`
IsWorkspace bool `protobuf:"varint,2,opt,name=isWorkspace,proto3" json:"isWorkspace,omitempty"`
}
func (m *TreeHeader) Reset() { *m = TreeHeader{} }
func (m *TreeHeader) String() string { return proto.CompactTextString(m) }
func (*TreeHeader) ProtoMessage() {}
func (*TreeHeader) Descriptor() ([]byte, []int) {
return fileDescriptor_e7d760b855878644, []int{0}
}
func (m *TreeHeader) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *TreeHeader) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_TreeHeader.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *TreeHeader) XXX_Merge(src proto.Message) {
xxx_messageInfo_TreeHeader.Merge(m, src)
}
func (m *TreeHeader) XXX_Size() int {
return m.Size()
}
func (m *TreeHeader) XXX_DiscardUnknown() {
xxx_messageInfo_TreeHeader.DiscardUnknown(m)
}
var xxx_messageInfo_TreeHeader proto.InternalMessageInfo
func (m *TreeHeader) GetFirstChangeId() string {
if m != nil {
return m.FirstChangeId
}
return ""
}
func (m *TreeHeader) GetIsWorkspace() bool {
if m != nil {
return m.IsWorkspace
}
return false
}
func init() {
proto.RegisterType((*TreeHeader)(nil), "tree.TreeHeader")
}
func init() {
proto.RegisterFile("pkg/acl/treestorage/treepb/protos/tree.proto", fileDescriptor_e7d760b855878644)
}
var fileDescriptor_e7d760b855878644 = []byte{
// 165 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0x29, 0xc8, 0x4e, 0xd7,
0x4f, 0x4c, 0xce, 0xd1, 0x2f, 0x29, 0x4a, 0x4d, 0x2d, 0x2e, 0xc9, 0x2f, 0x4a, 0x4c, 0x4f, 0x05,
0xb3, 0x0b, 0x92, 0xf4, 0x0b, 0x8a, 0xf2, 0x4b, 0xf2, 0x8b, 0xc1, 0x3c, 0x3d, 0x30, 0x5b, 0x88,
0x05, 0xc4, 0x56, 0x0a, 0xe1, 0xe2, 0x0a, 0x29, 0x4a, 0x4d, 0xf5, 0x48, 0x4d, 0x4c, 0x49, 0x2d,
0x12, 0x52, 0xe1, 0xe2, 0x4d, 0xcb, 0x2c, 0x2a, 0x2e, 0x71, 0xce, 0x48, 0xcc, 0x4b, 0x4f, 0xf5,
0x4c, 0x91, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x42, 0x15, 0x14, 0x52, 0xe0, 0xe2, 0xce, 0x2c,
0x0e, 0xcf, 0x2f, 0xca, 0x2e, 0x2e, 0x48, 0x4c, 0x4e, 0x95, 0x60, 0x52, 0x60, 0xd4, 0xe0, 0x08,
0x42, 0x16, 0x72, 0x52, 0x38, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4,
0x18, 0x27, 0x3c, 0x96, 0x63, 0xb8, 0xf0, 0x58, 0x8e, 0xe1, 0xc6, 0x63, 0x39, 0x86, 0x28, 0x36,
0x88, 0x7b, 0x92, 0xd8, 0xc0, 0x8e, 0x30, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0x09, 0x4f, 0xc6,
0xec, 0xb4, 0x00, 0x00, 0x00,
}
func (m *TreeHeader) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *TreeHeader) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *TreeHeader) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.IsWorkspace {
i--
if m.IsWorkspace {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i--
dAtA[i] = 0x10
}
if len(m.FirstChangeId) > 0 {
i -= len(m.FirstChangeId)
copy(dAtA[i:], m.FirstChangeId)
i = encodeVarintTree(dAtA, i, uint64(len(m.FirstChangeId)))
i--
dAtA[i] = 0xa
}
return len(dAtA) - i, nil
}
func encodeVarintTree(dAtA []byte, offset int, v uint64) int {
offset -= sovTree(v)
base := offset
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
dAtA[offset] = uint8(v)
return base
}
func (m *TreeHeader) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.FirstChangeId)
if l > 0 {
n += 1 + l + sovTree(uint64(l))
}
if m.IsWorkspace {
n += 2
}
return n
}
func sovTree(x uint64) (n int) {
return (math_bits.Len64(x|1) + 6) / 7
}
func sozTree(x uint64) (n int) {
return sovTree(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *TreeHeader) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTree
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: TreeHeader: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: TreeHeader: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field FirstChangeId", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTree
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthTree
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthTree
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.FirstChangeId = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IsWorkspace", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTree
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
m.IsWorkspace = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipTree(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthTree
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipTree(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
depth := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowTree
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowTree
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if dAtA[iNdEx-1] < 0x80 {
break
}
}
case 1:
iNdEx += 8
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowTree
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if length < 0 {
return 0, ErrInvalidLengthTree
}
iNdEx += length
case 3:
depth++
case 4:
if depth == 0 {
return 0, ErrUnexpectedEndOfGroupTree
}
depth--
case 5:
iNdEx += 4
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
if iNdEx < 0 {
return 0, ErrInvalidLengthTree
}
if depth == 0 {
return iNdEx, nil
}
}
return 0, io.ErrUnexpectedEOF
}
var (
ErrInvalidLengthTree = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowTree = fmt.Errorf("proto: integer overflow")
ErrUnexpectedEndOfGroupTree = fmt.Errorf("proto: unexpected end of group")
)

2
service/account/service.go
View File

@ -61,7 +61,7 @@ func (s *service) Init(ctx context.Context, a *app.App) (err error) {
Identity: identity,
SignKey: signKey,
EncKey: decodedEncryptionKey.(encryptionkey.PrivKey),
Decoder: signingkey.NewEd25519PubKeyDecoder(),
Decoder: signingkey.NewEDPubKeyDecoder(),
}
s.peerId = acc.PeerId

22
service/api/service.go
View File

@ -6,8 +6,8 @@ import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/app"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app/logger"
"github.com/anytypeio/go-anytype-infrastructure-experiments/config"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/acltree"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/sync/document"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/tree"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/document"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/treecache"
"go.uber.org/zap"
"io"
@ -53,7 +53,7 @@ func (s *service) Run(ctx context.Context) (err error) {
}
mux := http.NewServeMux()
mux.HandleFunc("/treeDump", s.treeDump)
mux.HandleFunc("/createDocument", s.createDocument)
mux.HandleFunc("/createDocumentTree", s.createDocumentTree)
mux.HandleFunc("/appendDocument", s.appendDocument)
s.srv.Handler = mux
@ -79,8 +79,9 @@ func (s *service) treeDump(w http.ResponseWriter, req *http.Request) {
dump string
err error
)
err = s.treeCache.Do(context.Background(), treeId, func(tree acltree.ACLTree) error {
dump, err = tree.DebugDump()
err = s.treeCache.Do(context.Background(), treeId, func(obj interface{}) error {
t := obj.(tree.ObjectTree)
dump, err = t.DebugDump()
if err != nil {
return err
}
@ -93,13 +94,14 @@ func (s *service) treeDump(w http.ResponseWriter, req *http.Request) {
sendText(w, http.StatusOK, dump)
}
func (s *service) createDocument(w http.ResponseWriter, req *http.Request) {
func (s *service) createDocumentTree(w http.ResponseWriter, req *http.Request) {
var (
query = req.URL.Query()
text = query.Get("text")
query = req.URL.Query()
text = query.Get("text")
aclListId = query.Get("aclListId")
)
timeoutCtx, cancel := context.WithTimeout(context.Background(), time.Second*30)
treeId, err := s.documentService.CreateDocument(timeoutCtx, fmt.Sprintf("created document with id: %s", text))
treeId, err := s.documentService.CreateDocumentTree(timeoutCtx, aclListId, text)
cancel()
if err != nil {
sendText(w, http.StatusInternalServerError, err.Error())
@ -115,7 +117,7 @@ func (s *service) appendDocument(w http.ResponseWriter, req *http.Request) {
treeId = query.Get("treeId")
)
timeoutCtx, cancel := context.WithTimeout(context.Background(), time.Second*30)
err := s.documentService.UpdateDocument(timeoutCtx, treeId, text)
err := s.documentService.UpdateDocumentTree(timeoutCtx, treeId, text)
cancel()
if err != nil {
sendText(w, http.StatusInternalServerError, err.Error())

147
service/sync/document/service.go → service/document/service.go
View File

@ -2,14 +2,16 @@ package document
import (
"context"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app/logger"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/acltree"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/testchanges/testchangepb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/list"
testchanges "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/testchanges/proto"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/tree"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/node"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/storage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/sync/message"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/treecache"
"github.com/anytypeio/go-anytype-infrastructure-experiments/syncproto"
@ -25,13 +27,14 @@ type service struct {
messageService message.Service
treeCache treecache.Service
account account.Service
storage storage.Service
// to create new documents we need to know all nodes
nodes []*node.Node
}
type Service interface {
UpdateDocument(ctx context.Context, id, text string) error
CreateDocument(ctx context.Context, text string) (string, error)
UpdateDocumentTree(ctx context.Context, id, text string) error
CreateDocumentTree(ctx context.Context, aclTreeId string, text string) (id string, err error)
}
func New() app.Component {
@ -42,6 +45,7 @@ func (s *service) Init(ctx context.Context, a *app.App) (err error) {
s.account = a.MustComponent(account.CName).(account.Service)
s.messageService = a.MustComponent(message.CName).(message.Service)
s.treeCache = a.MustComponent(treecache.CName).(treecache.Service)
s.storage = a.MustComponent(storage.CName).(storage.Service)
nodesService := a.MustComponent(node.CName).(node.Service)
s.nodes = nodesService.Nodes()
@ -54,41 +58,65 @@ func (s *service) Name() (name string) {
}
func (s *service) Run(ctx context.Context) (err error) {
return nil
syncData := s.storage.ImportedACLSyncData()
// we could have added a timeout or some additional logic,
// but let's just use the ACL id of the latest started node :-)
return s.messageService.SendToSpaceAsync("", syncproto.WrapACLList(
&syncproto.SyncACLList{Records: syncData.Records},
syncData.Header,
syncData.Id,
))
}
func (s *service) Close(ctx context.Context) (err error) {
return nil
}
func (s *service) UpdateDocument(ctx context.Context, id, text string) (err error) {
func (s *service) UpdateDocumentTree(ctx context.Context, id, text string) (err error) {
var (
ch *aclpb.RawChange
header *treepb.TreeHeader
header *aclpb.Header
snapshotPath []string
heads []string
)
log.With(zap.String("id", id), zap.String("text", text)).
Debug("updating document")
err = s.treeCache.Do(ctx, id, func(tree acltree.ACLTree) error {
ch, err = tree.AddContent(ctx, func(builder acltree.ChangeBuilder) error {
builder.AddChangeContent(
&testchangepb.PlainTextChangeData{
Content: []*testchangepb.PlainTextChangeContent{
createAppendTextChangeContent(text),
},
})
err = s.treeCache.Do(ctx, id, func(obj interface{}) error {
docTree, ok := obj.(tree.ObjectTree)
if !ok {
return fmt.Errorf("can't update acl trees with text")
}
docTree.Lock()
defer docTree.Unlock()
err = s.treeCache.Do(ctx, docTree.Header().AclListId, func(obj interface{}) error {
aclTree := obj.(list.ACLList)
aclTree.RLock()
defer aclTree.RUnlock()
content := createAppendTextChange(text)
signable := tree.SignableChangeContent{
Proto: content,
Key: s.account.Account().SignKey,
Identity: s.account.Account().Identity,
IsSnapshot: false,
}
ch, err = docTree.AddContent(ctx, signable)
if err != nil {
return err
}
return nil
})
if err != nil {
return err
}
id = tree.ID()
heads = tree.Heads()
header = tree.Header()
snapshotPath = tree.SnapshotPath()
id = docTree.ID()
heads = docTree.Heads()
header = docTree.Header()
snapshotPath = docTree.SnapshotPath()
return nil
})
if err != nil {
@ -103,84 +131,87 @@ func (s *service) UpdateDocument(ctx context.Context, id, text string) (err erro
return s.messageService.SendToSpaceAsync("", syncproto.WrapHeadUpdate(&syncproto.SyncHeadUpdate{
Heads: heads,
Changes: []*aclpb.RawChange{ch},
TreeId: id,
SnapshotPath: snapshotPath,
TreeHeader: header,
}))
}, header, id))
}
func (s *service) CreateDocument(ctx context.Context, text string) (id string, err error) {
func (s *service) CreateDocumentTree(ctx context.Context, aclListId string, text string) (id string, err error) {
acc := s.account.Account()
var (
ch *aclpb.RawChange
header *treepb.TreeHeader
header *aclpb.Header
snapshotPath []string
heads []string
)
err = s.treeCache.Do(ctx, aclListId, func(obj interface{}) error {
t := obj.(list.ACLList)
t.RLock()
defer t.RUnlock()
err = s.treeCache.Create(ctx, func(builder acltree.ChangeBuilder) error {
err := builder.UserAdd(acc.Identity, acc.EncKey.GetPublic(), aclpb.ACLChange_Admin)
content := createInitialTextChange(text)
doc, err := tree.CreateNewTreeStorage(acc, t, content, s.storage.CreateTreeStorage)
if err != nil {
return err
}
// adding all predefined nodes to the document as admins
for _, n := range s.nodes {
err = builder.UserAdd(n.SigningKeyString, n.EncryptionKey, aclpb.ACLChange_Admin)
if err != nil {
return err
}
}
builder.AddChangeContent(createInitialChangeContent(text))
return nil
}, func(tree acltree.ACLTree) error {
id = tree.ID()
heads = tree.Heads()
header = tree.Header()
snapshotPath = tree.SnapshotPath()
ch, err = tree.Storage().GetChange(ctx, heads[0])
id, err = doc.ID()
if err != nil {
return err
}
log.With(
zap.String("id", id),
zap.Strings("heads", heads),
zap.String("header", header.String())).
Debug("document created in the database")
header, err = doc.Header()
if err != nil {
return err
}
heads = []string{header.FirstId}
snapshotPath = []string{header.FirstId}
ch, err = doc.GetRawChange(ctx, header.FirstId)
if err != nil {
return err
}
return nil
})
if err != nil {
return "", err
}
log.With(zap.String("id", id), zap.String("text", text)).
Debug("creating document")
err = s.messageService.SendToSpaceAsync("", syncproto.WrapHeadUpdate(&syncproto.SyncHeadUpdate{
Heads: heads,
Changes: []*aclpb.RawChange{ch},
TreeId: id,
SnapshotPath: snapshotPath,
TreeHeader: header,
}))
}, header, id))
if err != nil {
return "", err
}
return id, err
}
func createInitialChangeContent(text string) proto.Marshaler {
return &testchangepb.PlainTextChangeData{
Content: []*testchangepb.PlainTextChangeContent{
func createInitialTextChange(text string) proto.Marshaler {
return &testchanges.PlainTextChangeData{
Content: []*testchanges.PlainTextChangeContent{
createAppendTextChangeContent(text),
},
Snapshot: &testchangepb.PlainTextChangeSnapshot{Text: text},
Snapshot: &testchanges.PlainTextChangeSnapshot{Text: text},
}
}
func createAppendTextChangeContent(text string) *testchangepb.PlainTextChangeContent {
return &testchangepb.PlainTextChangeContent{
Value: &testchangepb.PlainTextChangeContentValueOfTextAppend{
TextAppend: &testchangepb.PlainTextChangeTextAppend{
func createAppendTextChange(text string) proto.Marshaler {
return &testchanges.PlainTextChangeData{
Content: []*testchanges.PlainTextChangeContent{
createAppendTextChangeContent(text),
},
}
}
func createAppendTextChangeContent(text string) *testchanges.PlainTextChangeContent {
return &testchanges.PlainTextChangeContent{
Value: &testchanges.PlainTextChangeContentValueOfTextAppend{
TextAppend: &testchanges.PlainTextChangeTextAppend{
Text: text,
},
},

3
service/space/service.go
View File

@ -78,7 +78,8 @@ func (s *service) Handle(ctx context.Context, data []byte) (resp proto.Marshaler
return
}
if spaceReq.SpaceId != "" {
sp, err := s.get(ctx, spaceReq.SpaceId)
var sp Space
sp, err = s.get(ctx, spaceReq.SpaceId)
if err != nil {
return
}

2
service/space/space.go
View File

@ -19,8 +19,6 @@ type Space interface {
Close() error
}
//
type space struct {
id string
conf configuration.Configuration

136
service/storage/service.go Normal file
View File

@ -0,0 +1,136 @@
package storage
import (
"context"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app/logger"
"github.com/anytypeio/go-anytype-infrastructure-experiments/etc"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/list"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/testutils/acllistbuilder"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/node"
)
var CName = "storage"
var log = logger.NewNamed("storage").Sugar()
type ImportedACLSyncData struct {
Id string
Header *aclpb.Header
Records []*aclpb.RawRecord
}
type Service interface {
storage.Provider
ImportedACLSyncData() ImportedACLSyncData
}
func New() app.Component {
return &service{}
}
type service struct {
storageProvider storage.Provider
importedACLSyncData ImportedACLSyncData
}
func (s *service) Init(ctx context.Context, a *app.App) (err error) {
s.storageProvider = storage.NewInMemoryTreeStorageProvider()
// importing hardcoded acl list, check that the keys there are correct
return s.importACLList(a)
}
func (s *service) Storage(treeId string) (storage.Storage, error) {
return s.storageProvider.Storage(treeId)
}
func (s *service) AddStorage(id string, st storage.Storage) error {
return s.storageProvider.AddStorage(id, st)
}
func (s *service) CreateTreeStorage(payload storage.TreeStorageCreatePayload) (storage.TreeStorage, error) {
return s.storageProvider.CreateTreeStorage(payload)
}
func (s *service) CreateACLListStorage(payload storage.ACLListStorageCreatePayload) (storage.ListStorage, error) {
return s.storageProvider.CreateACLListStorage(payload)
}
func (s *service) Name() (name string) {
return CName
}
func (s *service) ImportedACLSyncData() ImportedACLSyncData {
return s.importedACLSyncData
}
func (s *service) Run(ctx context.Context) (err error) {
return nil
}
func (s service) Close(ctx context.Context) (err error) {
return nil
}
func (s *service) importACLList(a *app.App) (err error) {
path := fmt.Sprintf("%s/%s", etc.Path(), "acl.yml")
st, err := acllistbuilder.NewACLListStorageBuilderFromFile(path)
if err != nil {
return err
}
id, err := st.ID()
if err != nil {
return err
}
header, err := st.Header()
if err != nil {
return err
}
// checking that acl list contains all the needed permissions for all our nodes
err = s.checkActualNodesPermissions(st, a)
if err != nil {
return err
}
s.importedACLSyncData = ImportedACLSyncData{
Id: id,
Header: header,
Records: st.GetRawRecords(),
}
log.Infof("imported ACLList with id %s", id)
return s.storageProvider.AddStorage(id, st)
}
func (s *service) checkActualNodesPermissions(st *acllistbuilder.ACLListStorageBuilder, a *app.App) error {
nodes := a.MustComponent(node.CName).(node.Service)
acc := a.MustComponent(account.CName).(account.Service)
aclList, err := list.BuildACLListWithIdentity(acc.Account(), st)
if err != nil {
return err
}
state := aclList.ACLState()
// checking own state
if state.GetUserStates()[acc.Account().Identity].Permissions != aclpb.ACLChange_Admin {
return fmt.Errorf("own node with signing key %s should be admin", acc.Account().Identity)
}
// checking other nodes' states
for _, n := range nodes.Nodes() {
if state.GetUserStates()[n.SigningKeyString].Permissions != aclpb.ACLChange_Admin {
return fmt.Errorf("other node with signing key %s should be admin", n.SigningKeyString)
}
}
return nil
}

14
service/sync/message/service.go
View File

@ -2,6 +2,7 @@ package message
import (
"context"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app/logger"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/net/pool"
@ -85,7 +86,7 @@ func (s *service) SendMessageAsync(peerId string, msg *syncproto.Sync) (err erro
return
}
go s.sendAsync(peerId, msgType(msg), marshalled)
go s.sendAsync(peerId, msgInfo(msg), marshalled)
return
}
@ -108,15 +109,16 @@ func (s *service) sendAsync(peerId string, msgTypeStr string, marshalled []byte)
})
}
func msgType(content *syncproto.Sync) string {
func msgInfo(content *syncproto.Sync) (syncMethod string) {
msg := content.GetMessage()
switch {
case msg.GetFullSyncRequest() != nil:
return "FullSyncRequest"
syncMethod = "FullSyncRequest"
case msg.GetFullSyncResponse() != nil:
return "FullSyncResponse"
syncMethod = "FullSyncResponse"
case msg.GetHeadUpdate() != nil:
return "HeadUpdate"
syncMethod = "HeadUpdate"
}
return "UnknownMessage"
syncMethod = fmt.Sprintf("method: %s, treeType: %s", syncMethod, content.TreeHeader.DocType.String())
return
}

249
service/sync/requesthandler/requesthandler.go
View File

@ -5,9 +5,8 @@ import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/app"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app/logger"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/acltree"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/tree"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/treecache"
"github.com/anytypeio/go-anytype-infrastructure-experiments/syncproto"
@ -61,142 +60,147 @@ func (r *requestHandler) HandleSyncMessage(ctx context.Context, senderId string,
msg := content.GetMessage()
switch {
case msg.GetFullSyncRequest() != nil:
return r.HandleFullSyncRequest(ctx, senderId, msg.GetFullSyncRequest())
return r.HandleFullSyncRequest(ctx, senderId, msg.GetFullSyncRequest(), content.GetTreeHeader(), content.GetTreeId())
case msg.GetFullSyncResponse() != nil:
return r.HandleFullSyncResponse(ctx, senderId, msg.GetFullSyncResponse())
return r.HandleFullSyncResponse(ctx, senderId, msg.GetFullSyncResponse(), content.GetTreeHeader(), content.GetTreeId())
case msg.GetHeadUpdate() != nil:
return r.HandleHeadUpdate(ctx, senderId, msg.GetHeadUpdate())
return r.HandleHeadUpdate(ctx, senderId, msg.GetHeadUpdate(), content.GetTreeHeader(), content.GetTreeId())
case msg.GetAclList() != nil:
return r.HandleACLList(ctx, senderId, msg.GetAclList(), content.GetTreeHeader(), content.GetTreeId())
}
return nil
}
func (r *requestHandler) HandleHeadUpdate(ctx context.Context, senderId string, update *syncproto.SyncHeadUpdate) (err error) {
func (r *requestHandler) HandleHeadUpdate(
ctx context.Context,
senderId string,
update *syncproto.SyncHeadUpdate,
header *aclpb.Header,
treeId string) (err error) {
var (
fullRequest *syncproto.SyncFullRequest
snapshotPath []string
result acltree.AddResult
result tree.AddResult
)
log.With(zap.String("peerId", senderId), zap.String("treeId", update.TreeId)).
log.With(zap.String("peerId", senderId), zap.String("treeId", treeId)).
Debug("processing head update")
err = r.treeCache.Do(ctx, update.TreeId, func(tree acltree.ACLTree) error {
// TODO: check if we already have those changes
result, err = tree.AddRawChanges(ctx, update.Changes...)
err = r.treeCache.Do(ctx, treeId, func(obj any) error {
objTree := obj.(tree.ObjectTree)
objTree.Lock()
defer objTree.Unlock()
if slice.UnsortedEquals(update.Heads, objTree.Heads()) {
return nil
}
result, err = objTree.AddRawChanges(ctx, update.Changes...)
if err != nil {
return err
}
log.With(zap.Strings("update heads", update.Heads), zap.Strings("tree heads", tree.Heads())).
Debug("comparing heads after head update")
shouldFullSync := !slice.UnsortedEquals(update.Heads, tree.Heads())
snapshotPath = tree.SnapshotPath()
// if we couldn't add all the changes
shouldFullSync := len(update.Changes) != len(result.Added)
snapshotPath = objTree.SnapshotPath()
if shouldFullSync {
fullRequest, err = r.prepareFullSyncRequest(update.TreeId, update.TreeHeader, update.SnapshotPath, tree)
fullRequest, err = r.prepareFullSyncRequest(objTree)
if err != nil {
return err
}
}
return nil
})
// if there are no such tree
if err == treestorage.ErrUnknownTreeId {
// TODO: maybe we can optimize this by sending the header and stuff right away, so when the tree is created we are able to add it on first request
fullRequest = &syncproto.SyncFullRequest{
TreeId: update.TreeId,
TreeHeader: update.TreeHeader,
}
if err == storage.ErrUnknownTreeId {
fullRequest = &syncproto.SyncFullRequest{}
}
// if we have incompatible heads, or we haven't seen the tree at all
if fullRequest != nil {
return r.messageService.SendMessageAsync(senderId, syncproto.WrapFullRequest(fullRequest))
return r.messageService.SendMessageAsync(senderId, syncproto.WrapFullRequest(fullRequest, header, treeId))
}
// if error or nothing has changed
if err != nil || len(result.Added) == 0 {
return err
}
// otherwise sending heads update message
newUpdate := &syncproto.SyncHeadUpdate{
Heads: result.Heads,
Changes: result.Added,
SnapshotPath: snapshotPath,
TreeId: update.TreeId,
TreeHeader: update.TreeHeader,
}
return r.messageService.SendToSpaceAsync("", syncproto.WrapHeadUpdate(newUpdate))
return r.messageService.SendToSpaceAsync("", syncproto.WrapHeadUpdate(newUpdate, header, treeId))
}
func (r *requestHandler) HandleFullSyncRequest(ctx context.Context, senderId string, request *syncproto.SyncFullRequest) (err error) {
var (
fullResponse *syncproto.SyncFullResponse
snapshotPath []string
result acltree.AddResult
)
log.With(zap.String("peerId", senderId), zap.String("treeId", request.TreeId)).
Debug("processing full sync request")
func (r *requestHandler) HandleFullSyncRequest(
ctx context.Context,
senderId string,
request *syncproto.SyncFullRequest,
header *aclpb.Header,
treeId string) (err error) {
err = r.treeCache.Do(ctx, request.TreeId, func(tree acltree.ACLTree) error {
// TODO: check if we already have those changes
// if we have non-empty request
if len(request.Heads) != 0 {
result, err = tree.AddRawChanges(ctx, request.Changes...)
if err != nil {
return err
}
}
snapshotPath = tree.SnapshotPath()
fullResponse, err = r.prepareFullSyncResponse(request.TreeId, request.SnapshotPath, request.Changes, tree)
var fullResponse *syncproto.SyncFullResponse
err = r.treeCache.Do(ctx, treeId, func(obj any) error {
objTree := obj.(tree.ObjectTree)
objTree.Lock()
defer objTree.Unlock()
fullResponse, err = r.prepareFullSyncResponse(treeId, request.SnapshotPath, request.Heads, objTree)
if err != nil {
return err
}
return nil
})
if err != nil {
return err
}
err = r.messageService.SendMessageAsync(senderId, syncproto.WrapFullResponse(fullResponse))
// if error or nothing has changed
if err != nil || len(result.Added) == 0 {
return err
}
// otherwise sending heads update message
newUpdate := &syncproto.SyncHeadUpdate{
Heads: result.Heads,
Changes: result.Added,
SnapshotPath: snapshotPath,
TreeId: request.TreeId,
TreeHeader: request.TreeHeader,
}
return r.messageService.SendToSpaceAsync("", syncproto.WrapHeadUpdate(newUpdate))
return r.messageService.SendMessageAsync(senderId, syncproto.WrapFullResponse(fullResponse, header, treeId))
}
func (r *requestHandler) HandleFullSyncResponse(ctx context.Context, senderId string, response *syncproto.SyncFullResponse) (err error) {
func (r *requestHandler) HandleFullSyncResponse(
ctx context.Context,
senderId string,
response *syncproto.SyncFullResponse,
header *aclpb.Header,
treeId string) (err error) {
var (
snapshotPath []string
result acltree.AddResult
result tree.AddResult
)
log.With(zap.String("peerId", senderId), zap.String("treeId", response.TreeId)).
Debug("processing full sync response")
err = r.treeCache.Do(ctx, response.TreeId, func(tree acltree.ACLTree) error {
// TODO: check if we already have those changes
result, err = tree.AddRawChanges(ctx, response.Changes...)
err = r.treeCache.Do(ctx, treeId, func(obj interface{}) error {
objTree := obj.(tree.ObjectTree)
objTree.Lock()
defer objTree.Unlock()
// if we already have the heads for whatever reason
if slice.UnsortedEquals(response.Heads, objTree.Heads()) {
return nil
}
result, err = objTree.AddRawChanges(ctx, response.Changes...)
if err != nil {
return err
}
snapshotPath = tree.SnapshotPath()
snapshotPath = objTree.SnapshotPath()
return nil
})
// if error or nothing has changed
if (err != nil || len(result.Added) == 0) && err != treestorage.ErrUnknownTreeId {
if (err != nil || len(result.Added) == 0) && err != storage.ErrUnknownTreeId {
return err
}
// if we have a new tree
if err == treestorage.ErrUnknownTreeId {
err = r.createTree(ctx, response)
if err == storage.ErrUnknownTreeId {
err = r.createTree(ctx, response, header, treeId)
if err != nil {
return err
}
result = acltree.AddResult{
result = tree.AddResult{
OldHeads: []string{},
Heads: response.Heads,
Added: response.Changes,
@ -207,66 +211,83 @@ func (r *requestHandler) HandleFullSyncResponse(ctx context.Context, senderId st
Heads: result.Heads,
Changes: result.Added,
SnapshotPath: snapshotPath,
TreeId: response.TreeId,
}
return r.messageService.SendToSpaceAsync("", syncproto.WrapHeadUpdate(newUpdate))
return r.messageService.SendToSpaceAsync("", syncproto.WrapHeadUpdate(newUpdate, header, treeId))
}
func (r *requestHandler) prepareFullSyncRequest(treeId string, header *treepb.TreeHeader, theirPath []string, tree acltree.ACLTree) (*syncproto.SyncFullRequest, error) {
ourChanges, err := tree.ChangesAfterCommonSnapshot(theirPath)
if err != nil {
return nil, err
func (r *requestHandler) HandleACLList(
ctx context.Context,
senderId string,
req *syncproto.SyncACLList,
header *aclpb.Header,
id string) (err error) {
err = r.treeCache.Do(ctx, id, func(obj interface{}) error {
return nil
})
// do nothing if already added
if err == nil {
return nil
}
// if not found then add to storage
if err == storage.ErrUnknownTreeId {
return r.createACLList(ctx, req, header, id)
}
return err
}
func (r *requestHandler) prepareFullSyncRequest(t tree.ObjectTree) (*syncproto.SyncFullRequest, error) {
return &syncproto.SyncFullRequest{
Heads: tree.Heads(),
Changes: ourChanges,
TreeId: treeId,
SnapshotPath: tree.SnapshotPath(),
TreeHeader: header,
Heads: t.Heads(),
SnapshotPath: t.SnapshotPath(),
}, nil
}
func (r *requestHandler) prepareFullSyncResponse(
treeId string,
theirPath []string,
theirChanges []*aclpb.RawChange,
tree acltree.ACLTree) (*syncproto.SyncFullResponse, error) {
// TODO: we can probably use the common snapshot calculated on the request step from previous peer
ourChanges, err := tree.ChangesAfterCommonSnapshot(theirPath)
theirPath, theirHeads []string,
t tree.ObjectTree) (*syncproto.SyncFullResponse, error) {
ourChanges, err := t.ChangesAfterCommonSnapshot(theirPath, theirHeads)
if err != nil {
return nil, err
}
theirMap := make(map[string]struct{})
for _, ch := range theirChanges {
theirMap[ch.Id] = struct{}{}
}
// filtering our changes, so we will not send the same changes back
var final []*aclpb.RawChange
for _, ch := range ourChanges {
if _, exists := theirMap[ch.Id]; !exists {
final = append(final, ch)
}
}
log.With(zap.Int("len(changes)", len(final)), zap.String("id", treeId)).
Debug("preparing changes for tree")
return &syncproto.SyncFullResponse{
Heads: tree.Heads(),
Changes: final,
TreeId: treeId,
SnapshotPath: tree.SnapshotPath(),
TreeHeader: tree.Header(),
Heads: t.Heads(),
Changes: ourChanges,
SnapshotPath: t.SnapshotPath(),
}, nil
}
func (r *requestHandler) createTree(ctx context.Context, response *syncproto.SyncFullResponse) error {
func (r *requestHandler) createTree(
ctx context.Context,
response *syncproto.SyncFullResponse,
header *aclpb.Header,
treeId string) error {
return r.treeCache.Add(
ctx,
response.TreeId,
response.TreeHeader,
response.Changes,
func(tree acltree.ACLTree) error {
return nil
treeId,
storage.TreeStorageCreatePayload{
TreeId: treeId,
Header: header,
Changes: response.Changes,
Heads: response.Heads,
})
}
func (r *requestHandler) createACLList(
ctx context.Context,
req *syncproto.SyncACLList,
header *aclpb.Header,
treeId string) error {
return r.treeCache.Add(
ctx,
treeId,
storage.ACLListStorageCreatePayload{
ListId: treeId,
Header: header,
Records: req.Records,
})
}

108
service/treecache/service.go
View File

@ -2,88 +2,82 @@ package treecache
import (
"context"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app/logger"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/acltree"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/treestorage/treepb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/list"
aclstorage "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/tree"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/ocache"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/account"
"github.com/anytypeio/go-anytype-infrastructure-experiments/service/storage"
"go.uber.org/zap"
)
const CName = "treecache"
// TODO: add context
type ACLTreeFunc = func(tree acltree.ACLTree) error
type ChangeBuildFunc = func(builder acltree.ChangeBuilder) error
type ObjFunc = func(obj interface{}) error
var log = logger.NewNamed("treecache")
type Service interface {
Do(ctx context.Context, treeId string, f ACLTreeFunc) error
Add(ctx context.Context, treeId string, header *treepb.TreeHeader, changes []*aclpb.RawChange, f ACLTreeFunc) error
Create(ctx context.Context, build ChangeBuildFunc, f ACLTreeFunc) error
Do(ctx context.Context, id string, f ObjFunc) error
Add(ctx context.Context, id string, payload any) error
}
type service struct {
treeProvider treestorage.Provider
account account.Service
cache ocache.OCache
storage storage.Service
account account.Service
cache ocache.OCache
}
func New() app.ComponentRunnable {
return &service{}
}
func (s *service) Create(ctx context.Context, build ChangeBuildFunc, f ACLTreeFunc) error {
acc := s.account.Account()
st, err := acltree.CreateNewTreeStorageWithACL(acc, build, s.treeProvider.CreateTreeStorage)
if err != nil {
return err
}
id, err := st.TreeID()
if err != nil {
return err
}
return s.Do(ctx, id, f)
}
func (s *service) Do(ctx context.Context, treeId string, f ACLTreeFunc) error {
func (s *service) Do(ctx context.Context, treeId string, f ObjFunc) error {
log.
With(zap.String("treeId", treeId)).
Debug("requesting tree from cache to perform operation")
tree, err := s.cache.Get(ctx, treeId)
t, err := s.cache.Get(ctx, treeId)
defer s.cache.Release(treeId)
if err != nil {
return err
}
aclTree := tree.(acltree.ACLTree)
aclTree.Lock()
defer aclTree.Unlock()
return f(tree.(acltree.ACLTree))
return f(t)
}
func (s *service) Add(ctx context.Context, treeId string, header *treepb.TreeHeader, changes []*aclpb.RawChange, f ACLTreeFunc) error {
log.
With(zap.String("treeId", treeId), zap.Int("len(changes)", len(changes))).
Debug("adding tree with changes")
func (s *service) Add(ctx context.Context, treeId string, payload any) error {
switch pl := payload.(type) {
case aclstorage.TreeStorageCreatePayload:
log.
With(zap.String("treeId", treeId), zap.Int("len(changes)", len(pl.Changes))).
Debug("adding Tree with changes")
_, err := s.storage.CreateTreeStorage(payload.(aclstorage.TreeStorageCreatePayload))
if err != nil {
return err
}
case aclstorage.ACLListStorageCreatePayload:
log.
With(zap.String("treeId", treeId), zap.Int("len(changes)", len(pl.Records))).
Debug("adding ACLList with records")
_, err := s.storage.CreateACLListStorage(payload.(aclstorage.ACLListStorageCreatePayload))
if err != nil {
return err
}
_, err := s.treeProvider.CreateTreeStorage(treeId, header, changes)
if err != nil {
return err
}
return s.Do(ctx, treeId, f)
return nil
}
func (s *service) Init(ctx context.Context, a *app.App) (err error) {
s.cache = ocache.New(s.loadTree)
s.account = a.MustComponent(account.CName).(account.Service)
s.treeProvider = treestorage.NewInMemoryTreeStorageProvider()
s.storage = a.MustComponent(storage.CName).(storage.Service)
// TODO: for test we should load some predefined keys
return nil
}
@ -101,11 +95,33 @@ func (s *service) Close(ctx context.Context) (err error) {
}
func (s *service) loadTree(ctx context.Context, id string) (ocache.Object, error) {
tree, err := s.treeProvider.TreeStorage(id)
t, err := s.storage.Storage(id)
if err != nil {
return nil, err
}
// TODO: should probably accept nil listeners
aclTree, err := acltree.BuildACLTree(tree, s.account.Account(), acltree.NoOpListener{})
return aclTree, err
header, err := t.Header()
if err != nil {
return nil, err
}
switch header.DocType { // handler
case aclpb.Header_ACL:
return list.BuildACLListWithIdentity(s.account.Account(), t.(aclstorage.ListStorage))
case aclpb.Header_DocTree:
break
default:
return nil, fmt.Errorf("incorrect type")
}
log.Info("got header", zap.String("header", header.String()))
var objTree tree.ObjectTree
err = s.Do(ctx, header.AclListId, func(obj interface{}) error {
aclList := obj.(list.ACLList)
objTree, err = tree.BuildObjectTree(t.(aclstorage.TreeStorage), nil, aclList)
if err != nil {
return err
}
return nil
})
return objTree, err
}

50
syncproto/helpers.go
View File

@ -1,19 +1,45 @@
package syncproto
func WrapHeadUpdate(update *SyncHeadUpdate) *Sync {
return &Sync{Message: &SyncContentValue{
Value: &SyncContentValueValueOfHeadUpdate{HeadUpdate: update},
}}
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
)
func WrapHeadUpdate(update *SyncHeadUpdate, header *aclpb.Header, treeId string) *Sync {
return &Sync{
Message: &SyncContentValue{
Value: &SyncContentValueValueOfHeadUpdate{HeadUpdate: update},
},
TreeHeader: header,
TreeId: treeId,
}
}
func WrapFullRequest(request *SyncFullRequest) *Sync {
return &Sync{Message: &SyncContentValue{
Value: &SyncContentValueValueOfFullSyncRequest{FullSyncRequest: request},
}}
func WrapFullRequest(request *SyncFullRequest, header *aclpb.Header, treeId string) *Sync {
return &Sync{
Message: &SyncContentValue{
Value: &SyncContentValueValueOfFullSyncRequest{FullSyncRequest: request},
},
TreeHeader: header,
TreeId: treeId,
}
}
func WrapFullResponse(response *SyncFullResponse) *Sync {
return &Sync{Message: &SyncContentValue{
Value: &SyncContentValueValueOfFullSyncResponse{FullSyncResponse: response},
}}
func WrapFullResponse(response *SyncFullResponse, header *aclpb.Header, treeId string) *Sync {
return &Sync{
Message: &SyncContentValue{
Value: &SyncContentValueValueOfFullSyncResponse{FullSyncResponse: response},
},
TreeHeader: header,
TreeId: treeId,
}
}
func WrapACLList(aclList *SyncACLList, header *aclpb.Header, id string) *Sync {
return &Sync{
Message: &SyncContentValue{
Value: &SyncContentValueValueOfAclList{AclList: aclList},
},
TreeHeader: header,
TreeId: id,
}
}

20
syncproto/proto/sync.proto
View File

@ -3,7 +3,6 @@ package anytype;
option go_package = "/syncproto";
import "pkg/acl/aclchanges/aclpb/protos/aclchanges.proto";
import "pkg/acl/treestorage/treepb/protos/tree.proto";
message Message {
Header header = 1;
@ -52,21 +51,26 @@ message System {
message Sync {
string spaceId = 1;
ContentValue message = 2;
acl.Header treeHeader = 3;
string treeId = 4;
message ContentValue {
oneof value {
HeadUpdate headUpdate = 1;
Full.Request fullSyncRequest = 2;
Full.Response fullSyncResponse = 3;
ACLList aclList = 4;
}
}
message ACLList {
repeated acl.RawRecord records = 1;
}
message HeadUpdate {
repeated string heads = 1;
repeated acl.RawChange changes = 2;
string treeId = 3;
repeated string snapshotPath = 4;
tree.TreeHeader treeHeader = 5;
repeated string snapshotPath = 3;
}
message Full {
@ -74,17 +78,13 @@ message Sync {
message Request {
repeated string heads = 1;
repeated acl.RawChange changes = 2;
string treeId = 3;
repeated string snapshotPath = 4;
tree.TreeHeader treeHeader = 5;
repeated string snapshotPath = 3;
}
message Response {
repeated string heads = 1;
repeated acl.RawChange changes = 2;
string treeId = 3;
repeated string snapshotPath = 4;
tree.TreeHeader treeHeader = 5;
repeated string snapshotPath = 3;
}
}
}

834
syncproto/sync.pb.go
View File

File diff suppressed because it is too large Load Diff