Merge branch 'add-data-package-with-acl'

Makefile

@@ -0,0 +1,27 @@
+export GOPRIVATE=github.com/anytypeio
+
+ifndef $(GOPATH)
+    GOPATH=$(shell go env GOPATH)
+    export GOPATH
+endif
+
+ifndef $(GOROOT)
+    GOROOT=$(shell go env GOROOT)
+    export GOROOT
+endif
+
+export PATH=$(GOPATH)/bin:$(shell echo $$PATH)
+
+# TODO: folders were changed, so we should update Makefile and protos generation
+protos-go:
+	@echo 'Generating protobuf packages (Go)...'
+	$(eval P_TIMESTAMP := Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types)
+	$(eval P_STRUCT := Mgoogle/protobuf/struct.proto=github.com/gogo/protobuf/types)
+	@$(eval P_ACL_CHANGES := Mdata/pb/protos/aclchanges.proto=github.com/anytypeio/go-anytype-infrastructure-experiments/aclchanges/pb)
+	@$(eval P_THREAD := Mdata/pb/protos/aclchanges.proto=github.com/anytypeio/go-anytype-infrastructure-experiments/thread/pb)
+	@$(eval P_PLAINTEXT_CHANGES := Mdata/pb/protos/plaintextchanges.proto=github.com/anytypeio/go-anytype-infrastructure-experiments/testutils/testchanges/pb)
+
+	$(eval PKGMAP := $$(P_TIMESTAMP),$$(P_STRUCT),$$(P_ACL_CHANGES))
+	GOGO_NO_UNDERSCORE=1 GOGO_EXPORT_ONEOF_INTERFACE=1 protoc --gogofaster_out=$(PKGMAP):./aclchanges/pb aclchanges/pb/protos/*.*; mv aclchanges/pb/aclchanges/pb/protos/*.go aclchanges/pb; rm -rf aclchanges/pb/aclchanges
+	$(eval PKGMAP := $$(P_TIMESTAMP),$$(P_STRUCT),$$(P_THREAD))
+	GOGO_NO_UNDERSCORE=1 GOGO_EXPORT_ONEOF_INTERFACE=1 protoc --gogofaster_out=$(PKGMAP):./thread/pb thread/pb/protos/*.*; mv thread/pb/thread/pb/protos/*.go thread/pb; rm -rf thread/pb/thread

account/accountdata.go

@@ -0,0 +1,9 @@
+package account
+
+import "github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
+
+type AccountData struct {
+	Identity string
+	SignKey  keys.SigningPrivKey
+	EncKey   keys.EncryptionPrivKey
+}

aclchanges/change.go

@@ -0,0 +1,10 @@
+package aclchanges
+
+import "github.com/anytypeio/go-anytype-infrastructure-experiments/aclchanges/pb"
+
+type Change interface {
+	ProtoChange() *pb.ACLChange
+	DecryptedChangeContent() []byte
+	Signature() []byte
+	CID() string
+}

aclchanges/pb/protos/aclchanges.proto

@@ -0,0 +1,105 @@
+syntax = "proto3";
+package anytype;
+option go_package = "pb";
+
+// the element of change tree used to store and internal apply smartBlock history
+message ACLChange {
+  repeated string treeHeadIds = 1;
+  repeated string aclHeadIds = 2;
+  string snapshotBaseId = 3; // we will only have one base snapshot for both
+  ACLData aclData = 4;
+  // the data is encoded with read key and should be read in ChangesData format
+  bytes changesData = 5;
+  uint64 currentReadKeyHash = 6;
+  int64 timestamp = 7;
+  string identity = 8;
+
+  message ACLContentValue {
+    oneof value {
+      UserAdd userAdd = 1;
+      UserRemove userRemove = 2;
+      UserPermissionChange userPermissionChange = 3;
+      UserInvite userInvite = 4;
+      UserJoin userJoin = 5;
+      UserConfirm userConfirm = 6;
+    }
+  }
+
+  message ACLData {
+    ACLSnapshot aclSnapshot = 1;
+    repeated ACLContentValue aclContent = 2;
+  }
+
+  message ACLSnapshot {
+    // We don't need ACLState as a separate message now, because we simplified the snapshot model
+    ACLState aclState = 1;
+  }
+
+  message ACLState {
+    repeated uint64 readKeyHashes = 1;
+    repeated UserState userStates = 2;
+    map<string, UserInvite> invites = 3; // TODO: later
+    //    repeated string unconfirmedUsers = 4; // TODO: later
+  }
+
+  message UserState {
+    string identity = 1;
+    bytes encryptionKey = 2;
+    repeated bytes encryptedReadKeys = 3; // all read keys that we know
+    UserPermissions permissions = 4;
+    bool IsConfirmed = 5;
+  }
+
+  // we already know identity and encryptionKey
+  message UserAdd {
+    string identity = 1; // public signing key
+    bytes encryptionKey = 2; // public encryption key
+    repeated bytes encryptedReadKeys = 3; // all read keys that we know for the user
+    UserPermissions permissions = 4;
+  }
+
+  // TODO: this is not used as of now
+  message UserConfirm { // not needed for read permissions
+    string identity = 1; // not needed
+    string userAddId = 2;
+  }
+
+  message UserInvite {
+    bytes acceptPublicKey = 1;
+    bytes encryptPublicKey = 2;
+    repeated bytes encryptedReadKeys = 3; // all read keys that we know for the user
+    UserPermissions permissions = 4;
+    string InviteId = 5;
+  }
+
+  message UserJoin {
+    string identity = 1;
+    bytes encryptionKey = 2;
+    bytes acceptSignature = 3; // sign acceptPublicKey
+    string userInviteId = 4;
+    repeated bytes encryptedReadKeys = 5; // the idea is that user should itself reencrypt the keys with the pub key
+  }
+
+  message UserRemove {
+    string identity = 1;
+    repeated ReadKeyReplace readKeyReplaces = 3; // new read key encrypted for all users
+  }
+
+  message ReadKeyReplace {
+    string identity = 1;
+    bytes encryptionKey = 2;
+    bytes encryptedReadKey = 3;
+  }
+
+  message UserPermissionChange {
+    string identity = 1;
+    UserPermissions permissions = 2;
+  }
+
+  enum UserPermissions {
+    Admin = 0;
+    Writer = 1;
+    Reader = 2;
+    Removed = 3;
+  }
+}

acltree/aclstate.go

@@ -0,0 +1,411 @@
+package acltree
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"hash/fnv"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/aclchanges/pb"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
+	"github.com/textileio/go-threads/crypto/symmetric"
+)
+
+var ErrNoSuchUser = errors.New("no such user")
+var ErrFailedToDecrypt = errors.New("failed to decrypt key")
+var ErrUserRemoved = errors.New("user was removed from the document")
+var ErrDocumentForbidden = errors.New("your user was forbidden access to the document")
+var ErrUserAlreadyExists = errors.New("user already exists")
+
+type ACLState struct {
+	currentReadKeyHash   uint64
+	userReadKeys         map[uint64]*symmetric.Key
+	userStates           map[string]*pb.ACLChangeUserState
+	userInvites          map[string]*pb.ACLChangeUserInvite
+	signingPubKeyDecoder keys.SigningPubKeyDecoder
+	encryptionKey        keys.EncryptionPrivKey
+	identity             string
+}
+
+func newACLState(
+	identity string,
+	encryptionKey keys.EncryptionPrivKey,
+	signingPubKeyDecoder keys.SigningPubKeyDecoder) *ACLState {
+	return &ACLState{
+		identity:             identity,
+		encryptionKey:        encryptionKey,
+		userReadKeys:         make(map[uint64]*symmetric.Key),
+		userStates:           make(map[string]*pb.ACLChangeUserState),
+		userInvites:          make(map[string]*pb.ACLChangeUserInvite),
+		signingPubKeyDecoder: signingPubKeyDecoder,
+	}
+}
+
+func newACLStateFromSnapshotChange(
+	snapshotChange *pb.ACLChange,
+	identity string,
+	encryptionKey keys.EncryptionPrivKey,
+	signingPubKeyDecoder keys.SigningPubKeyDecoder) (*ACLState, error) {
+	st := &ACLState{
+		identity:             identity,
+		encryptionKey:        encryptionKey,
+		userReadKeys:         make(map[uint64]*symmetric.Key),
+		userStates:           make(map[string]*pb.ACLChangeUserState),
+		userInvites:          make(map[string]*pb.ACLChangeUserInvite),
+		signingPubKeyDecoder: signingPubKeyDecoder,
+	}
+	err := st.recreateFromSnapshotChange(snapshotChange)
+	if err != nil {
+		return nil, err
+	}
+	return st, nil
+}
+
+func (st *ACLState) recreateFromSnapshotChange(snapshotChange *pb.ACLChange) error {
+	snapshot := snapshotChange.GetAclData().GetAclSnapshot()
+	if snapshot == nil {
+		return fmt.Errorf("could not create state from snapshot, because it is nil")
+	}
+	state := snapshot.AclState
+	for _, userState := range state.UserStates {
+		st.userStates[userState.Identity] = userState
+	}
+
+	userState, exists := st.userStates[st.identity]
+	if !exists {
+		return ErrNoSuchUser
+	}
+	for _, key := range userState.EncryptedReadKeys {
+		key, hash, err := st.decryptReadKeyAndHash(key)
+		if err != nil {
+			return ErrFailedToDecrypt
+		}
+
+		st.userReadKeys[hash] = key
+	}
+	st.currentReadKeyHash = snapshotChange.CurrentReadKeyHash
+	if snapshot.GetAclState().GetInvites() != nil {
+		st.userInvites = snapshot.GetAclState().GetInvites()
+	}
+	return nil
+}
+
+func (st *ACLState) makeSnapshot() *pb.ACLChangeACLSnapshot {
+	var userStates []*pb.ACLChangeUserState
+	for _, st := range st.userStates {
+		userStates = append(userStates, st)
+	}
+
+	return &pb.ACLChangeACLSnapshot{AclState: &pb.ACLChangeACLState{
+		ReadKeyHashes: nil,
+		UserStates:    userStates, // TODO: make states and invites in same format
+		Invites:       st.userInvites,
+	}}
+}
+
+func (st *ACLState) applyChange(change *pb.ACLChange) (err error) {
+	defer func() {
+		if err != nil {
+			return
+		}
+		st.currentReadKeyHash = change.CurrentReadKeyHash
+	}()
+
+	// we can't check this for the user which is joining, because it will not be in our list
+	// the same is for the first change to be added
+	skipIdentityCheck := st.isUserJoin(change) || (st.currentReadKeyHash == 0 && st.isUserAdd(change))
+	if !skipIdentityCheck {
+		// we check signature when we add this to the Tree, so no need to do it here
+		if _, exists := st.userStates[change.Identity]; !exists {
+			err = ErrNoSuchUser
+			return
+		}
+
+		if !st.hasPermission(change.Identity, pb.ACLChange_Admin) {
+			err = fmt.Errorf("user %s must have admin permissions", change.Identity)
+			return
+		}
+	}
+
+	for _, ch := range change.GetAclData().GetAclContent() {
+		if err = st.applyChangeContent(ch); err != nil {
+			//log.Infof("error while applying changes: %v; ignore", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+// TODO: remove changeId, because it is not needed
+func (st *ACLState) applyChangeContent(ch *pb.ACLChangeACLContentValue) error {
+	switch {
+	case ch.GetUserPermissionChange() != nil:
+		return st.applyUserPermissionChange(ch.GetUserPermissionChange())
+	case ch.GetUserAdd() != nil:
+		return st.applyUserAdd(ch.GetUserAdd())
+	case ch.GetUserRemove() != nil:
+		return st.applyUserRemove(ch.GetUserRemove())
+	case ch.GetUserInvite() != nil:
+		return st.applyUserInvite(ch.GetUserInvite())
+	case ch.GetUserJoin() != nil:
+		return st.applyUserJoin(ch.GetUserJoin())
+	case ch.GetUserConfirm() != nil:
+		return st.applyUserConfirm(ch.GetUserConfirm())
+	default:
+		return fmt.Errorf("unexpected change type: %v", ch)
+	}
+}
+
+func (st *ACLState) applyUserPermissionChange(ch *pb.ACLChangeUserPermissionChange) error {
+	if _, exists := st.userStates[ch.Identity]; !exists {
+		return ErrNoSuchUser
+	}
+
+	st.userStates[ch.Identity].Permissions = ch.Permissions
+	return nil
+}
+
+func (st *ACLState) applyUserInvite(ch *pb.ACLChangeUserInvite) error {
+	st.userInvites[ch.InviteId] = ch
+	return nil
+}
+
+func (st *ACLState) applyUserJoin(ch *pb.ACLChangeUserJoin) error {
+	invite, exists := st.userInvites[ch.UserInviteId]
+	if !exists {
+		return fmt.Errorf("no such invite with id %s", ch.UserInviteId)
+	}
+
+	if _, exists = st.userStates[ch.Identity]; exists {
+		return ErrUserAlreadyExists
+	}
+
+	// validating signature
+	signature := ch.GetAcceptSignature()
+	verificationKey, err := st.signingPubKeyDecoder.DecodeFromBytes(invite.AcceptPublicKey)
+	if err != nil {
+		return fmt.Errorf("public key verifying invite accepts is given in incorrect format: %v", err)
+	}
+
+	rawSignedId, err := st.signingPubKeyDecoder.DecodeFromStringIntoBytes(ch.Identity)
+	if err != nil {
+		return fmt.Errorf("failed to decode signing identity as bytes")
+	}
+
+	res, err := verificationKey.Verify(rawSignedId, signature)
+	if err != nil {
+		return fmt.Errorf("verification returned error: %w", err)
+	}
+	if !res {
+		return fmt.Errorf("signature is invalid")
+	}
+
+	// if ourselves -> we need to decrypt the read keys
+	if st.identity == ch.Identity {
+		for _, key := range ch.EncryptedReadKeys {
+			key, hash, err := st.decryptReadKeyAndHash(key)
+			if err != nil {
+				return ErrFailedToDecrypt
+			}
+
+			st.userReadKeys[hash] = key
+		}
+	}
+
+	// adding user to the list
+	userState := &pb.ACLChangeUserState{
+		Identity:          ch.Identity,
+		EncryptionKey:     ch.EncryptionKey,
+		EncryptedReadKeys: ch.EncryptedReadKeys,
+		Permissions:       invite.Permissions,
+		IsConfirmed:       true,
+	}
+	st.userStates[ch.Identity] = userState
+	return nil
+}
+
+func (st *ACLState) applyUserAdd(ch *pb.ACLChangeUserAdd) error {
+	if _, exists := st.userStates[ch.Identity]; exists {
+		return ErrUserAlreadyExists
+	}
+
+	st.userStates[ch.Identity] = &pb.ACLChangeUserState{
+		Identity:          ch.Identity,
+		EncryptionKey:     ch.EncryptionKey,
+		Permissions:       ch.Permissions,
+		EncryptedReadKeys: ch.EncryptedReadKeys,
+	}
+
+	if ch.Identity == st.identity {
+		for _, key := range ch.EncryptedReadKeys {
+			key, hash, err := st.decryptReadKeyAndHash(key)
+			if err != nil {
+				return ErrFailedToDecrypt
+			}
+
+			st.userReadKeys[hash] = key
+		}
+	}
+
+	return nil
+}
+
+func (st *ACLState) applyUserRemove(ch *pb.ACLChangeUserRemove) error {
+	if ch.Identity == st.identity {
+		return ErrDocumentForbidden
+	}
+
+	if _, exists := st.userStates[ch.Identity]; !exists {
+		return ErrNoSuchUser
+	}
+
+	delete(st.userStates, ch.Identity)
+
+	for _, replace := range ch.ReadKeyReplaces {
+		userState, exists := st.userStates[replace.Identity]
+		if !exists {
+			continue
+		}
+
+		userState.EncryptedReadKeys = append(userState.EncryptedReadKeys, replace.EncryptedReadKey)
+		// if this is our identity then we have to decrypt the key
+		if replace.Identity == st.identity {
+			key, hash, err := st.decryptReadKeyAndHash(replace.EncryptedReadKey)
+			if err != nil {
+				return ErrFailedToDecrypt
+			}
+
+			st.currentReadKeyHash = hash
+			st.userReadKeys[st.currentReadKeyHash] = key
+		}
+	}
+	return nil
+}
+
+func (st *ACLState) applyUserConfirm(ch *pb.ACLChangeUserConfirm) error {
+	if _, exists := st.userStates[ch.Identity]; !exists {
+		return ErrNoSuchUser
+	}
+
+	userState := st.userStates[ch.Identity]
+	userState.IsConfirmed = true
+	return nil
+}
+
+func (st *ACLState) decryptReadKeyAndHash(msg []byte) (*symmetric.Key, uint64, error) {
+	decrypted, err := st.encryptionKey.Decrypt(msg)
+	if err != nil {
+		return nil, 0, ErrFailedToDecrypt
+	}
+
+	key, err := symmetric.FromBytes(decrypted)
+	if err != nil {
+		return nil, 0, ErrFailedToDecrypt
+	}
+
+	hasher := fnv.New64()
+	hasher.Write(decrypted)
+	return key, hasher.Sum64(), nil
+}
+
+func (st *ACLState) hasPermission(identity string, permission pb.ACLChangeUserPermissions) bool {
+	state, exists := st.userStates[identity]
+	if !exists {
+		return false
+	}
+
+	return state.Permissions == permission
+}
+
+func (st *ACLState) isUserJoin(ch *pb.ACLChange) bool {
+	// if we have a UserJoin, then it should always be the first one applied
+	return ch.AclData.GetAclContent() != nil && ch.AclData.GetAclContent()[0].GetUserJoin() != nil
+}
+
+func (st *ACLState) isUserAdd(ch *pb.ACLChange) bool {
+	// if we have a UserAdd, then it should always be the first one applied
+	userAdd := ch.AclData.GetAclContent()[0].GetUserAdd()
+	return ch.AclData.GetAclContent() != nil && userAdd != nil && userAdd.GetIdentity() == ch.Identity
+}
+
+func (st *ACLState) getPermissionDecreasedUsers(ch *pb.ACLChange) (identities []*pb.ACLChangeUserPermissionChange) {
+	// this should be called after general checks are completed
+	if ch.GetAclData().GetAclContent() == nil {
+		return nil
+	}
+
+	contents := ch.GetAclData().GetAclContent()
+	for _, c := range contents {
+		if c.GetUserPermissionChange() != nil {
+			content := c.GetUserPermissionChange()
+
+			currentState := st.userStates[content.Identity]
+			// the comparison works in different direction :-)
+			if content.Permissions > currentState.Permissions {
+				identities = append(identities, &pb.ACLChangeUserPermissionChange{
+					Identity:    content.Identity,
+					Permissions: content.Permissions,
+				})
+			}
+		}
+		if c.GetUserRemove() != nil {
+			content := c.GetUserRemove()
+			identities = append(identities, &pb.ACLChangeUserPermissionChange{
+				Identity:    content.Identity,
+				Permissions: pb.ACLChange_Removed,
+			})
+		}
+	}
+
+	return identities
+}
+
+func (st *ACLState) equal(other *ACLState) bool {
+	if st == nil && other == nil {
+		return true
+	}
+
+	if st == nil || other == nil {
+		return false
+	}
+
+	if st.currentReadKeyHash != other.currentReadKeyHash {
+		return false
+	}
+
+	if st.identity != other.identity {
+		return false
+	}
+
+	if len(st.userStates) != len(other.userStates) {
+		return false
+	}
+
+	for _, st := range st.userStates {
+		otherSt, exists := other.userStates[st.Identity]
+		if !exists {
+			return false
+		}
+
+		if st.Permissions != otherSt.Permissions {
+			return false
+		}
+
+		if bytes.Compare(st.EncryptionKey, otherSt.EncryptionKey) != 0 {
+			return false
+		}
+	}
+
+	if len(st.userInvites) != len(other.userInvites) {
+		return false
+	}
+
+	// TODO: add detailed user invites comparison + compare other stuff
+	return true
+}
+
+func (st *ACLState) GetUserStates() map[string]*pb.ACLChangeUserState {
+	// TODO: we should provide better API that would not allow to change this map from the outside
+	return st.userStates
+}

acltree/aclstatebuilder.go

@@ -0,0 +1,186 @@
+package acltree
+
+import (
+	"fmt"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/account"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/aclchanges/pb"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
+)
+
+type aclStateBuilder struct {
+	tree     *Tree
+	identity string
+	key      keys.EncryptionPrivKey
+	decoder  keys.SigningPubKeyDecoder
+}
+
+type decreasedPermissionsParameters struct {
+	users       []*pb.ACLChangeUserPermissionChange
+	startChange string
+}
+
+func newACLStateBuilder(decoder keys.SigningPubKeyDecoder, accountData *account.AccountData) *aclStateBuilder {
+	return &aclStateBuilder{
+		decoder:  decoder,
+		identity: accountData.Identity,
+		key:      accountData.EncKey,
+	}
+}
+
+func (sb *aclStateBuilder) Init(tree *Tree) error {
+	sb.tree = tree
+	return nil
+}
+
+func (sb *aclStateBuilder) Build() (*ACLState, error) {
+	state, _, err := sb.BuildBefore("")
+	return state, err
+}
+
+// TODO: we can probably have only one state builder, because we can Build both at the same time
+func (sb *aclStateBuilder) BuildBefore(beforeId string) (*ACLState, bool, error) {
+	var (
+		err                  error
+		startChange          = sb.tree.root
+		foundId              bool
+		idSeenMap            = make(map[string][]*Change)
+		decreasedPermissions *decreasedPermissionsParameters
+	)
+	root := sb.tree.Root()
+	if !root.IsSnapshot {
+		return nil, false, fmt.Errorf("root should always be a snapshot")
+	}
+
+	state, err := newACLStateFromSnapshotChange(
+		root.Content,
+		sb.identity,
+		sb.key,
+		sb.decoder)
+	if err != nil {
+		return nil, false, fmt.Errorf("could not build ACLState from snapshot: %w", err)
+	}
+
+	idSeenMap[startChange.Content.Identity] = append(idSeenMap[startChange.Content.Identity], startChange)
+
+	if startChange.Content.GetChangesData() != nil {
+		key, exists := state.userReadKeys[startChange.Content.CurrentReadKeyHash]
+		if !exists {
+			return nil, false, fmt.Errorf("no first snapshot")
+		}
+
+		err = startChange.DecryptContents(key)
+		if err != nil {
+			return nil, false, fmt.Errorf("failed to decrypt contents of first snapshot")
+		}
+	}
+
+	if beforeId == startChange.Id {
+		return state, true, nil
+	}
+
+	for {
+		// TODO: we should optimize this method to just remember last state of iterator and not iterate from the start and skip if nothing was removed from the Tree
+		sb.tree.IterateSkip(sb.tree.root.Id, startChange.Id, func(c *Change) (isContinue bool) {
+			defer func() {
+				if err == nil {
+					startChange = c
+				} else if err != ErrDocumentForbidden {
+					//log.Errorf("marking change %s as invalid: %v", c.Id, err)
+					sb.tree.RemoveInvalidChange(c.Id)
+				}
+			}()
+
+			// not applying root change
+			if c.Id == startChange.Id {
+				return true
+			}
+
+			idSeenMap[c.Content.Identity] = append(idSeenMap[c.Content.Identity], c)
+			if c.Content.GetAclData() != nil {
+				err = state.applyChange(c.Content)
+				if err != nil {
+					return false
+				}
+
+				// if we have some users who have less permissions now
+				users := state.getPermissionDecreasedUsers(c.Content)
+				if len(users) > 0 {
+					decreasedPermissions = &decreasedPermissionsParameters{
+						users:       users,
+						startChange: c.Id,
+					}
+					return false
+				}
+			}
+
+			// the user can't make changes
+			if !state.hasPermission(c.Content.Identity, pb.ACLChange_Writer) && !state.hasPermission(c.Content.Identity, pb.ACLChange_Admin) {
+				err = fmt.Errorf("user %s cannot make changes", c.Content.Identity)
+				return false
+			}
+
+			// decrypting contents on the fly
+			if c.Content.GetChangesData() != nil {
+				key, exists := state.userReadKeys[c.Content.CurrentReadKeyHash]
+				if !exists {
+					err = fmt.Errorf("failed to find key with hash: %d", c.Content.CurrentReadKeyHash)
+					return false
+				}
+
+				err = c.DecryptContents(key)
+				if err != nil {
+					err = fmt.Errorf("failed to decrypt contents for hash: %d", c.Content.CurrentReadKeyHash)
+					return false
+				}
+			}
+
+			if c.Id == beforeId {
+				foundId = true
+				return false
+			}
+
+			return true
+		})
+
+		// if we have users with decreased permissions
+		if decreasedPermissions != nil {
+			var removed bool
+			validChanges := sb.tree.dfs(decreasedPermissions.startChange)
+
+			for _, permChange := range decreasedPermissions.users {
+				seenChanges := idSeenMap[permChange.Identity]
+
+				for _, seen := range seenChanges {
+					// if we find some invalid changes
+					if _, exists := validChanges[seen.Id]; !exists {
+						// if the user didn't have enough permission to make changes
+						if seen.IsACLChange() || permChange.Permissions > pb.ACLChange_Writer {
+							removed = true
+							sb.tree.RemoveInvalidChange(seen.Id)
+						}
+					}
+				}
+			}
+
+			decreasedPermissions = nil
+			if removed {
+				// starting from the beginning but with updated Tree
+				return sb.BuildBefore(beforeId)
+			}
+		} else if err == nil {
+			// we can finish the acl state building process
+			break
+		}
+
+		// the user is forbidden to access the document
+		if err == ErrDocumentForbidden {
+			return nil, foundId, err
+		}
+
+		// otherwise we have to continue from the change which we had
+		err = nil
+	}
+
+	return state, foundId, err
+}

acltree/acltree.go

@@ -0,0 +1,325 @@
+package acltree
+
+import (
+	"sync"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/account"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/thread"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
+)
+
+type AddResultSummary int
+
+const (
+	AddResultSummaryNothing AddResultSummary = iota
+	AddResultSummaryAppend
+	AddResultSummaryRebuild
+)
+
+type AddResult struct {
+	OldHeads []string
+	Heads    []string
+	// TODO: add summary for changes
+	Summary AddResultSummary
+}
+
+type TreeUpdateListener interface {
+	Update(tree ACLTree)
+	Rebuild(tree ACLTree)
+}
+
+type ACLTree interface {
+	ACLState() *ACLState
+	AddContent(f func(builder ChangeBuilder) error) (*Change, error)
+	AddChanges(changes ...*Change) (AddResult, error)
+	Heads() []string
+	Root() *Change
+	Iterate(func(change *Change) bool)
+	IterateFrom(string, func(change *Change) bool)
+	HasChange(string) bool
+}
+
+type aclTree struct {
+	thread         thread.Thread
+	accountData    *account.AccountData
+	updateListener TreeUpdateListener
+
+	fullTree         *Tree
+	aclTreeFromStart *Tree // TODO: right now we don't use it, we can probably have only local var for now. This tree is built from start of the document
+	aclState         *ACLState
+
+	treeBuilder       *treeBuilder
+	aclTreeBuilder    *aclTreeBuilder
+	aclStateBuilder   *aclStateBuilder
+	snapshotValidator *snapshotValidator
+	changeBuilder     *changeBuilder
+
+	sync.RWMutex
+}
+
+func BuildACLTree(
+	t thread.Thread,
+	acc *account.AccountData,
+	listener TreeUpdateListener) (ACLTree, error) {
+	decoder := keys.NewEd25519Decoder()
+	aclTreeBuilder := newACLTreeBuilder(t, decoder)
+	treeBuilder := newTreeBuilder(t, decoder)
+	snapshotValidator := newSnapshotValidator(decoder, acc)
+	aclStateBuilder := newACLStateBuilder(decoder, acc)
+	changeBuilder := newChangeBuilder()
+
+	aclTree := &aclTree{
+		thread:            t,
+		accountData:       acc,
+		fullTree:          nil,
+		aclState:          nil,
+		treeBuilder:       treeBuilder,
+		aclTreeBuilder:    aclTreeBuilder,
+		aclStateBuilder:   aclStateBuilder,
+		snapshotValidator: snapshotValidator,
+		changeBuilder:     changeBuilder,
+		updateListener:    listener,
+	}
+	err := aclTree.rebuildFromThread(false)
+	if err != nil {
+		return nil, err
+	}
+	aclTree.removeOrphans()
+	t.SetHeads(aclTree.Heads())
+	listener.Rebuild(aclTree)
+
+	return aclTree, nil
+}
+
+// TODO: this is not used for now, in future we should think about not making full tree rebuild
+//func (a *aclTree) rebuildFromTree(validateSnapshot bool) (err error) {
+//	if validateSnapshot {
+//		err = a.snapshotValidator.Init(a.aclTreeFromStart)
+//		if err != nil {
+//			return err
+//		}
+//
+//		valid, err := a.snapshotValidator.ValidateSnapshot(a.fullTree.root)
+//		if err != nil {
+//			return err
+//		}
+//		if !valid {
+//			return a.rebuildFromThread(true)
+//		}
+//	}
+//
+//	err = a.aclStateBuilder.Init(a.fullTree)
+//	if err != nil {
+//		return err
+//	}
+//
+//	a.aclState, err = a.aclStateBuilder.Build()
+//	if err != nil {
+//		return err
+//	}
+//
+//	return nil
+//}
+
+func (a *aclTree) removeOrphans() {
+	// removing attached or invalid orphans
+	var toRemove []string
+
+	for _, orphan := range a.thread.Orphans() {
+		if _, exists := a.fullTree.attached[orphan]; exists {
+			toRemove = append(toRemove, orphan)
+		}
+		if _, exists := a.fullTree.invalidChanges[orphan]; exists {
+			toRemove = append(toRemove, orphan)
+		}
+	}
+	a.thread.RemoveOrphans(toRemove...)
+}
+
+func (a *aclTree) rebuildFromThread(fromStart bool) error {
+	a.treeBuilder.Init()
+	a.aclTreeBuilder.Init()
+
+	var err error
+	a.fullTree, err = a.treeBuilder.Build(fromStart)
+	if err != nil {
+		return err
+	}
+
+	// TODO: remove this from context as this is used only to validate snapshot
+	a.aclTreeFromStart, err = a.aclTreeBuilder.Build()
+	if err != nil {
+		return err
+	}
+
+	if !fromStart {
+		err = a.snapshotValidator.Init(a.aclTreeFromStart)
+		if err != nil {
+			return err
+		}
+
+		valid, err := a.snapshotValidator.ValidateSnapshot(a.fullTree.root)
+		if err != nil {
+			return err
+		}
+		if !valid {
+			return a.rebuildFromThread(true)
+		}
+	}
+	// TODO: there is a question how we can validate not only that the full tree is built correctly
+	//  but also that the ACL prev ids are not messed up. I think we should probably compare the resulting
+	//  acl state with the acl state which is built in aclTreeFromStart
+
+	err = a.aclStateBuilder.Init(a.fullTree)
+	if err != nil {
+		return err
+	}
+
+	a.aclState, err = a.aclStateBuilder.Build()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (a *aclTree) ACLState() *ACLState {
+	a.RLock()
+	defer a.RUnlock()
+	return a.aclState
+}
+
+func (a *aclTree) AddContent(build func(builder ChangeBuilder) error) (*Change, error) {
+	// TODO: add snapshot creation logic
+	a.Lock()
+	defer func() {
+		a.Unlock()
+		// TODO: should this be called in a separate goroutine to prevent accidental cycles (tree->updater->tree)
+		a.updateListener.Update(a)
+	}()
+
+	a.changeBuilder.Init(a.aclState, a.fullTree, a.accountData)
+	err := build(a.changeBuilder)
+	if err != nil {
+		return nil, err
+	}
+
+	ch, marshalled, err := a.changeBuilder.BuildAndApply()
+	if err != nil {
+		return nil, err
+	}
+	a.fullTree.AddFast(ch)
+
+	err = a.thread.AddRawChange(&thread.RawChange{
+		Payload:   marshalled,
+		Signature: ch.Signature(),
+		Id:        ch.Id,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	a.thread.SetHeads([]string{ch.Id})
+	return ch, nil
+}
+
+func (a *aclTree) AddChanges(changes ...*Change) (AddResult, error) {
+	a.Lock()
+	// TODO: make proper error handling, because there are a lot of corner cases where this will break
+	var err error
+	var mode Mode
+
+	defer func() {
+		if err != nil {
+			return
+		}
+		a.removeOrphans()
+		a.thread.SetHeads(a.fullTree.Heads())
+		a.Unlock()
+		switch mode {
+		case Append:
+			a.updateListener.Update(a)
+		case Rebuild:
+			a.updateListener.Rebuild(a)
+		default:
+			break
+		}
+	}()
+
+	for _, ch := range changes {
+		err = a.thread.AddChange(ch)
+		if err != nil {
+			return AddResult{}, err
+		}
+		a.thread.AddOrphans(ch.Id)
+	}
+
+	prevHeads := a.fullTree.Heads()
+	mode = a.fullTree.Add(changes...)
+	switch mode {
+	case Nothing:
+		return AddResult{
+			OldHeads: prevHeads,
+			Heads:    prevHeads,
+			Summary:  AddResultSummaryNothing,
+		}, nil
+
+	case Rebuild:
+		err = a.rebuildFromThread(false)
+		if err != nil {
+			return AddResult{}, err
+		}
+
+		return AddResult{
+			OldHeads: prevHeads,
+			Heads:    a.fullTree.Heads(),
+			Summary:  AddResultSummaryRebuild,
+		}, nil
+	default:
+		// just rebuilding the state from start without reloading everything from thread
+		// as an optimization we could've started from current heads, but I didn't implement that
+		a.aclState, err = a.aclStateBuilder.Build()
+		if err != nil {
+			return AddResult{}, err
+		}
+
+		return AddResult{
+			OldHeads: prevHeads,
+			Heads:    a.fullTree.Heads(),
+			Summary:  AddResultSummaryAppend,
+		}, nil
+	}
+}
+
+func (a *aclTree) Iterate(f func(change *Change) bool) {
+	a.RLock()
+	defer a.RUnlock()
+	a.fullTree.Iterate(a.fullTree.RootId(), f)
+}
+
+func (a *aclTree) IterateFrom(s string, f func(change *Change) bool) {
+	a.RLock()
+	defer a.RUnlock()
+	a.fullTree.Iterate(s, f)
+}
+
+func (a *aclTree) HasChange(s string) bool {
+	a.RLock()
+	defer a.RUnlock()
+	_, attachedExists := a.fullTree.attached[s]
+	_, unattachedExists := a.fullTree.unAttached[s]
+	_, invalidExists := a.fullTree.invalidChanges[s]
+	return attachedExists || unattachedExists || invalidExists
+}
+
+func (a *aclTree) Heads() []string {
+	a.RLock()
+	defer a.RUnlock()
+	return a.fullTree.Heads()
+}
+
+func (a *aclTree) Root() *Change {
+	a.RLock()
+	defer a.RUnlock()
+	return a.fullTree.Root()
+}

acltree/acltree_test.go

@@ -0,0 +1,271 @@
+package acltree
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/account"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/aclchanges/pb"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/testutils/threadbuilder"
+)
+
+type mockListener struct{}
+
+func (m *mockListener) Update(tree ACLTree) {}
+
+func (m *mockListener) Rebuild(tree ACLTree) {}
+
+func TestACLTree_UserJoinBuild(t *testing.T) {
+	thr, err := threadbuilder.NewThreadBuilderWithTestName("userjoinexample.yml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	keychain := thr.GetKeychain()
+	accountData := &account.AccountData{
+		Identity: keychain.GetIdentity("A"),
+		SignKey:  keychain.SigningKeys["A"],
+		EncKey:   keychain.EncryptionKeys["A"],
+	}
+	listener := &mockListener{}
+	tree, err := BuildACLTree(thr, accountData, listener)
+	if err != nil {
+		t.Fatalf("should Build acl ACLState without err: %v", err)
+	}
+	aclState := tree.ACLState()
+	aId := keychain.GeneratedIdentities["A"]
+	bId := keychain.GeneratedIdentities["B"]
+	cId := keychain.GeneratedIdentities["C"]
+
+	assert.Equal(t, aclState.identity, aId)
+	assert.Equal(t, aclState.userStates[aId].Permissions, pb.ACLChange_Admin)
+	assert.Equal(t, aclState.userStates[bId].Permissions, pb.ACLChange_Writer)
+	assert.Equal(t, aclState.userStates[cId].Permissions, pb.ACLChange_Reader)
+
+	var changeIds []string
+	tree.Iterate(func(c *Change) (isContinue bool) {
+		changeIds = append(changeIds, c.Id)
+		return true
+	})
+	assert.Equal(t, changeIds, []string{"A.1.1", "A.1.2", "B.1.1", "B.1.2"})
+}
+
+func TestACLTree_UserJoinUpdate_Append(t *testing.T) {
+	thr, err := threadbuilder.NewThreadBuilderWithTestName("userjoinexampleupdate.yml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	keychain := thr.GetKeychain()
+	accountData := &account.AccountData{
+		Identity: keychain.GetIdentity("A"),
+		SignKey:  keychain.SigningKeys["A"],
+		EncKey:   keychain.EncryptionKeys["A"],
+	}
+	listener := &mockListener{}
+	tree, err := BuildACLTree(thr, accountData, listener)
+	if err != nil {
+		t.Fatalf("should Build acl ACLState without err: %v", err)
+	}
+	rawChanges := thr.GetUpdates("append")
+	var changes []*Change
+	for _, ch := range rawChanges {
+		newCh, err := NewFromRawChange(ch)
+		if err != nil {
+			t.Fatalf("should be able to create change from raw: %v", err)
+		}
+		changes = append(changes, newCh)
+	}
+
+	res, err := tree.AddChanges(changes...)
+	assert.Equal(t, res.Summary, AddResultSummaryAppend)
+
+	aclState := tree.ACLState()
+	aId := keychain.GeneratedIdentities["A"]
+	bId := keychain.GeneratedIdentities["B"]
+	cId := keychain.GeneratedIdentities["C"]
+	dId := keychain.GeneratedIdentities["D"]
+
+	assert.Equal(t, aclState.identity, aId)
+	assert.Equal(t, aclState.userStates[aId].Permissions, pb.ACLChange_Admin)
+	assert.Equal(t, aclState.userStates[bId].Permissions, pb.ACLChange_Writer)
+	assert.Equal(t, aclState.userStates[cId].Permissions, pb.ACLChange_Reader)
+	assert.Equal(t, aclState.userStates[dId].Permissions, pb.ACLChange_Writer)
+
+	var changeIds []string
+	tree.Iterate(func(c *Change) (isContinue bool) {
+		changeIds = append(changeIds, c.Id)
+		return true
+	})
+	assert.Equal(t, changeIds, []string{"A.1.1", "A.1.2", "B.1.1", "B.1.2", "B.1.3", "A.1.4"})
+}
+
+func TestACLTree_UserJoinUpdate_Rebuild(t *testing.T) {
+	thr, err := threadbuilder.NewThreadBuilderWithTestName("userjoinexampleupdate.yml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	keychain := thr.GetKeychain()
+	accountData := &account.AccountData{
+		Identity: keychain.GetIdentity("A"),
+		SignKey:  keychain.SigningKeys["A"],
+		EncKey:   keychain.EncryptionKeys["A"],
+	}
+	listener := &mockListener{}
+	tree, err := BuildACLTree(thr, accountData, listener)
+	if err != nil {
+		t.Fatalf("should Build acl ACLState without err: %v", err)
+	}
+	rawChanges := thr.GetUpdates("rebuild")
+	var changes []*Change
+	for _, ch := range rawChanges {
+		newCh, err := NewFromRawChange(ch)
+		if err != nil {
+			t.Fatalf("should be able to create change from raw: %v", err)
+		}
+		changes = append(changes, newCh)
+	}
+
+	res, err := tree.AddChanges(changes...)
+	assert.Equal(t, res.Summary, AddResultSummaryRebuild)
+
+	aclState := tree.ACLState()
+	aId := keychain.GeneratedIdentities["A"]
+	bId := keychain.GeneratedIdentities["B"]
+	cId := keychain.GeneratedIdentities["C"]
+	dId := keychain.GeneratedIdentities["D"]
+
+	assert.Equal(t, aclState.identity, aId)
+	assert.Equal(t, aclState.userStates[aId].Permissions, pb.ACLChange_Admin)
+	assert.Equal(t, aclState.userStates[bId].Permissions, pb.ACLChange_Writer)
+	assert.Equal(t, aclState.userStates[cId].Permissions, pb.ACLChange_Reader)
+	assert.Equal(t, aclState.userStates[dId].Permissions, pb.ACLChange_Writer)
+
+	var changeIds []string
+
+	tree.Iterate(func(c *Change) (isContinue bool) {
+		changeIds = append(changeIds, c.Id)
+		return true
+	})
+	assert.Equal(t, changeIds, []string{"A.1.1", "A.1.2", "B.1.1", "B.1.2", "A.1.4"})
+}
+
+func TestACLTree_UserRemoveBuild(t *testing.T) {
+	thr, err := threadbuilder.NewThreadBuilderWithTestName("userremoveexample.yml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	keychain := thr.GetKeychain()
+	accountData := &account.AccountData{
+		Identity: keychain.GetIdentity("A"),
+		SignKey:  keychain.SigningKeys["A"],
+		EncKey:   keychain.EncryptionKeys["A"],
+	}
+	listener := &mockListener{}
+	tree, err := BuildACLTree(thr, accountData, listener)
+	if err != nil {
+		t.Fatalf("should Build acl ACLState without err: %v", err)
+	}
+	aclState := tree.ACLState()
+	aId := keychain.GeneratedIdentities["A"]
+
+	assert.Equal(t, aclState.identity, aId)
+	assert.Equal(t, aclState.userStates[aId].Permissions, pb.ACLChange_Admin)
+
+	var changeIds []string
+	tree.Iterate(func(c *Change) (isContinue bool) {
+		changeIds = append(changeIds, c.Id)
+		return true
+	})
+	assert.Equal(t, changeIds, []string{"A.1.1", "A.1.2", "B.1.1", "A.1.3", "A.1.4"})
+}
+
+func TestACLTree_UserRemoveBeforeBuild(t *testing.T) {
+	thr, err := threadbuilder.NewThreadBuilderWithTestName("userremovebeforeexample.yml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	keychain := thr.GetKeychain()
+	accountData := &account.AccountData{
+		Identity: keychain.GetIdentity("A"),
+		SignKey:  keychain.SigningKeys["A"],
+		EncKey:   keychain.EncryptionKeys["A"],
+	}
+	listener := &mockListener{}
+	tree, err := BuildACLTree(thr, accountData, listener)
+	if err != nil {
+		t.Fatalf("should Build acl ACLState without err: %v", err)
+	}
+	aclState := tree.ACLState()
+	for _, s := range []string{"A", "C", "E"} {
+		assert.Equal(t, aclState.userStates[keychain.GetIdentity(s)].Permissions, pb.ACLChange_Admin)
+	}
+	assert.Equal(t, aclState.identity, keychain.GetIdentity("A"))
+	assert.Nil(t, aclState.userStates[keychain.GetIdentity("B")])
+
+	var changeIds []string
+	tree.Iterate(func(c *Change) (isContinue bool) {
+		changeIds = append(changeIds, c.Id)
+		return true
+	})
+	assert.Equal(t, changeIds, []string{"A.1.1", "B.1.1", "A.1.2", "A.1.3"})
+}
+
+func TestACLTree_InvalidSnapshotBuild(t *testing.T) {
+	thr, err := threadbuilder.NewThreadBuilderWithTestName("invalidsnapshotexample.yml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	keychain := thr.GetKeychain()
+	accountData := &account.AccountData{
+		Identity: keychain.GetIdentity("A"),
+		SignKey:  keychain.SigningKeys["A"],
+		EncKey:   keychain.EncryptionKeys["A"],
+	}
+	listener := &mockListener{}
+	tree, err := BuildACLTree(thr, accountData, listener)
+	if err != nil {
+		t.Fatalf("should Build acl ACLState without err: %v", err)
+	}
+	aclState := tree.ACLState()
+	for _, s := range []string{"A", "B", "C", "D", "E", "F"} {
+		assert.Equal(t, aclState.userStates[keychain.GetIdentity(s)].Permissions, pb.ACLChange_Admin)
+	}
+	assert.Equal(t, aclState.identity, keychain.GetIdentity("A"))
+
+	var changeIds []string
+	tree.Iterate(func(c *Change) (isContinue bool) {
+		changeIds = append(changeIds, c.Id)
+		return true
+	})
+	assert.Equal(t, []string{"A.1.1", "B.1.1", "A.1.2", "A.1.3", "B.1.2"}, changeIds)
+}
+
+func TestACLTree_ValidSnapshotBuild(t *testing.T) {
+	thr, err := threadbuilder.NewThreadBuilderWithTestName("validsnapshotexample.yml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	keychain := thr.GetKeychain()
+	accountData := &account.AccountData{
+		Identity: keychain.GetIdentity("A"),
+		SignKey:  keychain.SigningKeys["A"],
+		EncKey:   keychain.EncryptionKeys["A"],
+	}
+	listener := &mockListener{}
+	tree, err := BuildACLTree(thr, accountData, listener)
+	if err != nil {
+		t.Fatalf("should Build acl ACLState without err: %v", err)
+	}
+	aclState := tree.ACLState()
+	for _, s := range []string{"A", "B", "C", "D", "E", "F"} {
+		assert.Equal(t, aclState.userStates[keychain.GetIdentity(s)].Permissions, pb.ACLChange_Admin)
+	}
+	assert.Equal(t, aclState.identity, keychain.GetIdentity("A"))
+
+	var changeIds []string
+	tree.Iterate(func(c *Change) (isContinue bool) {
+		changeIds = append(changeIds, c.Id)
+		return true
+	})
+	assert.Equal(t, []string{"A.1.2", "A.1.3", "B.1.2"}, changeIds)
+}

acltree/acltreebuilder.go

@@ -0,0 +1,142 @@
+package acltree
+
+import (
+	"fmt"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/thread"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice"
+)
+
+type aclTreeBuilder struct {
+	cache                map[string]*Change
+	identityKeys         map[string]keys.SigningPubKey
+	signingPubKeyDecoder keys.SigningPubKeyDecoder
+	tree                 *Tree
+	thread               thread.Thread
+
+	*changeLoader
+}
+
+func newACLTreeBuilder(t thread.Thread, decoder keys.SigningPubKeyDecoder) *aclTreeBuilder {
+	return &aclTreeBuilder{
+		signingPubKeyDecoder: decoder,
+		thread:               t,
+		changeLoader: newChangeLoader(
+			t,
+			decoder,
+			NewACLChange),
+	}
+}
+
+func (tb *aclTreeBuilder) Init() {
+	tb.cache = make(map[string]*Change)
+	tb.identityKeys = make(map[string]keys.SigningPubKey)
+	tb.tree = &Tree{}
+	tb.changeLoader.Init(tb.cache, tb.identityKeys)
+}
+
+func (tb *aclTreeBuilder) Build() (*Tree, error) {
+	var headsAndOrphans []string
+	headsAndOrphans = append(headsAndOrphans, tb.thread.Orphans()...)
+	headsAndOrphans = append(headsAndOrphans, tb.thread.Heads()...)
+	aclHeads, err := tb.getACLHeads(headsAndOrphans)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if err = tb.buildTreeFromStart(aclHeads); err != nil {
+		return nil, fmt.Errorf("buildTree error: %v", err)
+	}
+	tb.cache = nil
+
+	return tb.tree, nil
+}
+
+func (tb *aclTreeBuilder) buildTreeFromStart(heads []string) (err error) {
+	changes, root, err := tb.dfsFromStart(heads)
+	if err != nil {
+		return err
+	}
+
+	tb.tree.AddFast(root)
+	tb.tree.AddFast(changes...)
+	return
+}
+
+func (tb *aclTreeBuilder) dfsFromStart(heads []string) (buf []*Change, root *Change, err error) {
+	var possibleRoots []*Change
+	stack := make([]string, len(heads), len(heads)*2)
+	copy(stack, heads)
+
+	buf = make([]*Change, 0, len(stack)*2)
+	uniqMap := make(map[string]struct{})
+	for len(stack) > 0 {
+		id := stack[len(stack)-1]
+		stack = stack[:len(stack)-1]
+		if _, exists := uniqMap[id]; exists {
+			continue
+		}
+
+		ch, err := tb.loadChange(id)
+		if err != nil {
+			continue
+		}
+
+		uniqMap[id] = struct{}{}
+		buf = append(buf, ch)
+
+		for _, prev := range ch.PreviousIds {
+			stack = append(stack, prev)
+		}
+		if len(ch.PreviousIds) == 0 {
+			possibleRoots = append(possibleRoots, ch)
+		}
+	}
+	header := tb.thread.Header()
+	for _, r := range possibleRoots {
+		if r.Id == header.FirstChangeId {
+			return buf, r, nil
+		}
+	}
+
+	return nil, nil, fmt.Errorf("could not find root change")
+}
+
+func (tb *aclTreeBuilder) getACLHeads(heads []string) (aclTreeHeads []string, err error) {
+	for _, head := range heads {
+		if slice.FindPos(aclTreeHeads, head) != -1 { // do not scan known heads
+			continue
+		}
+		precedingHeads, err := tb.getPrecedingACLHeads(head)
+		if err != nil {
+			continue
+		}
+
+		for _, aclHead := range precedingHeads {
+			if slice.FindPos(aclTreeHeads, aclHead) != -1 {
+				continue
+			}
+			aclTreeHeads = append(aclTreeHeads, aclHead)
+		}
+	}
+
+	if len(aclTreeHeads) == 0 {
+		return nil, fmt.Errorf("no usable ACL heads in thread")
+	}
+	return aclTreeHeads, nil
+}
+
+func (tb *aclTreeBuilder) getPrecedingACLHeads(head string) ([]string, error) {
+	headChange, err := tb.loadChange(head)
+	if err != nil {
+		return nil, err
+	}
+
+	if headChange.Content.GetAclData() != nil {
+		return []string{head}, nil
+	} else {
+		return headChange.PreviousIds, nil
+	}
+}

acltree/change.go

@@ -0,0 +1,98 @@
+package acltree
+
+import (
+	"fmt"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/thread"
+	"github.com/gogo/protobuf/proto"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/aclchanges/pb"
+	"github.com/textileio/go-threads/crypto/symmetric"
+)
+
+type ChangeContent struct {
+	ChangesData proto.Marshaler
+	ACLData     *pb.ACLChangeACLData
+	Id          string // TODO: this is just for testing, because id should be created automatically from content
+}
+
+// Change is an abstract type for all types of changes
+type Change struct {
+	Next                    []*Change
+	Unattached              []*Change
+	PreviousIds             []string
+	Id                      string
+	SnapshotId              string
+	IsSnapshot              bool
+	DecryptedDocumentChange []byte
+
+	Content *pb.ACLChange
+	Sign    []byte
+}
+
+func (ch *Change) DecryptContents(key *symmetric.Key) error {
+	if ch.Content.ChangesData == nil {
+		return nil
+	}
+
+	decrypted, err := key.Decrypt(ch.Content.ChangesData)
+	if err != nil {
+		return fmt.Errorf("failed to decrypt changes data: %w", err)
+	}
+
+	ch.DecryptedDocumentChange = decrypted
+	return nil
+}
+
+func (ch *Change) IsACLChange() bool {
+	return ch.Content.GetAclData() != nil
+}
+
+func NewFromRawChange(rawChange *thread.RawChange) (*Change, error) {
+	unmarshalled := &pb.ACLChange{}
+	err := proto.Unmarshal(rawChange.Payload, unmarshalled)
+	if err != nil {
+		return nil, err
+	}
+
+	ch := NewChange(rawChange.Id, unmarshalled)
+	ch.Sign = rawChange.Signature
+	return ch, nil
+}
+
+func NewChange(id string, ch *pb.ACLChange) *Change {
+	return &Change{
+		Next:        nil,
+		PreviousIds: ch.TreeHeadIds,
+		Id:          id,
+		Content:     ch,
+		SnapshotId:  ch.SnapshotBaseId,
+		IsSnapshot:  ch.GetAclData().GetAclSnapshot() != nil,
+	}
+}
+
+func NewACLChange(id string, ch *pb.ACLChange) *Change {
+	return &Change{
+		Next:        nil,
+		PreviousIds: ch.AclHeadIds,
+		Id:          id,
+		Content:     ch,
+		SnapshotId:  ch.SnapshotBaseId,
+		IsSnapshot:  ch.GetAclData().GetAclSnapshot() != nil,
+	}
+}
+
+func (ch *Change) ProtoChange() *pb.ACLChange {
+	return ch.Content
+}
+
+func (ch *Change) DecryptedChangeContent() []byte {
+	return ch.DecryptedDocumentChange
+}
+
+func (ch *Change) Signature() []byte {
+	return ch.Sign
+}
+
+func (ch *Change) CID() string {
+	return ch.Id
+}

acltree/changebuilder.go

@@ -0,0 +1,163 @@
+package acltree
+
+import (
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/account"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/aclchanges/pb"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/cid"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
+	"github.com/gogo/protobuf/proto"
+	"github.com/textileio/go-threads/crypto/symmetric"
+	"hash/fnv"
+	"time"
+)
+
+type MarshalledChange = []byte
+
+type ACLChangeBuilder interface {
+	UserAdd(identity string, encryptionKey keys.EncryptionPubKey, permissions pb.ACLChangeUserPermissions) error
+	AddId(id string)      // TODO: this is only for testing
+	SetMakeSnapshot(bool) // TODO: who should decide this? probably ACLTree so we can delete it
+}
+
+type ChangeBuilder interface {
+	ACLChangeBuilder
+	AddChangeContent(marshaler proto.Marshaler) // user code should be responsible for making regular snapshots
+}
+
+type changeBuilder struct {
+	aclState *ACLState
+	tree     *Tree
+	acc      *account.AccountData
+
+	aclData       *pb.ACLChangeACLData
+	changeContent proto.Marshaler
+	id            string
+	makeSnapshot  bool
+	readKey       *symmetric.Key
+	readKeyHash   uint64
+}
+
+func newChangeBuilder() *changeBuilder {
+	return &changeBuilder{}
+}
+
+func (c *changeBuilder) Init(state *ACLState, tree *Tree, acc *account.AccountData) {
+	c.aclState = state
+	c.tree = tree
+	c.acc = acc
+
+	c.aclData = &pb.ACLChangeACLData{}
+	// setting read key for further encryption etc
+	if state.currentReadKeyHash == 0 {
+		c.readKey, _ = symmetric.NewRandom()
+
+		hasher := fnv.New64()
+		hasher.Write(c.readKey.Bytes())
+		c.readKeyHash = hasher.Sum64()
+	} else {
+		c.readKey = c.aclState.userReadKeys[c.aclState.currentReadKeyHash]
+		c.readKeyHash = c.aclState.currentReadKeyHash
+	}
+}
+
+func (c *changeBuilder) AddId(id string) {
+	c.id = id
+}
+
+func (c *changeBuilder) SetMakeSnapshot(b bool) {
+	c.makeSnapshot = b
+}
+
+func (c *changeBuilder) UserAdd(identity string, encryptionKey keys.EncryptionPubKey, permissions pb.ACLChangeUserPermissions) error {
+	var allKeys []*symmetric.Key
+	if c.aclState.currentReadKeyHash != 0 {
+		for _, key := range c.aclState.userReadKeys {
+			allKeys = append(allKeys, key)
+		}
+	} else {
+		allKeys = append(allKeys, c.readKey)
+	}
+
+	var encryptedKeys [][]byte
+	for _, k := range allKeys {
+		res, err := encryptionKey.Encrypt(k.Bytes())
+		if err != nil {
+			return err
+		}
+
+		encryptedKeys = append(encryptedKeys, res)
+	}
+	rawKey, err := encryptionKey.Raw()
+	if err != nil {
+		return err
+	}
+	ch := &pb.ACLChangeACLContentValue{
+		Value: &pb.ACLChangeACLContentValueValueOfUserAdd{
+			UserAdd: &pb.ACLChangeUserAdd{
+				Identity:          identity,
+				EncryptionKey:     rawKey,
+				EncryptedReadKeys: encryptedKeys,
+				Permissions:       permissions,
+			},
+		},
+	}
+	c.aclData.AclContent = append(c.aclData.AclContent, ch)
+	return nil
+}
+
+func (c *changeBuilder) BuildAndApply() (*Change, []byte, error) {
+	aclChange := &pb.ACLChange{
+		TreeHeadIds:        c.tree.Heads(),
+		AclHeadIds:         c.tree.ACLHeads(),
+		SnapshotBaseId:     c.tree.RootId(),
+		AclData:            c.aclData,
+		CurrentReadKeyHash: c.readKeyHash,
+		Timestamp:          int64(time.Now().Nanosecond()),
+		Identity:           c.acc.Identity,
+	}
+	err := c.aclState.applyChange(aclChange)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if c.makeSnapshot {
+		c.aclData.AclSnapshot = c.aclState.makeSnapshot()
+	}
+
+	var marshalled []byte
+	if c.changeContent != nil {
+		marshalled, err = c.changeContent.Marshal()
+		if err != nil {
+			return nil, nil, err
+		}
+
+		encrypted, err := c.aclState.userReadKeys[c.aclState.currentReadKeyHash].
+			Encrypt(marshalled)
+		if err != nil {
+			return nil, nil, err
+		}
+		aclChange.ChangesData = encrypted
+	}
+
+	fullMarshalledChange, err := proto.Marshal(aclChange)
+	if err != nil {
+		return nil, nil, err
+	}
+	signature, err := c.acc.SignKey.Sign(fullMarshalledChange)
+	if err != nil {
+		return nil, nil, err
+	}
+	id, err := cid.NewCIDFromBytes(fullMarshalledChange)
+	if err != nil {
+		return nil, nil, err
+	}
+	ch := NewChange(id, aclChange)
+	ch.DecryptedDocumentChange = marshalled
+	ch.Sign = signature
+
+	return ch, fullMarshalledChange, nil
+}
+
+func (c *changeBuilder) AddChangeContent(marshaler proto.Marshaler) {
+	c.changeContent = marshaler
+}

acltree/changeloader.go

@@ -0,0 +1,93 @@
+package acltree
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/aclchanges/pb"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/thread"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
+	"github.com/gogo/protobuf/proto"
+)
+
+type changeLoader struct {
+	cache                map[string]*Change
+	identityKeys         map[string]keys.SigningPubKey
+	signingPubKeyDecoder keys.SigningPubKeyDecoder
+	thread               thread.Thread
+	changeCreator        func(id string, ch *pb.ACLChange) *Change
+}
+
+func newChangeLoader(
+	thread thread.Thread,
+	signingPubKeyDecoder keys.SigningPubKeyDecoder,
+	changeCreator func(id string, ch *pb.ACLChange) *Change) *changeLoader {
+	return &changeLoader{
+		signingPubKeyDecoder: signingPubKeyDecoder,
+		thread:               thread,
+		changeCreator:        changeCreator,
+	}
+}
+
+func (c *changeLoader) Init(cache map[string]*Change,
+	identityKeys map[string]keys.SigningPubKey) {
+	c.cache = cache
+	c.identityKeys = identityKeys
+}
+
+func (c *changeLoader) loadChange(id string) (ch *Change, err error) {
+	if ch, ok := c.cache[id]; ok {
+		return ch, nil
+	}
+
+	// TODO: Add virtual changes logic
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+	defer cancel()
+
+	change, err := c.thread.GetChange(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	aclChange, err := c.makeVerifiedACLChange(change)
+	if err != nil {
+		return nil, err
+	}
+
+	ch = c.changeCreator(id, aclChange)
+	c.cache[id] = ch
+
+	return ch, nil
+}
+
+func (c *changeLoader) verify(identity string, payload, signature []byte) (isVerified bool, err error) {
+	identityKey, exists := c.identityKeys[identity]
+	if !exists {
+		identityKey, err = c.signingPubKeyDecoder.DecodeFromString(identity)
+		if err != nil {
+			return
+		}
+		c.identityKeys[identity] = identityKey
+	}
+	return identityKey.Verify(payload, signature)
+}
+
+func (c *changeLoader) makeVerifiedACLChange(change *thread.RawChange) (aclChange *pb.ACLChange, err error) {
+	aclChange = new(pb.ACLChange)
+
+	// TODO: think what should we do with such cases, because this can be used by attacker to break our Tree
+	if err = proto.Unmarshal(change.Payload, aclChange); err != nil {
+		return
+	}
+	var verified bool
+	verified, err = c.verify(aclChange.Identity, change.Payload, change.Signature)
+	if err != nil {
+		return
+	}
+	if !verified {
+		err = fmt.Errorf("the signature of the payload cannot be verified")
+		return
+	}
+	return
+}

acltree/snapshotvalidator.go

@@ -0,0 +1,50 @@
+package acltree
+
+import (
+	"fmt"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/account"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
+)
+
+type snapshotValidator struct {
+	aclTree      *Tree
+	identity     string
+	key          keys.EncryptionPrivKey
+	decoder      keys.SigningPubKeyDecoder
+	stateBuilder *aclStateBuilder
+}
+
+func newSnapshotValidator(
+	decoder keys.SigningPubKeyDecoder,
+	accountData *account.AccountData) *snapshotValidator {
+	return &snapshotValidator{
+		identity:     accountData.Identity,
+		key:          accountData.EncKey,
+		decoder:      decoder,
+		stateBuilder: newACLStateBuilder(decoder, accountData),
+	}
+}
+
+func (s *snapshotValidator) Init(aclTree *Tree) error {
+	s.aclTree = aclTree
+	return s.stateBuilder.Init(aclTree)
+}
+
+func (s *snapshotValidator) ValidateSnapshot(ch *Change) (bool, error) {
+	st, found, err := s.stateBuilder.BuildBefore(ch.Id)
+	if err != nil {
+		return false, err
+	}
+
+	if !found {
+		return false, fmt.Errorf("didn't find snapshot in ACL Tree")
+	}
+
+	otherSt, err := newACLStateFromSnapshotChange(ch.Content, s.identity, s.key, s.decoder)
+	if err != nil {
+		return false, err
+	}
+
+	return st.equal(otherSt), nil
+}

acltree/threadutility.go

@@ -0,0 +1,42 @@
+package acltree
+
+import (
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/account"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/thread"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
+)
+
+func BuildThreadWithACL(
+	acc *account.AccountData,
+	build func(builder ChangeBuilder) error,
+	create func(change *thread.RawChange) (thread.Thread, error)) (thread.Thread, error) {
+	bld := newChangeBuilder()
+	bld.Init(
+		newACLState(acc.Identity, acc.EncKey, keys.NewEd25519Decoder()),
+		&Tree{},
+		acc)
+	err := build(bld)
+	if err != nil {
+		return nil, err
+	}
+	bld.SetMakeSnapshot(true)
+
+	change, payload, err := bld.BuildAndApply()
+	if err != nil {
+		return nil, err
+	}
+
+	rawChange := &thread.RawChange{
+		Payload:   payload,
+		Signature: change.Signature(),
+		Id:        change.CID(),
+	}
+
+	thr, err := create(rawChange)
+	if err != nil {
+		return nil, err
+	}
+
+	thr.SetHeads([]string{change.CID()})
+	return thr, nil
+}

acltree/threadutility_test.go

@@ -0,0 +1,51 @@
+package acltree
+
+import (
+	"context"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/account"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/aclchanges/pb"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/testutils/threadbuilder"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/thread"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestBuildThreadWithACL(t *testing.T) {
+	keychain := threadbuilder.NewKeychain()
+	keychain.AddSigningKey("A")
+	keychain.AddEncryptionKey("A")
+	data := &account.AccountData{
+		Identity: keychain.GetIdentity("A"),
+		SignKey:  keychain.SigningKeys["A"],
+		EncKey:   keychain.EncryptionKeys["A"],
+	}
+	thr, err := BuildThreadWithACL(
+		data,
+		func(builder ChangeBuilder) error {
+			return builder.UserAdd(
+				keychain.GetIdentity("A"),
+				keychain.EncryptionKeys["A"].GetPublic(),
+				pb.ACLChange_Admin)
+		},
+		thread.NewInMemoryThread)
+	if err != nil {
+		t.Fatalf("build should not return error")
+	}
+	if len(thr.Heads()) == 0 {
+		t.Fatalf("thread should have non-empty heads")
+	}
+	if thr.Header() == nil {
+		t.Fatalf("thread should have non-empty header")
+	}
+	assert.Equal(t, thr.Heads()[0], thr.Header().FirstChangeId)
+	assert.NotEmpty(t, thr.ID())
+	ch, err := thr.GetChange(context.Background(), thr.Header().FirstChangeId)
+	if err != nil {
+		t.Fatalf("get change should not return error: %v", err)
+	}
+
+	_, err = NewFromRawChange(ch)
+	if err != nil {
+		t.Fatalf("we should be able to unmarshall change: %v", err)
+	}
+}

acltree/tree.go

@@ -0,0 +1,416 @@
+package acltree
+
+import (
+	"bytes"
+	"crypto/md5"
+	"fmt"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice"
+	"sort"
+)
+
+type Mode int
+
+const (
+	Append Mode = iota
+	Rebuild
+	Nothing
+)
+
+// TODO: consider abstracting into separate package with iterator, remove
+type Tree struct {
+	root        *Change
+	headIds     []string
+	metaHeadIds []string
+	attached    map[string]*Change
+	unAttached  map[string]*Change
+	// missed id -> list of dependency ids
+	waitList       map[string][]string
+	invalidChanges map[string]struct{}
+
+	// bufs
+	iterCompBuf []*Change
+	iterQueue   []*Change
+
+	duplicateEvents int
+}
+
+func (t *Tree) GetUnattachedChanges(changes ...*Change) []*Change {
+	return nil
+}
+
+func (t *Tree) RootId() string {
+	if t.root != nil {
+		return t.root.Id
+	}
+	return ""
+}
+
+func (t *Tree) Root() *Change {
+	return t.root
+}
+
+func (t *Tree) AddFast(changes ...*Change) {
+	for _, c := range changes {
+		// ignore existing
+		if _, ok := t.attached[c.Id]; ok {
+			continue
+		} else if _, ok := t.unAttached[c.Id]; ok {
+			continue
+		}
+		t.add(c)
+	}
+	t.updateHeads()
+}
+
+func (t *Tree) Add(changes ...*Change) (mode Mode) {
+	var beforeHeadIds = t.headIds
+	var attached bool
+	var empty = t.Len() == 0
+	for _, c := range changes {
+		// ignore existing
+		if _, ok := t.attached[c.Id]; ok {
+			continue
+		} else if _, ok := t.unAttached[c.Id]; ok {
+			continue
+		}
+		if t.add(c) {
+			attached = true
+		}
+	}
+	if !attached {
+		return Nothing
+	}
+	t.updateHeads()
+	if empty {
+		return Rebuild
+	}
+	for _, hid := range beforeHeadIds {
+		for _, newCh := range changes {
+			if _, ok := t.attached[newCh.Id]; ok {
+				if !t.after(newCh.Id, hid) {
+					return Rebuild
+				}
+			}
+		}
+	}
+	return Append
+}
+
+func (t *Tree) RemoveInvalidChange(id string) {
+	stack := []string{id}
+	// removing all children of this id (either next or unattached)
+	for len(stack) > 0 {
+		var exists bool
+		top := stack[len(stack)-1]
+		stack = stack[:len(stack)-1]
+
+		if _, exists = t.invalidChanges[top]; exists {
+			continue
+		}
+
+		var rem *Change
+		t.invalidChanges[top] = struct{}{}
+		if rem, exists = t.unAttached[top]; exists {
+			delete(t.unAttached, top)
+		} else if rem, exists = t.attached[top]; exists {
+			// remove from all prev changes
+			for _, id := range rem.PreviousIds {
+				prev, exists := t.attached[id]
+				if !exists {
+					continue
+				}
+				for i, next := range prev.Next {
+					if next.Id == top {
+						prev.Next[i] = nil
+						prev.Next = append(prev.Next[:i], prev.Next[i+1:]...)
+						break
+					}
+				}
+			}
+			delete(t.attached, top)
+		}
+		for _, el := range rem.Unattached {
+			stack = append(stack, el.Id)
+		}
+		for _, el := range rem.Next {
+			stack = append(stack, el.Id)
+		}
+	}
+	t.updateHeads()
+}
+
+func (t *Tree) add(c *Change) (attached bool) {
+	if c == nil {
+		return false
+	}
+	if _, exists := t.invalidChanges[c.Id]; exists {
+		return false
+	}
+
+	if t.root == nil { // first element
+		t.root = c
+		t.attached = map[string]*Change{
+			c.Id: c,
+		}
+		t.unAttached = make(map[string]*Change)
+		t.waitList = make(map[string][]string)
+		t.invalidChanges = make(map[string]struct{})
+		return true
+	}
+	if len(c.PreviousIds) > 1 {
+		sort.Strings(c.PreviousIds)
+	}
+	// attaching only if all prev ids are attached
+	attached = true
+	for _, pid := range c.PreviousIds {
+		if prev, ok := t.attached[pid]; ok {
+			prev.Unattached = append(prev.Unattached, c)
+			continue
+		}
+		attached = false
+		if prev, ok := t.unAttached[pid]; ok {
+			prev.Unattached = append(prev.Unattached, c)
+			continue
+		}
+		wl := t.waitList[pid]
+		wl = append(wl, c.Id)
+		t.waitList[pid] = wl
+	}
+	if attached {
+		t.attach(c, true)
+	} else {
+		// clearing wait list
+		for _, wid := range t.waitList[c.Id] {
+			c.Unattached = append(c.Unattached, t.unAttached[wid])
+		}
+		delete(t.waitList, c.Id)
+		t.unAttached[c.Id] = c
+	}
+	return
+}
+
+func (t *Tree) canAttach(c *Change) (attach bool) {
+	if c == nil {
+		return false
+	}
+	attach = true
+	for _, id := range c.PreviousIds {
+		if _, exists := t.attached[id]; !exists {
+			attach = false
+		}
+	}
+	return
+}
+
+func (t *Tree) attach(c *Change, newEl bool) {
+	t.attached[c.Id] = c
+	if !newEl {
+		delete(t.unAttached, c.Id)
+	}
+
+	// add next to all prev changes
+	for _, id := range c.PreviousIds {
+		// prev id must be attached if we attach this id
+		prev := t.attached[id]
+		prev.Next = append(prev.Next, c)
+		if len(prev.Next) > 1 {
+			sort.Sort(sortChanges(prev.Next))
+		}
+		for i, next := range prev.Unattached {
+			if next.Id == c.Id {
+				prev.Unattached[i] = nil
+				prev.Unattached = append(prev.Unattached[:i], prev.Unattached[i+1:]...)
+				break
+			}
+		}
+	}
+
+	// clearing wait list
+	if waitIds, ok := t.waitList[c.Id]; ok {
+		for _, wid := range waitIds {
+			next := t.unAttached[wid]
+			if t.canAttach(next) {
+				t.attach(next, false)
+			}
+		}
+		delete(t.waitList, c.Id)
+	}
+
+	for _, next := range c.Unattached {
+		if t.canAttach(next) {
+			t.attach(next, false)
+		}
+	}
+}
+
+func (t *Tree) after(id1, id2 string) (found bool) {
+	t.iterate(t.attached[id2], func(c *Change) (isContinue bool) {
+		if c.Id == id1 {
+			found = true
+			return false
+		}
+		return true
+	})
+	return
+}
+
+func (t *Tree) dfs(startChange string) (uniqMap map[string]*Change) {
+	stack := make([]*Change, 0, 10)
+	stack = append(stack, t.attached[startChange])
+	uniqMap = map[string]*Change{}
+
+	for len(stack) > 0 {
+		ch := stack[len(stack)-1]
+		stack = stack[:len(stack)-1]
+		if _, exists := uniqMap[ch.Id]; exists {
+			continue
+		}
+
+		uniqMap[ch.Id] = ch
+
+		for _, prev := range ch.PreviousIds {
+			stack = append(stack, t.attached[prev])
+		}
+	}
+	return uniqMap
+}
+
+func (t *Tree) updateHeads() {
+	var newHeadIds, newMetaHeadIds []string
+	t.iterate(t.root, func(c *Change) (isContinue bool) {
+		if len(c.Next) == 0 {
+			newHeadIds = append(newHeadIds, c.Id)
+		}
+		return true
+	})
+	t.headIds = newHeadIds
+	t.metaHeadIds = newMetaHeadIds
+	sort.Strings(t.headIds)
+	sort.Strings(t.metaHeadIds)
+}
+
+func (t *Tree) ACLHeads() []string {
+	var aclTreeHeads []string
+	for _, head := range t.Heads() {
+		if slice.FindPos(aclTreeHeads, head) != -1 { // do not scan known heads
+			continue
+		}
+		precedingHeads := t.getPrecedingACLHeads(head)
+
+		for _, aclHead := range precedingHeads {
+			if slice.FindPos(aclTreeHeads, aclHead) != -1 {
+				continue
+			}
+			aclTreeHeads = append(aclTreeHeads, aclHead)
+		}
+	}
+	return aclTreeHeads
+}
+
+func (t *Tree) getPrecedingACLHeads(head string) []string {
+	headChange := t.attached[head]
+
+	if headChange.Content.GetAclData() != nil {
+		return []string{head}
+	} else {
+		return headChange.Content.AclHeadIds
+	}
+}
+
+func (t *Tree) iterate(start *Change, f func(c *Change) (isContinue bool)) {
+	it := newIterator()
+	defer freeIterator(it)
+	it.iterate(start, f)
+}
+
+func (t *Tree) iterateSkip(start *Change, skipBefore *Change, f func(c *Change) (isContinue bool)) {
+	it := newIterator()
+	defer freeIterator(it)
+	it.iterateSkip(start, skipBefore, f)
+}
+
+func (t *Tree) IterateSkip(startId string, skipBeforeId string, f func(c *Change) (isContinue bool)) {
+	it := newIterator()
+	defer freeIterator(it)
+	it.iterateSkip(t.attached[startId], t.attached[skipBeforeId], f)
+}
+
+func (t *Tree) Iterate(startId string, f func(c *Change) (isContinue bool)) {
+	t.iterate(t.attached[startId], f)
+}
+
+func (t *Tree) IterateBranching(startId string, f func(c *Change, branchLevel int) (isContinue bool)) {
+	// branchLevel indicates the number of parallel branches
+	var bc int
+	t.iterate(t.attached[startId], func(c *Change) (isContinue bool) {
+		if pl := len(c.PreviousIds); pl > 1 {
+			bc -= pl - 1
+		}
+		bl := bc
+		if nl := len(c.Next); nl > 1 {
+			bc += nl - 1
+		}
+		return f(c, bl)
+	})
+}
+
+func (t *Tree) Hash() string {
+	h := md5.New()
+	n := 0
+	t.iterate(t.root, func(c *Change) (isContinue bool) {
+		n++
+		fmt.Fprintf(h, "-%s", c.Id)
+		return true
+	})
+	return fmt.Sprintf("%d-%x", n, h.Sum(nil))
+}
+
+func (t *Tree) GetDuplicateEvents() int {
+	return t.duplicateEvents
+}
+
+func (t *Tree) ResetDuplicateEvents() {
+	t.duplicateEvents = 0
+}
+
+func (t *Tree) Len() int {
+	return len(t.attached)
+}
+
+func (t *Tree) Heads() []string {
+	return t.headIds
+}
+
+func (t *Tree) String() string {
+	var buf = bytes.NewBuffer(nil)
+	t.Iterate(t.RootId(), func(c *Change) (isContinue bool) {
+		buf.WriteString(c.Id)
+		if len(c.Next) > 1 {
+			buf.WriteString("-<")
+		} else if len(c.Next) > 0 {
+			buf.WriteString("->")
+		} else {
+			buf.WriteString("-|")
+		}
+		return true
+	})
+	return buf.String()
+}
+
+func (t *Tree) Get(id string) *Change {
+	return t.attached[id]
+}
+
+type sortChanges []*Change
+
+func (s sortChanges) Len() int {
+	return len(s)
+}
+
+func (s sortChanges) Less(i, j int) bool {
+	return s[i].Id < s[j].Id
+}
+
+func (s sortChanges) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}

acltree/treebuilder.go

@@ -0,0 +1,295 @@
+package acltree
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/thread"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
+	//"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/lib/logging"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice"
+	"github.com/prometheus/common/log"
+)
+
+var (
+	//log      = logging.Logger("anytype-data")
+	ErrEmpty = errors.New("logs empty")
+)
+
+type treeBuilder struct {
+	cache                map[string]*Change
+	identityKeys         map[string]keys.SigningPubKey
+	signingPubKeyDecoder keys.SigningPubKeyDecoder
+	tree                 *Tree
+	thread               thread.Thread
+
+	*changeLoader
+}
+
+func newTreeBuilder(t thread.Thread, decoder keys.SigningPubKeyDecoder) *treeBuilder {
+	return &treeBuilder{
+		signingPubKeyDecoder: decoder,
+		thread:               t,
+		changeLoader: newChangeLoader(
+			t,
+			decoder,
+			NewChange),
+	}
+}
+
+func (tb *treeBuilder) Init() {
+	tb.cache = make(map[string]*Change)
+	tb.identityKeys = make(map[string]keys.SigningPubKey)
+	tb.tree = &Tree{}
+	tb.changeLoader.Init(tb.cache, tb.identityKeys)
+}
+
+func (tb *treeBuilder) Build(fromStart bool) (*Tree, error) {
+	var headsAndOrphans []string
+	headsAndOrphans = append(headsAndOrphans, tb.thread.Orphans()...)
+	headsAndOrphans = append(headsAndOrphans, tb.thread.Heads()...)
+
+	if fromStart {
+		if err := tb.buildTreeFromStart(headsAndOrphans); err != nil {
+			return nil, fmt.Errorf("buildTree error: %v", err)
+		}
+	} else {
+		breakpoint, err := tb.findBreakpoint(headsAndOrphans)
+		if err != nil {
+			return nil, fmt.Errorf("findBreakpoint error: %v", err)
+		}
+
+		if err = tb.buildTree(headsAndOrphans, breakpoint); err != nil {
+			return nil, fmt.Errorf("buildTree error: %v", err)
+		}
+	}
+
+	tb.cache = nil
+
+	return tb.tree, nil
+}
+
+func (tb *treeBuilder) buildTreeFromStart(heads []string) (err error) {
+	changes, root, err := tb.dfsFromStart(heads)
+	if err != nil {
+		return err
+	}
+
+	tb.tree.AddFast(root)
+	tb.tree.AddFast(changes...)
+	return
+}
+
+func (tb *treeBuilder) dfsFromStart(heads []string) (buf []*Change, root *Change, err error) {
+	var possibleRoots []*Change
+	stack := make([]string, len(heads), len(heads)*2)
+	copy(stack, heads)
+
+	buf = make([]*Change, 0, len(stack)*2)
+	uniqMap := make(map[string]struct{})
+	for len(stack) > 0 {
+		id := stack[len(stack)-1]
+		stack = stack[:len(stack)-1]
+		if _, exists := uniqMap[id]; exists {
+			continue
+		}
+
+		ch, err := tb.loadChange(id)
+		if err != nil {
+			continue
+		}
+
+		uniqMap[id] = struct{}{}
+		buf = append(buf, ch)
+
+		for _, prev := range ch.PreviousIds {
+			stack = append(stack, prev)
+		}
+		if len(ch.PreviousIds) == 0 {
+			possibleRoots = append(possibleRoots, ch)
+		}
+	}
+	header := tb.thread.Header()
+	for _, r := range possibleRoots {
+		if r.Id == header.FirstChangeId {
+			return buf, r, nil
+		}
+	}
+
+	return nil, nil, fmt.Errorf("could not find root change")
+}
+
+func (tb *treeBuilder) buildTree(heads []string, breakpoint string) (err error) {
+	ch, err := tb.loadChange(breakpoint)
+	if err != nil {
+		return
+	}
+	tb.tree.AddFast(ch)
+	changes, err := tb.dfs(heads, breakpoint)
+
+	tb.tree.AddFast(changes...)
+	return
+}
+
+func (tb *treeBuilder) dfs(heads []string, breakpoint string) (buf []*Change, err error) {
+	stack := make([]string, len(heads), len(heads)*2)
+	copy(stack, heads)
+
+	buf = make([]*Change, 0, len(stack)*2)
+	uniqMap := map[string]struct{}{breakpoint: {}}
+	for len(stack) > 0 {
+		id := stack[len(stack)-1]
+		stack = stack[:len(stack)-1]
+		if _, exists := uniqMap[id]; exists {
+			continue
+		}
+
+		ch, err := tb.loadChange(id)
+		if err != nil {
+			continue
+		}
+
+		uniqMap[id] = struct{}{}
+		buf = append(buf, ch)
+
+		for _, prev := range ch.PreviousIds {
+			stack = append(stack, prev)
+		}
+	}
+	return buf, nil
+}
+
+func (tb *treeBuilder) findBreakpoint(heads []string) (breakpoint string, err error) {
+	var (
+		ch          *Change
+		snapshotIds []string
+	)
+	for _, head := range heads {
+		if ch, err = tb.loadChange(head); err != nil {
+			return
+		}
+		shId := ch.SnapshotId
+		if ch.IsSnapshot {
+			shId = ch.Id
+		}
+		if slice.FindPos(snapshotIds, shId) == -1 {
+			snapshotIds = append(snapshotIds, shId)
+		}
+	}
+	return tb.findCommonSnapshot(snapshotIds)
+}
+
+func (tb *treeBuilder) findCommonSnapshot(snapshotIds []string) (snapshotId string, err error) {
+	if len(snapshotIds) == 1 {
+		return snapshotIds[0], nil
+	} else if len(snapshotIds) == 0 {
+		return "", fmt.Errorf("snapshots not found")
+	}
+
+	for len(snapshotIds) > 1 {
+		l := len(snapshotIds)
+		shId, e := tb.findCommonForTwoSnapshots(snapshotIds[l-2], snapshotIds[l-1])
+		if e != nil {
+			return "", e
+		}
+		snapshotIds[l-2] = shId
+		snapshotIds = snapshotIds[:l-1]
+	}
+	return snapshotIds[0], nil
+}
+
+func (tb *treeBuilder) findCommonForTwoSnapshots(s1, s2 string) (s string, err error) {
+	// fast cases
+	if s1 == s2 {
+		return s1, nil
+	}
+	ch1, err := tb.loadChange(s1)
+	if err != nil {
+		return "", err
+	}
+	if ch1.SnapshotId == s2 {
+		return s2, nil
+	}
+	ch2, err := tb.loadChange(s2)
+	if err != nil {
+		return "", err
+	}
+	if ch2.SnapshotId == s1 {
+		return s1, nil
+	}
+	if ch1.SnapshotId == ch2.SnapshotId && ch1.SnapshotId != "" {
+		return ch1.SnapshotId, nil
+	}
+	// traverse
+	var t1 = make([]string, 0, 5)
+	var t2 = make([]string, 0, 5)
+	t1 = append(t1, ch1.Id, ch1.SnapshotId)
+	t2 = append(t2, ch2.Id, ch2.SnapshotId)
+	for {
+		lid1 := t1[len(t1)-1]
+		if lid1 != "" {
+			l1, e := tb.loadChange(lid1)
+			if e != nil {
+				return "", e
+			}
+			if l1.SnapshotId != "" {
+				if slice.FindPos(t2, l1.SnapshotId) != -1 {
+					return l1.SnapshotId, nil
+				}
+			}
+			t1 = append(t1, l1.SnapshotId)
+		}
+		lid2 := t2[len(t2)-1]
+		if lid2 != "" {
+			l2, e := tb.loadChange(t2[len(t2)-1])
+			if e != nil {
+				return "", e
+			}
+			if l2.SnapshotId != "" {
+				if slice.FindPos(t1, l2.SnapshotId) != -1 {
+					return l2.SnapshotId, nil
+				}
+			}
+			t2 = append(t2, l2.SnapshotId)
+		}
+		if lid1 == "" && lid2 == "" {
+			break
+		}
+	}
+
+	log.Warnf("changes build Tree: possible versions split")
+
+	// prefer not first snapshot
+	if len(ch1.PreviousIds) == 0 && len(ch2.PreviousIds) > 0 {
+		log.Warnf("changes build Tree: prefer %s(%d prevIds) over %s(%d prevIds)", s2, len(ch2.PreviousIds), s1, len(ch1.PreviousIds))
+		return s2, nil
+	} else if len(ch1.PreviousIds) > 0 && len(ch2.PreviousIds) == 0 {
+		log.Warnf("changes build Tree: prefer %s(%d prevIds) over %s(%d prevIds)", s1, len(ch1.PreviousIds), s2, len(ch2.PreviousIds))
+		return s1, nil
+	}
+
+	isEmptySnapshot := func(ch *Change) bool {
+		// TODO: add more sophisticated checks in Change for snapshots
+		return !ch.IsSnapshot
+	}
+
+	// TODO: can we even have empty snapshots?
+	// prefer not empty snapshot
+	if isEmptySnapshot(ch1) && !isEmptySnapshot(ch2) {
+		log.Warnf("changes build Tree: prefer %s(not empty) over %s(empty)", s2, s1)
+		return s2, nil
+	} else if isEmptySnapshot(ch2) && !isEmptySnapshot(ch1) {
+		log.Warnf("changes build Tree: prefer %s(not empty) over %s(empty)", s1, s2)
+		return s1, nil
+	}
+
+	// TODO: add virtual change mechanics
+	// unexpected behavior - just return lesser id
+	if s1 < s2 {
+		log.Warnf("changes build Tree: prefer %s (%s<%s)", s1, s1, s2)
+		return s1, nil
+	}
+	log.Warnf("changes build Tree: prefer %s (%s<%s)", s2, s2, s1)
+
+	return s2, nil
+}

acltree/treebuilder_test.go

@@ -0,0 +1,64 @@
+package acltree
+
+//func createTreeFromThread(t thread.Thread, fromStart bool) (*Tree, error) {
+//	treeBuilder := newTreeBuilder(t, keys.NewEd25519Decoder())
+//	treeBuilder.Init()
+//	return treeBuilder.Build(fromStart)
+//}
+//
+//func TestACLTreeBuilder_UserJoinCorrectHeadsAndLen(t *testing.T) {
+//	thread, err := threadbuilder.NewThreadBuilderWithTestName("threadbuilder/userjoinexample.yml")
+//	if err != nil {
+//		t.Fatal(err)
+//	}
+//
+//	res, err := createTreeFromThread(thread)
+//	if err != nil {
+//		t.Fatalf("build Tree should not result in an error: %v", res)
+//	}
+//
+//	assert.equal(t, res.Heads(), []string{"C.1.1"})
+//	assert.equal(t, res.Len(), 4)
+//}
+//
+//func TestTreeBuilder_UserJoinTestTreeIterate(t *testing.T) {
+//	thread, err := threadbuilder.NewThreadBuilderWithTestName("threadbuilder/userjoinexample.yml")
+//	if err != nil {
+//		t.Fatal(err)
+//	}
+//
+//	res, err := createTreeFromThread(thread)
+//	if err != nil {
+//		t.Fatalf("build Tree should not result in an error: %v", res)
+//	}
+//
+//	assert.equal(t, res.Heads(), []string{"C.1.1"})
+//	assert.equal(t, res.Len(), 4)
+//	var changeIds []string
+//	res.iterate(res.root, func(c *Change) (isContinue bool) {
+//		changeIds = append(changeIds, c.Id)
+//		return true
+//	})
+//	assert.equal(t, changeIds, []string{"A.1.1", "A.1.2", "B.1.1", "C.1.1"})
+//}
+//
+//func TestTreeBuilder_UserRemoveTestTreeIterate(t *testing.T) {
+//	thread, err := threadbuilder.NewThreadBuilderWithTestName("threadbuilder/userremoveexample.yml")
+//	if err != nil {
+//		t.Fatal(err)
+//	}
+//
+//	res, err := createTreeFromThread(thread)
+//	if err != nil {
+//		t.Fatalf("build Tree should not result in an error: %v", res)
+//	}
+//
+//	assert.equal(t, res.Heads(), []string{"A.1.3"})
+//	assert.equal(t, res.Len(), 4)
+//	var changeIds []string
+//	res.iterate(res.root, func(c *Change) (isContinue bool) {
+//		changeIds = append(changeIds, c.Id)
+//		return true
+//	})
+//	assert.equal(t, changeIds, []string{"A.1.1", "A.1.2", "B.1.1", "A.1.3"})
+//}

acltree/treegraph.go

@@ -0,0 +1,11 @@
+//go:build ((!linux && !darwin) || android || ios || nographviz) && !amd64
+// +build !linux,!darwin android ios nographviz
+// +build !amd64
+
+package acltree
+
+import "fmt"
+
+func (t *Tree) Graph() (data string, err error) {
+	return "", fmt.Errorf("not supported")
+}

acltree/treegraph_nix.go

@@ -0,0 +1,151 @@
+//go:build (linux || darwin) && !android && !ios && !nographviz && (amd64 || arm64)
+// +build linux darwin
+// +build !android
+// +build !ios
+// +build !nographviz
+// +build amd64 arm64
+
+package acltree
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"time"
+	"unicode"
+
+	"github.com/goccy/go-graphviz"
+	"github.com/goccy/go-graphviz/cgraph"
+)
+
+func (t *Tree) Graph() (data string, err error) {
+	var order = make(map[string]string)
+	var seq = 0
+	t.Iterate(t.RootId(), func(c *Change) (isContinue bool) {
+		v := order[c.Id]
+		if v == "" {
+			order[c.Id] = fmt.Sprint(seq)
+		} else {
+			order[c.Id] = fmt.Sprintf("%s,%d", v, seq)
+		}
+		seq++
+		return true
+	})
+	g := graphviz.New()
+	defer g.Close()
+	graph, err := g.Graph()
+	if err != nil {
+		return
+	}
+	defer func() {
+		err = graph.Close()
+	}()
+	var nodes = make(map[string]*cgraph.Node)
+	var addChange = func(c *Change) error {
+		n, e := graph.CreateNode(c.Id)
+		if e != nil {
+			return e
+		}
+		if c.Content.GetAclData() != nil {
+			n.SetStyle(cgraph.FilledNodeStyle)
+		} else if c.IsSnapshot {
+			n.SetStyle(cgraph.DashedNodeStyle)
+		}
+		nodes[c.Id] = n
+		ord := order[c.Id]
+		if ord == "" {
+			ord = "miss"
+		}
+		var chSymbs []string
+		if c.Content.AclData != nil {
+			for _, chc := range c.Content.AclData.AclContent {
+				tp := fmt.Sprintf("%T", chc.Value)
+				tp = strings.Replace(tp, "ACLChangeACLContentValueValueOf", "", 1)
+				res := ""
+				for _, ts := range tp {
+					if unicode.IsUpper(ts) {
+						res += string(ts)
+					}
+				}
+				chSymbs = append(chSymbs, res)
+			}
+		}
+		if c.DecryptedDocumentChange != nil {
+			// TODO: add some parser to provide custom unmarshalling for the document change
+			//for _, chc := range c.DecryptedDocumentChange.Content {
+			//	tp := fmt.Sprintf("%T", chc.Value)
+			//	tp = strings.Replace(tp, "ChangeContentValueOf", "", 1)
+			//	res := ""
+			//	for _, ts := range tp {
+			//		if unicode.IsUpper(ts) {
+			//			res += string(ts)
+			//		}
+			//	}
+			//	chSymbs = append(chSymbs, res)
+			//}
+		}
+
+		shortId := c.Id
+		label := fmt.Sprintf("Id: %s\nOrd: %s\nTime: %s\nChanges: %s\n",
+			shortId,
+			ord,
+			time.Unix(c.Content.Timestamp, 0).Format("02.01.06 15:04:05"),
+			strings.Join(chSymbs, ","),
+		)
+		n.SetLabel(label)
+		return nil
+	}
+	for _, c := range t.attached {
+		if err = addChange(c); err != nil {
+			return
+		}
+	}
+	for _, c := range t.unAttached {
+		if err = addChange(c); err != nil {
+			return
+		}
+	}
+	var getNode = func(id string) (*cgraph.Node, error) {
+		if n, ok := nodes[id]; ok {
+			return n, nil
+		}
+		n, err := graph.CreateNode(fmt.Sprintf("%s: not in Tree", id))
+		if err != nil {
+			return nil, err
+		}
+		nodes[id] = n
+		return n, nil
+	}
+	var addLinks = func(c *Change) error {
+		for _, prevId := range c.PreviousIds {
+			self, e := getNode(c.Id)
+			if e != nil {
+				return e
+			}
+			prev, e := getNode(prevId)
+			if e != nil {
+				return e
+			}
+			_, e = graph.CreateEdge("", self, prev)
+			if e != nil {
+				return e
+			}
+		}
+		return nil
+	}
+	for _, c := range t.attached {
+		if err = addLinks(c); err != nil {
+			return
+		}
+	}
+	for _, c := range t.unAttached {
+		if err = addLinks(c); err != nil {
+			return
+		}
+	}
+	var buf bytes.Buffer
+	if err = g.Render(graph, "dot", &buf); err != nil {
+		return
+	}
+	return buf.String(), nil
+}

acltree/treeiterator.go

@@ -0,0 +1,158 @@
+package acltree
+
+import "sync"
+
+var itPool = &sync.Pool{
+	New: func() interface{} {
+		return &iterator{}
+	},
+}
+
+func newIterator() *iterator {
+	return itPool.Get().(*iterator)
+}
+
+func freeIterator(i *iterator) {
+	itPool.Put(i)
+}
+
+type iterator struct {
+	compBuf    []*Change
+	queue      []*Change
+	doneMap    map[*Change]struct{}
+	breakpoint *Change
+	f          func(c *Change) bool
+}
+
+func (i *iterator) iterateSkip(start *Change, skipBefore *Change, f func(c *Change) (isContinue bool)) {
+	skipping := true
+	i.iterate(start, func(c *Change) (isContinue bool) {
+		if skipping && c != skipBefore {
+			return true
+		}
+		skipping = false
+		return f(c)
+	})
+}
+
+func (i *iterator) iterate(start *Change, f func(c *Change) (isContinue bool)) {
+	if start == nil {
+		return
+	}
+	// reset
+	i.queue = i.queue[:0]
+	i.compBuf = i.compBuf[:0]
+	i.doneMap = make(map[*Change]struct{})
+	i.queue = append(i.queue, start)
+	i.breakpoint = nil
+	i.f = f
+
+	for len(i.queue) > 0 {
+		c := i.queue[0]
+		i.queue = i.queue[1:]
+		nl := len(c.Next)
+		if nl == 1 {
+			if !i.iterateLin(c) {
+				return
+			}
+			if i.breakpoint != nil {
+				i.toQueue(i.breakpoint)
+				i.breakpoint = nil
+			}
+		} else {
+			_, done := i.doneMap[c]
+			if !done {
+				if !f(c) {
+					return
+				}
+				i.doneMap[c] = struct{}{}
+			}
+			if nl != 0 {
+				for _, next := range c.Next {
+					i.toQueue(next)
+				}
+			}
+		}
+	}
+}
+
+func (i *iterator) iterateLin(c *Change) bool {
+	for len(c.Next) == 1 {
+		_, done := i.doneMap[c]
+		if !done {
+			if !i.f(c) {
+				return false
+			}
+			i.doneMap[c] = struct{}{}
+		}
+
+		c = c.Next[0]
+		if len(c.PreviousIds) > 1 {
+			break
+		}
+	}
+	if len(c.Next) == 0 && len(c.PreviousIds) <= 1 {
+		if !i.f(c) {
+			return false
+		}
+		i.doneMap[c] = struct{}{}
+	} else {
+		i.breakpoint = c
+	}
+
+	return true
+}
+
+func (i *iterator) comp(c1, c2 *Change) uint8 {
+	if c1.Id == c2.Id {
+		return 0
+	}
+	i.compBuf = i.compBuf[:0]
+	i.compBuf = append(i.compBuf, c1.Next...)
+	var uniq = make(map[*Change]struct{})
+	var appendUniqueToBuf = func(next []*Change) {
+		for _, n := range next {
+			if _, ok := uniq[n]; !ok {
+				i.compBuf = append(i.compBuf, n)
+				uniq[n] = struct{}{}
+			}
+		}
+	}
+	var used int
+	for len(i.compBuf)-used > 0 {
+		l := len(i.compBuf) - used
+		for _, n := range i.compBuf[used:] {
+			delete(uniq, n)
+			if n.Id == c2.Id {
+				return 1
+			} else {
+				appendUniqueToBuf(n.Next)
+			}
+		}
+		used += l
+	}
+	return 2
+}
+
+func (i *iterator) toQueue(c *Change) {
+	var pos = -1
+For:
+	for idx, qc := range i.queue {
+		switch i.comp(c, qc) {
+		// exists
+		case 0:
+			return
+		//
+		case 1:
+			pos = idx
+			break For
+		}
+	}
+	if pos == -1 {
+		i.queue = append(i.queue, c)
+	} else if pos == 0 {
+		i.queue = append([]*Change{c}, i.queue...)
+	} else {
+		i.queue = append(i.queue[:pos], append([]*Change{c}, i.queue[pos:]...)...)
+	}
+}

go.mod

@@ -0,0 +1,60 @@
+module github.com/anytypeio/go-anytype-infrastructure-experiments
+
+go 1.18
+
+require (
+	github.com/awalterschulze/gographviz v0.0.0-20190522210029-fa59802746ab
+	github.com/goccy/go-graphviz v0.0.9
+	github.com/gogo/protobuf v1.3.2
+	github.com/libp2p/go-libp2p-core v0.8.5
+	github.com/mr-tron/base58 v1.2.0
+	github.com/prometheus/common v0.18.0
+	github.com/stretchr/testify v1.7.0
+	github.com/textileio/go-threads v1.0.2-0.20210304072541-d0f91da84404
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
+	github.com/multiformats/go-base32 v0.0.3
+	github.com/multiformats/go-multiaddr v0.3.3
+	github.com/multiformats/go-multiaddr-dns v0.3.1
+	github.com/multiformats/go-multibase v0.0.3
+	github.com/multiformats/go-multihash v0.0.15
+)
+
+require (
+	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
+	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
+	github.com/btcsuite/btcd v0.21.0-beta // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
+	github.com/fogleman/gg v1.3.0 // indirect
+	github.com/gogo/googleapis v1.3.1 // indirect
+	github.com/gogo/status v1.1.0 // indirect
+	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.2.1 // indirect
+	github.com/ipfs/go-cid v0.0.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.4 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
+	github.com/libp2p/go-buffer-pool v0.0.2 // indirect
+	github.com/libp2p/go-openssl v0.0.7 // indirect
+	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/multiformats/go-base32 v0.0.3 // indirect
+	github.com/multiformats/go-base36 v0.1.0 // indirect
+	github.com/multiformats/go-multiaddr v0.3.3 // indirect
+	github.com/multiformats/go-multibase v0.0.3 // indirect
+	github.com/multiformats/go-multihash v0.0.15 // indirect
+	github.com/multiformats/go-varint v0.0.6 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.6.0 // indirect
+	github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
+	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect
+	golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
+	golang.org/x/sys v0.0.0-20210426080607-c94f62235c83 // indirect
+	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
+	google.golang.org/grpc v1.33.2 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
+)
+
+replace github.com/textileio/go-threads => github.com/anytypeio/go-threads v1.1.0-rc1.0.20220223104843-a67245cee80e

plaintextdocument/document.go

@@ -0,0 +1,167 @@
+package plaintextdocument
+
+import (
+	"fmt"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/account"
+	aclpb "github.com/anytypeio/go-anytype-infrastructure-experiments/aclchanges/pb"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/acltree"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/testutils/testchanges/pb"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/thread"
+	"github.com/gogo/protobuf/proto"
+)
+
+type PlainTextDocument interface {
+	Text() string
+	AddText(text string) error
+}
+
+type plainTextDocument struct {
+	heads   []string
+	aclTree acltree.ACLTree
+	state   *DocumentState
+}
+
+func (p *plainTextDocument) Text() string {
+	if p.state != nil {
+		return p.state.Text
+	}
+	return ""
+}
+
+func (p *plainTextDocument) AddText(text string) error {
+	_, err := p.aclTree.AddContent(func(builder acltree.ChangeBuilder) error {
+		builder.AddChangeContent(
+			&pb.PlainTextChangeData{
+				Content: []*pb.PlainTextChangeContent{
+					createAppendTextChangeContent(text),
+				},
+			})
+		return nil
+	})
+	return err
+}
+
+func (p *plainTextDocument) Update(tree acltree.ACLTree) {
+	p.aclTree = tree
+	var err error
+	defer func() {
+		if err != nil {
+			fmt.Println("rebuild has returned error:", err)
+		}
+	}()
+
+	prevHeads := p.heads
+	p.heads = tree.Heads()
+	startId := prevHeads[0]
+	tree.IterateFrom(startId, func(change *acltree.Change) (isContinue bool) {
+		if change.Id == startId {
+			return true
+		}
+		if change.DecryptedDocumentChange != nil {
+			p.state, err = p.state.ApplyChange(change.DecryptedDocumentChange, change.Id)
+			if err != nil {
+				return false
+			}
+		}
+		return true
+	})
+}
+
+func (p *plainTextDocument) Rebuild(tree acltree.ACLTree) {
+	p.aclTree = tree
+	p.heads = tree.Heads()
+	var startId string
+	var err error
+	defer func() {
+		if err != nil {
+			fmt.Println("rebuild has returned error:", err)
+		}
+	}()
+
+	rootChange := tree.Root()
+
+	if rootChange.DecryptedDocumentChange == nil {
+		err = fmt.Errorf("root doesn't have decrypted change")
+		return
+	}
+
+	state, err := BuildDocumentStateFromChange(rootChange.DecryptedDocumentChange, rootChange.Id)
+	if err != nil {
+		return
+	}
+
+	startId = rootChange.Id
+	tree.Iterate(func(change *acltree.Change) (isContinue bool) {
+		if startId == change.Id {
+			return true
+		}
+		if change.DecryptedDocumentChange != nil {
+			state, err = state.ApplyChange(change.DecryptedDocumentChange, change.Id)
+			if err != nil {
+				return false
+			}
+		}
+		return true
+	})
+	if err != nil {
+		return
+	}
+	p.state = state
+}
+
+func NewInMemoryPlainTextDocument(acc *account.AccountData, text string) (PlainTextDocument, error) {
+	return NewPlainTextDocument(acc, thread.NewInMemoryThread, text)
+}
+
+func NewPlainTextDocument(
+	acc *account.AccountData,
+	create func(change *thread.RawChange) (thread.Thread, error),
+	text string) (PlainTextDocument, error) {
+	changeBuilder := func(builder acltree.ChangeBuilder) error {
+		err := builder.UserAdd(acc.Identity, acc.EncKey.GetPublic(), aclpb.ACLChange_Admin)
+		if err != nil {
+			return err
+		}
+		builder.AddChangeContent(createInitialChangeContent(text))
+		return nil
+	}
+	t, err := acltree.BuildThreadWithACL(
+		acc,
+		changeBuilder,
+		create)
+	if err != nil {
+		return nil, err
+	}
+
+	doc := &plainTextDocument{
+		heads:   nil,
+		aclTree: nil,
+		state:   nil,
+	}
+	tree, err := acltree.BuildACLTree(t, acc, doc)
+	if err != nil {
+		return nil, err
+	}
+	doc.aclTree = tree
+	return doc, nil
+}
+
+func createInitialChangeContent(text string) proto.Marshaler {
+	return &pb.PlainTextChangeData{
+		Content: []*pb.PlainTextChangeContent{
+			createAppendTextChangeContent(text),
+		},
+		Snapshot: &pb.PlainTextChangeSnapshot{Text: text},
+	}
+}
+
+func createAppendTextChangeContent(text string) *pb.PlainTextChangeContent {
+	return &pb.PlainTextChangeContent{
+		Value: &pb.PlainTextChangeContentValueOfTextAppend{
+			TextAppend: &pb.PlainTextChangeTextAppend{
+				Text: text,
+			},
+		},
+	}
+}

plaintextdocument/document_test.go

@@ -0,0 +1,54 @@
+package plaintextdocument
+
+import (
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/account"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/testutils/threadbuilder"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/thread"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestDocument_NewPlainTextDocument(t *testing.T) {
+	keychain := threadbuilder.NewKeychain()
+	keychain.AddSigningKey("A")
+	keychain.AddEncryptionKey("A")
+	data := &account.AccountData{
+		Identity: keychain.GetIdentity("A"),
+		SignKey:  keychain.SigningKeys["A"],
+		EncKey:   keychain.EncryptionKeys["A"],
+	}
+
+	doc, err := NewPlainTextDocument(data, thread.NewInMemoryThread, "Some text")
+	if err != nil {
+		t.Fatalf("should not create document with error: %v", err)
+	}
+	assert.Equal(t, doc.Text(), "Some text")
+}
+
+func TestDocument_PlainTextDocument_AddText(t *testing.T) {
+	keychain := threadbuilder.NewKeychain()
+	keychain.AddSigningKey("A")
+	keychain.AddEncryptionKey("A")
+	data := &account.AccountData{
+		Identity: keychain.GetIdentity("A"),
+		SignKey:  keychain.SigningKeys["A"],
+		EncKey:   keychain.EncryptionKeys["A"],
+	}
+
+	doc, err := NewPlainTextDocument(data, thread.NewInMemoryThread, "Some text")
+	if err != nil {
+		t.Fatalf("should not create document with error: %v", err)
+	}
+
+	err = doc.AddText("Next")
+	if err != nil {
+		t.Fatalf("should be able to add document: %v", err)
+	}
+	assert.Equal(t, doc.Text(), "Some text|Next")
+
+	err = doc.AddText("Shmext")
+	if err != nil {
+		t.Fatalf("should be able to add document: %v", err)
+	}
+	assert.Equal(t, doc.Text(), "Some text|Next|Shmext")
+}

plaintextdocument/plaintextdocstate.go

@@ -0,0 +1,59 @@
+package plaintextdocument
+
+import (
+	"fmt"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/testutils/testchanges/pb"
+	"github.com/gogo/protobuf/proto"
+)
+
+type DocumentState struct {
+	LastChangeId string
+	Text         string
+}
+
+func NewDocumentState(text string, id string) *DocumentState {
+	return &DocumentState{
+		LastChangeId: id,
+		Text:         text,
+	}
+}
+
+func BuildDocumentStateFromChange(change []byte, id string) (*DocumentState, error) {
+	var changesData pb.PlainTextChangeData
+	err := proto.Unmarshal(change, &changesData)
+	if err != nil {
+		return nil, err
+	}
+
+	if changesData.GetSnapshot() == nil {
+		return nil, fmt.Errorf("could not create state from empty snapshot")
+	}
+	return NewDocumentState(changesData.GetSnapshot().GetText(), id), nil
+}
+
+func (p *DocumentState) ApplyChange(change []byte, id string) (*DocumentState, error) {
+	var changesData pb.PlainTextChangeData
+	err := proto.Unmarshal(change, &changesData)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, content := range changesData.GetContent() {
+		err = p.applyChange(content)
+		if err != nil {
+			return nil, err
+		}
+	}
+	p.LastChangeId = id
+	return p, nil
+}
+
+func (p *DocumentState) applyChange(ch *pb.PlainTextChangeContent) error {
+	switch {
+	case ch.GetTextAppend() != nil:
+		text := ch.GetTextAppend().GetText()
+		p.Text += "|" + text
+	}
+	return nil
+}

testutils/testchanges/pb/protos/testdocumentchanges.proto

@@ -0,0 +1,26 @@
+syntax = "proto3";
+package anytype;
+option go_package = "pb";
+
+// TODO: move to separate package
+
+message PlainTextChange {
+  message Content {
+    oneof value {
+      TextAppend textAppend = 1;
+    }
+  }
+
+  message TextAppend {
+    string text = 1;
+  }
+
+  message Snapshot {
+    string text = 1;
+  }
+
+  message Data {
+    repeated Content content = 1;
+    Snapshot snapshot = 2;
+  }
+}

testutils/threadbuilder/keychain.go

@@ -0,0 +1,149 @@
+package threadbuilder
+
+import (
+	"hash/fnv"
+	"strings"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
+
+	"github.com/textileio/go-threads/crypto/symmetric"
+)
+
+type SymKey struct {
+	Hash uint64
+	Key  *symmetric.Key
+}
+
+type Keychain struct {
+	SigningKeys           map[string]keys.SigningPrivKey
+	SigningKeysByIdentity map[string]keys.SigningPrivKey
+	EncryptionKeys        map[string]keys.EncryptionPrivKey
+	ReadKeys              map[string]*SymKey
+	ReadKeysByHash        map[uint64]*SymKey
+	GeneratedIdentities   map[string]string
+	coder                 *keys.Ed25519SigningPubKeyDecoder
+}
+
+func NewKeychain() *Keychain {
+	return &Keychain{
+		SigningKeys:           map[string]keys.SigningPrivKey{},
+		SigningKeysByIdentity: map[string]keys.SigningPrivKey{},
+		EncryptionKeys:        map[string]keys.EncryptionPrivKey{},
+		GeneratedIdentities:   map[string]string{},
+		ReadKeys:              map[string]*SymKey{},
+		ReadKeysByHash:        map[uint64]*SymKey{},
+		coder:                 keys.NewEd25519Decoder(),
+	}
+}
+
+func (k *Keychain) ParseKeys(keys *Keys) {
+	for _, encKey := range keys.Enc {
+		k.AddEncryptionKey(encKey)
+	}
+
+	for _, signKey := range keys.Sign {
+		k.AddSigningKey(signKey)
+	}
+
+	for _, readKey := range keys.Read {
+		k.AddReadKey(readKey)
+	}
+}
+
+func (k *Keychain) AddEncryptionKey(name string) {
+	if _, exists := k.EncryptionKeys[name]; exists {
+		return
+	}
+	newPrivKey, _, err := keys.GenerateRandomRSAKeyPair(2048)
+	if err != nil {
+		panic(err)
+	}
+
+	k.EncryptionKeys[name] = newPrivKey
+}
+
+func (k *Keychain) AddSigningKey(name string) {
+	if _, exists := k.SigningKeys[name]; exists {
+		return
+	}
+	newPrivKey, pubKey, err := keys.GenerateRandomEd25519KeyPair()
+	if err != nil {
+		panic(err)
+	}
+
+	k.SigningKeys[name] = newPrivKey
+	res, err := k.coder.EncodeToString(pubKey)
+	if err != nil {
+		panic(err)
+	}
+	k.SigningKeysByIdentity[res] = newPrivKey
+	k.GeneratedIdentities[name] = res
+}
+
+func (k *Keychain) AddReadKey(name string) {
+	if _, exists := k.ReadKeys[name]; exists {
+		return
+	}
+	key, _ := symmetric.NewRandom()
+
+	hasher := fnv.New64()
+	hasher.Write(key.Bytes())
+
+	k.ReadKeys[name] = &SymKey{
+		Hash: hasher.Sum64(),
+		Key:  key,
+	}
+	k.ReadKeysByHash[hasher.Sum64()] = &SymKey{
+		Hash: hasher.Sum64(),
+		Key:  key,
+	}
+}
+
+func (k *Keychain) AddKey(key string) {
+	parts := strings.Split(key, ".")
+	if len(parts) != 3 {
+		panic("cannot parse a key")
+	}
+	name := parts[2]
+
+	switch parts[1] {
+	case "Sign":
+		k.AddSigningKey(name)
+	case "Enc":
+		k.AddEncryptionKey(name)
+	case "Read":
+		k.AddReadKey(name)
+	default:
+		panic("incorrect format")
+	}
+}
+
+func (k *Keychain) GetKey(key string) interface{} {
+	parts := strings.Split(key, ".")
+	if len(parts) != 3 {
+		panic("cannot parse a key")
+	}
+	name := parts[2]
+
+	switch parts[1] {
+	case "Sign":
+		if key, exists := k.SigningKeys[name]; exists {
+			return key
+		}
+	case "Enc":
+		if key, exists := k.EncryptionKeys[name]; exists {
+			return key
+		}
+	case "Read":
+		if key, exists := k.ReadKeys[name]; exists {
+			return key
+		}
+	default:
+		panic("incorrect format")
+	}
+	return nil
+}
+
+func (k *Keychain) GetIdentity(name string) string {
+	return k.GeneratedIdentities[name]
+}

testutils/threadbuilder/threadbuilder.go

@@ -0,0 +1,543 @@
+package threadbuilder
+
+import (
+	"context"
+	"fmt"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/aclchanges"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/testutils/yamltests"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice"
+	"io/ioutil"
+	"path"
+
+	"github.com/gogo/protobuf/proto"
+	"gopkg.in/yaml.v3"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/aclchanges/pb"
+	testpb "github.com/anytypeio/go-anytype-infrastructure-experiments/testutils/testchanges/pb"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/thread"
+	threadpb "github.com/anytypeio/go-anytype-infrastructure-experiments/thread/pb"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
+)
+
+const plainTextDocType uint16 = 1
+
+type threadChange struct {
+	*pb.ACLChange
+	id      string
+	readKey *SymKey
+	signKey keys.SigningPrivKey
+
+	changesDataDecrypted []byte
+}
+
+type updateUseCase struct {
+	changes map[string]*threadChange
+}
+
+type ThreadBuilder struct {
+	threadId   string
+	allChanges map[string]*threadChange
+	updates    map[string]*updateUseCase
+	heads      []string
+	orphans    []string
+	keychain   *Keychain
+	header     *threadpb.ThreadHeader
+}
+
+func NewThreadBuilder(keychain *Keychain) *ThreadBuilder {
+	return &ThreadBuilder{
+		allChanges: make(map[string]*threadChange),
+		updates:    make(map[string]*updateUseCase),
+		keychain:   keychain,
+	}
+}
+
+func NewThreadBuilderWithTestName(name string) (*ThreadBuilder, error) {
+	filePath := path.Join(yamltests.Path(), name)
+	return NewThreadBuilderFromFile(filePath)
+}
+
+func NewThreadBuilderFromFile(file string) (*ThreadBuilder, error) {
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+
+	thread := YMLThread{}
+	err = yaml.Unmarshal(content, &thread)
+	if err != nil {
+		return nil, err
+	}
+
+	tb := NewThreadBuilder(NewKeychain())
+	tb.Parse(&thread)
+
+	return tb, nil
+}
+
+func (t *ThreadBuilder) ID() string {
+	return t.threadId
+}
+
+func (t *ThreadBuilder) GetKeychain() *Keychain {
+	return t.keychain
+}
+
+func (t *ThreadBuilder) Heads() []string {
+	return t.heads
+}
+
+func (t *ThreadBuilder) AddRawChange(change *thread.RawChange) error {
+	aclChange := new(pb.ACLChange)
+	var err error
+
+	if err = proto.Unmarshal(change.Payload, aclChange); err != nil {
+		return fmt.Errorf("could not unmarshall changes")
+	}
+	var changesData []byte
+
+	// get correct readkey
+	readKey := t.keychain.ReadKeysByHash[aclChange.CurrentReadKeyHash]
+	if aclChange.ChangesData != nil {
+		changesData, err = readKey.Key.Decrypt(aclChange.ChangesData)
+		if err != nil {
+			return fmt.Errorf("failed to decrypt changes data: %w", err)
+		}
+	}
+
+	// get correct signing key
+	signKey := t.keychain.SigningKeysByIdentity[aclChange.Identity]
+
+	t.allChanges[change.Id] = &threadChange{
+		ACLChange:            aclChange,
+		id:                   change.Id,
+		readKey:              readKey,
+		signKey:              signKey,
+		changesDataDecrypted: changesData,
+	}
+	return nil
+}
+
+func (t *ThreadBuilder) AddOrphans(orphans ...string) {
+	t.orphans = append(t.orphans, orphans...)
+}
+
+func (t *ThreadBuilder) AddChange(change aclchanges.Change) error {
+	aclChange := change.ProtoChange()
+	var err error
+	var changesData []byte
+
+	// get correct readkey
+	readKey := t.keychain.ReadKeysByHash[aclChange.CurrentReadKeyHash]
+	if aclChange.ChangesData != nil {
+		changesData, err = readKey.Key.Decrypt(aclChange.ChangesData)
+		if err != nil {
+			return fmt.Errorf("failed to decrypt changes data: %w", err)
+		}
+	}
+
+	// get correct signing key
+	signKey := t.keychain.SigningKeysByIdentity[aclChange.Identity]
+
+	t.allChanges[change.CID()] = &threadChange{
+		ACLChange:            aclChange,
+		id:                   change.CID(),
+		readKey:              readKey,
+		signKey:              signKey,
+		changesDataDecrypted: changesData,
+	}
+	return nil
+}
+
+func (t *ThreadBuilder) Orphans() []string {
+	return t.orphans
+}
+
+func (t *ThreadBuilder) SetHeads(heads []string) {
+	// we should copy here instead of just setting the value
+	t.heads = heads
+}
+
+func (t *ThreadBuilder) RemoveOrphans(orphans ...string) {
+	t.orphans = slice.Difference(t.orphans, orphans)
+}
+
+func (t *ThreadBuilder) GetChange(ctx context.Context, recordID string) (*thread.RawChange, error) {
+	return t.getChange(recordID, t.allChanges), nil
+}
+
+func (t *ThreadBuilder) GetUpdates(useCase string) []*thread.RawChange {
+	var res []*thread.RawChange
+	update := t.updates[useCase]
+	for _, ch := range update.changes {
+		rawCh := t.getChange(ch.id, update.changes)
+		res = append(res, rawCh)
+	}
+	return res
+}
+
+func (t *ThreadBuilder) Header() *threadpb.ThreadHeader {
+	return t.header
+}
+
+func (t *ThreadBuilder) getChange(changeId string, m map[string]*threadChange) *thread.RawChange {
+	rec := m[changeId]
+
+	if rec.changesDataDecrypted != nil {
+		encrypted, err := rec.readKey.Key.Encrypt(rec.changesDataDecrypted)
+		if err != nil {
+			panic("should be able to encrypt data with read key!")
+		}
+
+		rec.ChangesData = encrypted
+	}
+
+	aclMarshaled, err := proto.Marshal(rec.ACLChange)
+	if err != nil {
+		panic("should be able to marshal final acl message!")
+	}
+
+	signature, err := rec.signKey.Sign(aclMarshaled)
+	if err != nil {
+		panic("should be able to sign final acl message!")
+	}
+
+	transformedRec := &thread.RawChange{
+		Payload:   aclMarshaled,
+		Signature: signature,
+		Id:        changeId,
+	}
+	return transformedRec
+}
+
+func (t *ThreadBuilder) Parse(thread *YMLThread) {
+	// Just to clarify - we are generating new identities for the ones that
+	// are specified in the yml file, because our identities should be Ed25519
+	// the same thing is happening for the encryption keys
+	t.keychain.ParseKeys(&thread.Keys)
+	t.threadId = t.parseThreadId(thread.Description)
+	for _, ch := range thread.Changes {
+		newChange := t.parseChange(ch)
+		t.allChanges[newChange.id] = newChange
+	}
+
+	t.parseGraph(thread)
+	t.parseOrphans(thread)
+	t.parseHeader(thread)
+	t.parseUpdates(thread.Updates)
+}
+
+func (t *ThreadBuilder) parseChange(ch *Change) *threadChange {
+	newChange := &threadChange{
+		id: ch.Id,
+	}
+	k := t.keychain.GetKey(ch.ReadKey).(*SymKey)
+	newChange.readKey = k
+	newChange.signKey = t.keychain.SigningKeys[ch.Identity]
+	aclChange := &pb.ACLChange{}
+	aclChange.Identity = t.keychain.GetIdentity(ch.Identity)
+	if len(ch.AclChanges) > 0 || ch.AclSnapshot != nil {
+		aclChange.AclData = &pb.ACLChangeACLData{}
+		if ch.AclSnapshot != nil {
+			aclChange.AclData.AclSnapshot = t.parseACLSnapshot(ch.AclSnapshot)
+		}
+		if ch.AclChanges != nil {
+			var aclChangeContents []*pb.ACLChangeACLContentValue
+			for _, ch := range ch.AclChanges {
+				aclChangeContent := t.parseACLChange(ch)
+				aclChangeContents = append(aclChangeContents, aclChangeContent)
+			}
+			aclChange.AclData.AclContent = aclChangeContents
+		}
+	}
+	if len(ch.Changes) > 0 || ch.Snapshot != nil {
+		changesData := &testpb.PlainTextChangeData{}
+		if ch.Snapshot != nil {
+			changesData.Snapshot = t.parseChangeSnapshot(ch.Snapshot)
+		}
+		if len(ch.Changes) > 0 {
+			var changeContents []*testpb.PlainTextChangeContent
+			for _, ch := range ch.Changes {
+				aclChangeContent := t.parseDocumentChange(ch)
+				changeContents = append(changeContents, aclChangeContent)
+			}
+			changesData.Content = changeContents
+		}
+		m, err := proto.Marshal(changesData)
+		if err != nil {
+			return nil
+		}
+		newChange.changesDataDecrypted = m
+	}
+	aclChange.CurrentReadKeyHash = k.Hash
+	newChange.ACLChange = aclChange
+	return newChange
+}
+
+func (t *ThreadBuilder) parseThreadId(description *ThreadDescription) string {
+	if description == nil {
+		panic("no author in thread")
+	}
+	key := t.keychain.SigningKeys[description.Author]
+	id, err := thread.CreateACLThreadID(key.GetPublic(), plainTextDocType)
+	if err != nil {
+		panic(err)
+	}
+
+	return id.String()
+}
+
+func (t *ThreadBuilder) parseChangeSnapshot(s *PlainTextSnapshot) *testpb.PlainTextChangeSnapshot {
+	return &testpb.PlainTextChangeSnapshot{
+		Text: s.Text,
+	}
+}
+
+func (t *ThreadBuilder) parseACLSnapshot(s *ACLSnapshot) *pb.ACLChangeACLSnapshot {
+	newState := &pb.ACLChangeACLState{}
+	for _, state := range s.UserStates {
+		aclUserState := &pb.ACLChangeUserState{}
+		aclUserState.Identity = t.keychain.GetIdentity(state.Identity)
+
+		encKey := t.keychain.
+			GetKey(state.EncryptionKey).(keys.EncryptionPrivKey)
+		rawKey, _ := encKey.GetPublic().Raw()
+		aclUserState.EncryptionKey = rawKey
+
+		aclUserState.EncryptedReadKeys = t.encryptReadKeys(state.EncryptedReadKeys, encKey)
+		aclUserState.Permissions = t.convertPermission(state.Permissions)
+		newState.UserStates = append(newState.UserStates, aclUserState)
+	}
+	return &pb.ACLChangeACLSnapshot{
+		AclState: newState,
+	}
+}
+
+func (t *ThreadBuilder) parseDocumentChange(ch *PlainTextChange) (convCh *testpb.PlainTextChangeContent) {
+	switch {
+	case ch.TextAppend != nil:
+		convCh = &testpb.PlainTextChangeContent{
+			Value: &testpb.PlainTextChangeContentValueOfTextAppend{
+				TextAppend: &testpb.PlainTextChangeTextAppend{
+					Text: ch.TextAppend.Text,
+				},
+			},
+		}
+	}
+	if convCh == nil {
+		panic("cannot have empty document change")
+	}
+
+	return convCh
+}
+
+func (t *ThreadBuilder) parseACLChange(ch *ACLChange) (convCh *pb.ACLChangeACLContentValue) {
+	switch {
+	case ch.UserAdd != nil:
+		add := ch.UserAdd
+
+		encKey := t.keychain.
+			GetKey(add.EncryptionKey).(keys.EncryptionPrivKey)
+		rawKey, _ := encKey.GetPublic().Raw()
+
+		convCh = &pb.ACLChangeACLContentValue{
+			Value: &pb.ACLChangeACLContentValueValueOfUserAdd{
+				UserAdd: &pb.ACLChangeUserAdd{
+					Identity:          t.keychain.GetIdentity(add.Identity),
+					EncryptionKey:     rawKey,
+					EncryptedReadKeys: t.encryptReadKeys(add.EncryptedReadKeys, encKey),
+					Permissions:       t.convertPermission(add.Permission),
+				},
+			},
+		}
+	case ch.UserJoin != nil:
+		join := ch.UserJoin
+
+		encKey := t.keychain.
+			GetKey(join.EncryptionKey).(keys.EncryptionPrivKey)
+		rawKey, _ := encKey.GetPublic().Raw()
+
+		idKey, _ := t.keychain.SigningKeys[join.Identity].GetPublic().Raw()
+		signKey := t.keychain.GetKey(join.AcceptSignature).(keys.SigningPrivKey)
+		signature, err := signKey.Sign(idKey)
+		if err != nil {
+			panic(err)
+		}
+
+		convCh = &pb.ACLChangeACLContentValue{
+			Value: &pb.ACLChangeACLContentValueValueOfUserJoin{
+				UserJoin: &pb.ACLChangeUserJoin{
+					Identity:          t.keychain.GetIdentity(join.Identity),
+					EncryptionKey:     rawKey,
+					AcceptSignature:   signature,
+					UserInviteId:      join.InviteId,
+					EncryptedReadKeys: t.encryptReadKeys(join.EncryptedReadKeys, encKey),
+				},
+			},
+		}
+	case ch.UserInvite != nil:
+		invite := ch.UserInvite
+		rawAcceptKey, _ := t.keychain.GetKey(invite.AcceptKey).(keys.SigningPrivKey).GetPublic().Raw()
+		encKey := t.keychain.
+			GetKey(invite.EncryptionKey).(keys.EncryptionPrivKey)
+		rawEncKey, _ := encKey.GetPublic().Raw()
+
+		convCh = &pb.ACLChangeACLContentValue{
+			Value: &pb.ACLChangeACLContentValueValueOfUserInvite{
+				UserInvite: &pb.ACLChangeUserInvite{
+					AcceptPublicKey:   rawAcceptKey,
+					EncryptPublicKey:  rawEncKey,
+					EncryptedReadKeys: t.encryptReadKeys(invite.EncryptedReadKeys, encKey),
+					Permissions:       t.convertPermission(invite.Permissions),
+					InviteId:          invite.InviteId,
+				},
+			},
+		}
+	case ch.UserConfirm != nil:
+		confirm := ch.UserConfirm
+
+		convCh = &pb.ACLChangeACLContentValue{
+			Value: &pb.ACLChangeACLContentValueValueOfUserConfirm{
+				UserConfirm: &pb.ACLChangeUserConfirm{
+					Identity:  t.keychain.GetIdentity(confirm.Identity),
+					UserAddId: confirm.UserAddId,
+				},
+			},
+		}
+	case ch.UserPermissionChange != nil:
+		permissionChange := ch.UserPermissionChange
+
+		convCh = &pb.ACLChangeACLContentValue{
+			Value: &pb.ACLChangeACLContentValueValueOfUserPermissionChange{
+				UserPermissionChange: &pb.ACLChangeUserPermissionChange{
+					Identity:    t.keychain.GetIdentity(permissionChange.Identity),
+					Permissions: t.convertPermission(permissionChange.Permission),
+				},
+			},
+		}
+	case ch.UserRemove != nil:
+		remove := ch.UserRemove
+
+		newReadKey := t.keychain.GetKey(remove.NewReadKey).(*SymKey)
+
+		var replaces []*pb.ACLChangeReadKeyReplace
+		for _, id := range remove.IdentitiesLeft {
+			identity := t.keychain.GetIdentity(id)
+			encKey := t.keychain.EncryptionKeys[id]
+			rawEncKey, _ := encKey.GetPublic().Raw()
+			encReadKey, err := encKey.GetPublic().Encrypt(newReadKey.Key.Bytes())
+			if err != nil {
+				panic(err)
+			}
+			replaces = append(replaces, &pb.ACLChangeReadKeyReplace{
+				Identity:         identity,
+				EncryptionKey:    rawEncKey,
+				EncryptedReadKey: encReadKey,
+			})
+		}
+
+		convCh = &pb.ACLChangeACLContentValue{
+			Value: &pb.ACLChangeACLContentValueValueOfUserRemove{
+				UserRemove: &pb.ACLChangeUserRemove{
+					Identity:        t.keychain.GetIdentity(remove.RemovedIdentity),
+					ReadKeyReplaces: replaces,
+				},
+			},
+		}
+	}
+	if convCh == nil {
+		panic("cannot have empty acl change")
+	}
+
+	return convCh
+}
+
+func (t *ThreadBuilder) encryptReadKeys(keys []string, encKey keys.EncryptionPrivKey) (enc [][]byte) {
+	for _, k := range keys {
+		realKey := t.keychain.GetKey(k).(*SymKey).Key.Bytes()
+		res, err := encKey.GetPublic().Encrypt(realKey)
+		if err != nil {
+			panic(err)
+		}
+
+		enc = append(enc, res)
+	}
+	return
+}
+
+func (t *ThreadBuilder) convertPermission(perm string) pb.ACLChangeUserPermissions {
+	switch perm {
+	case "admin":
+		return pb.ACLChange_Admin
+	case "writer":
+		return pb.ACLChange_Writer
+	case "reader":
+		return pb.ACLChange_Reader
+	default:
+		panic(fmt.Sprintf("incorrect permission: %s", perm))
+	}
+}
+
+func (t *ThreadBuilder) traverseFromHeads(f func(t *threadChange) error) error {
+	uniqMap := map[string]struct{}{}
+	stack := make([]string, len(t.orphans), 10)
+	copy(stack, t.orphans)
+	for len(stack) > 0 {
+		id := stack[len(stack)-1]
+		stack = stack[:len(stack)-1]
+		if _, exists := uniqMap[id]; exists {
+			continue
+		}
+
+		ch := t.allChanges[id]
+		uniqMap[id] = struct{}{}
+		if err := f(ch); err != nil {
+			return err
+		}
+
+		for _, prev := range ch.ACLChange.TreeHeadIds {
+			stack = append(stack, prev)
+		}
+	}
+	return nil
+}
+
+func (t *ThreadBuilder) parseUpdates(updates []*Update) {
+	for _, update := range updates {
+		useCase := &updateUseCase{
+			changes: map[string]*threadChange{},
+		}
+		for _, ch := range update.Changes {
+			newChange := t.parseChange(ch)
+			useCase.changes[newChange.id] = newChange
+		}
+		for _, node := range update.Graph {
+			rec := useCase.changes[node.Id]
+			rec.AclHeadIds = node.ACLHeads
+			rec.TreeHeadIds = node.TreeHeads
+			rec.SnapshotBaseId = node.BaseSnapshot
+		}
+
+		t.updates[update.UseCase] = useCase
+	}
+}
+
+func (t *ThreadBuilder) parseGraph(thread *YMLThread) {
+	for _, node := range thread.Graph {
+		rec := t.allChanges[node.Id]
+		rec.AclHeadIds = node.ACLHeads
+		rec.TreeHeadIds = node.TreeHeads
+		rec.SnapshotBaseId = node.BaseSnapshot
+	}
+}
+
+func (t *ThreadBuilder) parseOrphans(thread *YMLThread) {
+	t.orphans = thread.Orphans
+}
+
+func (t *ThreadBuilder) parseHeader(thread *YMLThread) {
+	t.header = &threadpb.ThreadHeader{
+		FirstChangeId: thread.Header.FirstChangeId,
+		IsWorkspace:   thread.Header.IsWorkspace,
+	}
+}

testutils/threadbuilder/threadbuildergraph.go

@@ -0,0 +1,11 @@
+//go:build ((!linux && !darwin) || android || ios || nographviz) && !amd64
+// +build !linux,!darwin android ios nographviz
+// +build !amd64
+
+package threadbuilder
+
+import "fmt"
+
+func (t *ThreadBuilder) Graph() (string, error) {
+	return "", fmt.Errorf("building graphs is not supported")
+}

testutils/threadbuilder/threadbuildergraph_nix.go

@@ -0,0 +1,163 @@
+//go:build (linux || darwin) && !android && !ios && !nographviz && (amd64 || arm64)
+// +build linux darwin
+// +build !android
+// +build !ios
+// +build !nographviz
+// +build amd64 arm64
+
+package threadbuilder
+
+import (
+	"fmt"
+
+	"github.com/gogo/protobuf/proto"
+	"strings"
+	"unicode"
+
+	"github.com/awalterschulze/gographviz"
+
+	testpb "github.com/anytypeio/go-anytype-infrastructure-experiments/testutils/testchanges/pb"
+)
+
+// To quickly look at visualized string you can use https://dreampuf.github.io/GraphvizOnline
+
+type EdgeParameters struct {
+	style string
+	color string
+	label string
+}
+
+func (t *ThreadBuilder) Graph() (string, error) {
+	// TODO: check updates on https://github.com/goccy/go-graphviz/issues/52 or make a fix yourself to use better library here
+	graph := gographviz.NewGraph()
+	graph.SetName("G")
+	graph.SetDir(true)
+	var nodes = make(map[string]struct{})
+
+	var addNodes = func(r *threadChange) error {
+		// TODO: revisit function after checking
+
+		style := "solid"
+		if r.GetAclData() != nil {
+			style = "filled"
+		} else if r.changesDataDecrypted != nil {
+			style = "dashed"
+		}
+
+		var chSymbs []string
+		if r.changesDataDecrypted != nil {
+			res := &testpb.PlainTextChangeData{}
+			err := proto.Unmarshal(r.changesDataDecrypted, res)
+			if err != nil {
+				return err
+			}
+
+			for _, chc := range res.Content {
+				tp := fmt.Sprintf("%T", chc.Value)
+				tp = strings.Replace(tp, "ChangeContentValueOf", "", 1)
+				res := ""
+				for _, ts := range tp {
+					if unicode.IsUpper(ts) {
+						res += string(ts)
+					}
+				}
+				chSymbs = append(chSymbs, res)
+			}
+		}
+		if r.GetAclData() != nil {
+			for _, chc := range r.GetAclData().AclContent {
+				tp := fmt.Sprintf("%T", chc.Value)
+				tp = strings.Replace(tp, "ACLChangeACLContentValueValueOf", "", 1)
+				res := ""
+				for _, ts := range tp {
+					if unicode.IsUpper(ts) {
+						res += string(ts)
+					}
+				}
+				chSymbs = append(chSymbs, res)
+			}
+		}
+
+		shortId := r.id
+		label := fmt.Sprintf("Id: %s\nChanges: %s\n",
+			shortId,
+			strings.Join(chSymbs, ","),
+		)
+		e := graph.AddNode("G", "\""+r.id+"\"", map[string]string{
+			"label": "\"" + label + "\"",
+			"style": "\"" + style + "\"",
+		})
+		if e != nil {
+			return e
+		}
+		nodes[r.id] = struct{}{}
+		return nil
+	}
+
+	var createEdge = func(firstId, secondId string, params EdgeParameters) error {
+		_, exists := nodes[firstId]
+		if !exists {
+			return fmt.Errorf("no such node")
+		}
+		_, exists = nodes[secondId]
+		if !exists {
+			return fmt.Errorf("no previous node")
+		}
+
+		err := graph.AddEdge("\""+firstId+"\"", "\""+secondId+"\"", true, map[string]string{
+			"color": params.color,
+			"style": params.style,
+		})
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	var addLinks = func(t *threadChange) error {
+		for _, prevId := range t.AclHeadIds {
+			err := createEdge(t.id, prevId, EdgeParameters{
+				style: "dashed",
+				color: "red",
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		for _, prevId := range t.TreeHeadIds {
+			err := createEdge(t.id, prevId, EdgeParameters{
+				style: "dashed",
+				color: "blue",
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		if t.SnapshotBaseId != "" {
+			err := createEdge(t.id, t.SnapshotBaseId, EdgeParameters{
+				style: "bold",
+				color: "blue",
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	err := t.traverseFromHeads(addNodes)
+	if err != nil {
+		return "", err
+	}
+
+	err = t.traverseFromHeads(addLinks)
+	if err != nil {
+		return "", err
+	}
+
+	return graph.String(), nil
+}

testutils/threadbuilder/ymlentities.go

@@ -0,0 +1,117 @@
+package threadbuilder
+
+type ThreadDescription struct {
+	Author string `yaml:"author"`
+}
+
+type Keys struct {
+	Enc  []string `yaml:"Enc"`
+	Sign []string `yaml:"Sign"`
+	Read []string `yaml:"Read"`
+}
+
+type ACLSnapshot struct {
+	UserStates []struct {
+		Identity          string   `yaml:"identity"`
+		EncryptionKey     string   `yaml:"encryptionKey"`
+		EncryptedReadKeys []string `yaml:"encryptedReadKeys"`
+		Permissions       string   `yaml:"permission"`
+		IsConfirmed       bool     `yaml:"isConfirmed"`
+	} `yaml:"userStates"`
+}
+
+type PlainTextSnapshot struct {
+	Text string `yaml:"text"`
+}
+
+type ACLChange struct {
+	UserAdd *struct {
+		Identity          string   `yaml:"identity"`
+		EncryptionKey     string   `yaml:"encryptionKey"`
+		EncryptedReadKeys []string `yaml:"encryptedReadKeys"`
+		Permission        string   `yaml:"permission"`
+	} `yaml:"userAdd"`
+
+	UserJoin *struct {
+		Identity          string   `yaml:"identity"`
+		EncryptionKey     string   `yaml:"encryptionKey"`
+		AcceptSignature   string   `yaml:"acceptSignature"`
+		InviteId          string   `yaml:"inviteId"`
+		EncryptedReadKeys []string `yaml:"encryptedReadKeys"`
+	} `yaml:"userJoin"`
+
+	UserInvite *struct {
+		AcceptKey         string   `yaml:"acceptKey"`
+		EncryptionKey     string   `yaml:"encryptionKey"`
+		EncryptedReadKeys []string `yaml:"encryptedReadKeys"`
+		Permissions       string   `yaml:"permissions"`
+		InviteId          string   `yaml:"inviteId"`
+	} `yaml:"userInvite"`
+
+	UserConfirm *struct {
+		Identity  string `yaml:"identity"`
+		UserAddId string `yaml:"UserAddId"`
+	} `yaml:"userConfirm"`
+
+	UserRemove *struct {
+		RemovedIdentity string   `yaml:"removedIdentity"`
+		NewReadKey      string   `yaml:"newReadKey"`
+		IdentitiesLeft  []string `yaml:"identitiesLeft"`
+	} `yaml:"userRemove"`
+
+	UserPermissionChange *struct {
+		Identity   string `yaml:"identity"`
+		Permission string `yaml:"permission"`
+	}
+}
+
+type PlainTextChange struct {
+	TextAppend *struct {
+		Text string `yaml:"text"`
+	} `yaml:"textAppend"`
+}
+
+type GraphNode struct {
+	Id           string   `yaml:"id"`
+	BaseSnapshot string   `yaml:"baseSnapshot"`
+	AclSnapshot  string   `yaml:"aclSnapshot"`
+	ACLHeads     []string `yaml:"aclHeads"`
+	TreeHeads    []string `yaml:"treeHeads"`
+}
+
+type Change struct {
+	Id       string `yaml:"id"`
+	Identity string `yaml:"identity"`
+
+	AclSnapshot *ACLSnapshot       `yaml:"aclSnapshot"`
+	Snapshot    *PlainTextSnapshot `yaml:"snapshot"`
+	AclChanges  []*ACLChange       `yaml:"aclChanges"`
+	Changes     []*PlainTextChange `yaml:"changes"`
+
+	ReadKey string `yaml:"readKey"`
+}
+
+type Header struct {
+	FirstChangeId string `yaml:"firstChangeId"`
+	IsWorkspace   bool   `yaml:"isWorkspace"`
+}
+
+type Update struct {
+	UseCase string       `yaml:"useCase"`
+	Changes []*Change    `yaml:"changes"`
+	Graph   []*GraphNode `yaml:"graph"`
+}
+
+type YMLThread struct {
+	Description *ThreadDescription `yaml:"thread"`
+	Changes     []*Change          `yaml:"changes"`
+	Updates     []*Update          `yaml:"updates"`
+
+	Keys Keys `yaml:"keys"`
+
+	Graph []*GraphNode `yaml:"graph"`
+
+	Heads   []string `yaml:"heads"`
+	Orphans []string `yaml:"orphans"`
+	Header  *Header  `yaml:"header"`
+}

testutils/threadbuilder/ymlentities_test.go

@@ -0,0 +1,12 @@
+package threadbuilder
+
+import (
+	"fmt"
+	"testing"
+)
+
+func Test_YamlParse(t *testing.T) {
+	tb, _ := NewThreadBuilderWithTestName("userjoinexampleupdate.yml")
+	gr, _ := tb.Graph()
+	fmt.Println(gr)
+}

testutils/yamltests/invalidsnapshotexample.yml

@@ -0,0 +1,126 @@
+thread:
+  author: A
+changes:
+  - id: A.1.1
+    identity: A
+    aclSnapshot:
+      userStates:
+        - identity: A
+          encryptionKey: key.Enc.A
+          encryptedReadKeys: [key.Read.1]
+          permission: admin
+        - identity: B
+          encryptionKey: key.Enc.B
+          encryptedReadKeys: [key.Read.1]
+          permission: admin
+    snapshot:
+      text: "some text"
+    aclChanges:
+      - userAdd:
+          identity: A
+          permission: admin
+          encryptionKey: key.Enc.A
+          encryptedReadKeys: [key.Read.1]
+      - userAdd:
+          identity: B
+          permission: admin
+          encryptionKey: key.Enc.B
+          encryptedReadKeys: [key.Read.1]
+    readKey: key.Read.1
+  - id: A.1.2
+    identity: A
+    aclSnapshot:
+      userStates:
+        - identity: A
+          encryptionKey: key.Enc.A
+          encryptedReadKeys: [key.Read.1]
+          permission: admin
+        - identity: B
+          encryptionKey: key.Enc.B
+          encryptedReadKeys: [key.Read.1]
+          permission: admin
+        - identity: D
+          encryptionKey: key.Enc.D
+          encryptedReadKeys: [ key.Read.1 ]
+          permission: admin
+    snapshot:
+      text: "some text"
+    aclChanges:
+      - userAdd:
+          identity: D
+          permission: admin
+          encryptionKey: key.Enc.D
+          encryptedReadKeys: [key.Read.1]
+    readKey: key.Read.1
+  - id: A.1.3
+    identity: A
+    aclChanges:
+      - userAdd:
+          identity: E
+          permission: admin
+          encryptionKey: key.Enc.E
+          encryptedReadKeys: [key.Read.1]
+    readKey: key.Read.1
+  - id: B.1.1
+    identity: B
+    aclChanges:
+      - userAdd:
+          identity: C
+          permission: admin
+          encryptionKey: key.Enc.C
+          encryptedReadKeys: [ key.Read.1 ]
+    readKey: key.Read.1
+  - id: B.1.2
+    identity: B
+    aclChanges:
+      - userAdd:
+          identity: F
+          permission: admin
+          encryptionKey: key.Enc.F
+          encryptedReadKeys: [ key.Read.1 ]
+    readKey: key.Read.1
+keys:
+  Enc:
+    - A
+    - B
+    - C
+    - D
+    - E
+    - F
+  Sign:
+    - A
+    - B
+    - C
+    - D
+    - E
+    - F
+  Read:
+    - 1
+    - 2
+graph:
+  - id: A.1.1
+    baseSnapshot: A.1.1
+    aclSnapshot: A.1.1
+  - id: A.1.2
+    baseSnapshot: A.1.1
+    aclHeads: [B.1.1]
+    treeHeads: [B.1.1]
+  - id: B.1.1
+    baseSnapshot: A.1.1
+    aclHeads: [A.1.1]
+    treeHeads: [A.1.1]
+  - id: B.1.2
+    baseSnapshot: A.1.2
+    aclHeads: [A.1.2]
+    treeHeads: [A.1.2]
+  - id: A.1.3
+    baseSnapshot: A.1.2
+    aclHeads: [A.1.2]
+    treeHeads: [A.1.2]
+header:
+  firstChangeId: A.1.1
+  isWorkspace: false
+orphans:
+  - A.1.3
+  - B.1.2
+

testutils/yamltests/path.go

@@ -0,0 +1,15 @@
+package yamltests
+
+import (
+	"path/filepath"
+	"runtime"
+)
+
+var (
+	_, b, _, _ = runtime.Caller(0)
+	basepath   = filepath.Dir(b)
+)
+
+func Path() string {
+	return basepath
+}

testutils/yamltests/userjoinexample.yml

@@ -0,0 +1,107 @@
+thread:
+  author: A
+changes:
+  - id: A.1.1
+    identity: A
+    aclSnapshot:
+      userStates:
+        - identity: A
+          encryptionKey: key.Enc.A
+          encryptedReadKeys: [key.Read.1]
+          permission: admin
+    snapshot:
+      text: "some text"
+    aclChanges:
+      - userAdd:
+          identity: A
+          permission: admin
+          encryptionKey: key.Enc.A
+          encryptedReadKeys: [key.Read.1]
+    changes:
+      - textAppend:
+          text: "some text"
+    readKey: key.Read.1
+  - id: A.1.2
+    identity: A
+    aclChanges:
+      - userInvite:
+          acceptKey: key.Sign.Onetime1
+          encryptionKey: key.Enc.Onetime1
+          encryptedReadKeys: [key.Read.1]
+          permissions: writer
+          inviteId: A.1.2
+      - userAdd:
+          identity: C
+          permission: reader
+          encryptionKey: key.Enc.C
+          encryptedReadKeys: [ key.Read.1 ]
+    readKey: key.Read.1
+  - id: A.1.3
+    identity: A
+    changes:
+      - textAppend:
+          text: "second"
+    readKey: key.Read.1
+  - id: B.1.1
+    identity: B
+    aclChanges:
+      - userJoin:
+          identity: B
+          encryptionKey: key.Enc.B
+          acceptSignature: key.Sign.Onetime1
+          inviteId: A.1.2
+          encryptedReadKeys: [key.Read.1]
+    readKey: key.Read.1
+  - id: B.1.2
+    identity: B
+    changes:
+      - textAppend:
+          text: "first"
+    readKey: key.Read.1
+  - id: C.1.1
+    identity: C
+    changes:
+      - textAppend:
+          text: "third"
+    readKey: key.Read.1
+keys:
+  Enc:
+    - A
+    - B
+    - C
+    - Onetime1
+  Sign:
+    - A
+    - B
+    - C
+    - Onetime1
+  Read:
+    - 1
+graph:
+  - id: A.1.1
+    baseSnapshot: A.1.1
+  - id: A.1.2
+    baseSnapshot: A.1.1
+    aclHeads: [A.1.1]
+    treeHeads: [A.1.1]
+  - id: B.1.1
+    baseSnapshot: A.1.1
+    aclHeads: [A.1.2]
+    treeHeads: [A.1.2]
+  - id: B.1.2
+    baseSnapshot: A.1.1
+    aclHeads: [B.1.1]
+    treeHeads: [B.1.1]
+  - id: A.1.3 # this should be invalid, because it is based on one of the invalid changes
+    baseSnapshot: A.1.1
+    aclHeads: [B.1.1]
+    treeHeads: [B.1.2, C.1.1]
+  - id: C.1.1 # this should be invalid, because C is a reader
+    baseSnapshot: A.1.1
+    aclHeads: [B.1.1]
+    treeHeads: [B.1.1]
+header:
+  firstChangeId: A.1.1
+  isWorkspace: false
+orphans:
+  - "A.1.3"

testutils/yamltests/userjoinexampleupdate.yml

@@ -0,0 +1,152 @@
+thread:
+  author: A
+changes:
+  - id: A.1.1
+    identity: A
+    aclSnapshot:
+      userStates:
+        - identity: A
+          encryptionKey: key.Enc.A
+          encryptedReadKeys: [key.Read.1]
+          permission: admin
+    snapshot:
+      text: "some text"
+    aclChanges:
+      - userAdd:
+          identity: A
+          permission: admin
+          encryptionKey: key.Enc.A
+          encryptedReadKeys: [key.Read.1]
+    changes:
+      - textAppend:
+          text: "some text"
+    readKey: key.Read.1
+  - id: A.1.2
+    identity: A
+    aclChanges:
+      - userInvite:
+          acceptKey: key.Sign.Onetime1
+          encryptionKey: key.Enc.Onetime1
+          encryptedReadKeys: [key.Read.1]
+          permissions: writer
+          inviteId: A.1.2
+      - userAdd:
+          identity: C
+          permission: reader
+          encryptionKey: key.Enc.C
+          encryptedReadKeys: [ key.Read.1 ]
+    readKey: key.Read.1
+  - id: A.1.3
+    identity: A
+    changes:
+      - textAppend:
+          text: "second"
+    readKey: key.Read.1
+  - id: B.1.1
+    identity: B
+    aclChanges:
+      - userJoin:
+          identity: B
+          encryptionKey: key.Enc.B
+          acceptSignature: key.Sign.Onetime1
+          inviteId: A.1.2
+          encryptedReadKeys: [key.Read.1]
+    readKey: key.Read.1
+  - id: B.1.2
+    identity: B
+    changes:
+      - textAppend:
+          text: "first"
+    readKey: key.Read.1
+  - id: C.1.1
+    identity: C
+    changes:
+      - textAppend:
+          text: "third"
+    readKey: key.Read.1
+keys:
+  Enc:
+    - A
+    - B
+    - C
+    - D
+    - Onetime1
+  Sign:
+    - A
+    - B
+    - C
+    - D
+    - Onetime1
+  Read:
+    - 1
+graph:
+  - id: A.1.1
+    baseSnapshot: A.1.1
+  - id: A.1.2
+    baseSnapshot: A.1.1
+    aclHeads: [A.1.1]
+    treeHeads: [A.1.1]
+  - id: B.1.1
+    baseSnapshot: A.1.1
+    aclHeads: [A.1.2]
+    treeHeads: [A.1.2]
+  - id: B.1.2
+    baseSnapshot: A.1.1
+    aclHeads: [B.1.1]
+    treeHeads: [B.1.1]
+  - id: A.1.3 # this should be invalid, because it is based on one of the invalid changes
+    baseSnapshot: A.1.1
+    aclHeads: [B.1.1]
+    treeHeads: [B.1.2, C.1.1]
+  - id: C.1.1 # this should be invalid, because C is a reader
+    baseSnapshot: A.1.1
+    aclHeads: [B.1.1]
+    treeHeads: [B.1.1]
+header:
+  firstChangeId: A.1.1
+  isWorkspace: false
+orphans:
+  - "A.1.3"
+updates:
+  - useCase: append
+    changes:
+    - id: B.1.3
+      identity: B
+      changes:
+        - textAppend:
+            text: "second"
+      readKey: key.Read.1
+    - id: A.1.4
+      identity: A
+      aclChanges:
+        - userAdd:
+            identity: D
+            permission: writer
+            encryptionKey: key.Enc.D
+            encryptedReadKeys: [ key.Read.1 ]
+      readKey: key.Read.1
+    graph:
+      - id: B.1.3
+        baseSnapshot: A.1.1
+        aclHeads: [ B.1.1 ]
+        treeHeads: [ B.1.2 ]
+      - id: A.1.4
+        baseSnapshot: A.1.1
+        aclHeads: [ B.1.1 ]
+        treeHeads: [ B.1.3 ]
+  - useCase: rebuild
+    changes:
+      - id: A.1.4
+        identity: A
+        aclChanges:
+          - userAdd:
+              identity: D
+              permission: writer
+              encryptionKey: key.Enc.D
+              encryptedReadKeys: [ key.Read.1 ]
+        readKey: key.Read.1
+    graph:
+      - id: A.1.4
+        baseSnapshot: A.1.1
+        aclHeads: [ A.1.1 ]
+        treeHeads: [ A.1.1 ]

testutils/yamltests/userremovebeforeexample.yml

@@ -0,0 +1,109 @@
+thread:
+  author: A
+changes:
+  - id: A.1.1
+    identity: A
+    aclSnapshot:
+      userStates:
+        - identity: A
+          encryptionKey: key.Enc.A
+          encryptedReadKeys: [key.Read.1]
+          permission: admin
+        - identity: B
+          encryptionKey: key.Enc.B
+          encryptedReadKeys: [key.Read.1]
+          permission: admin
+    snapshot:
+      text: "some text"
+    aclChanges:
+      - userAdd:
+          identity: A
+          permission: admin
+          encryptionKey: key.Enc.A
+          encryptedReadKeys: [key.Read.1]
+      - userAdd:
+          identity: B
+          permission: admin
+          encryptionKey: key.Enc.B
+          encryptedReadKeys: [key.Read.1]
+    changes:
+      - textAppend:
+          text: "some text"
+    readKey: key.Read.1
+  - id: A.1.2
+    identity: A
+    aclChanges:
+      - userRemove:
+          removedIdentity: B
+          newReadKey: key.Read.2
+          identitiesLeft: [A, C]
+    readKey: key.Read.2
+  - id: A.1.3
+    identity: A
+    aclChanges:
+      - userAdd:
+          identity: E
+          permission: admin
+          encryptionKey: key.Enc.E
+          encryptedReadKeys: [key.Read.1, key.Read.2]
+    readKey: key.Read.2
+  - id: B.1.1
+    identity: B
+    aclChanges:
+      - userAdd:
+          identity: C
+          permission: admin
+          encryptionKey: key.Enc.C
+          encryptedReadKeys: [ key.Read.1 ]
+    readKey: key.Read.1
+  - id: B.1.2
+    identity: B
+    aclChanges:
+      - userAdd:
+          identity: D
+          permission: admin
+          encryptionKey: key.Enc.D
+          encryptedReadKeys: [ key.Read.1 ]
+    readKey: key.Read.1
+keys:
+  Enc:
+    - A
+    - B
+    - C
+    - D
+    - E
+  Sign:
+    - A
+    - B
+    - C
+    - D
+    - E
+  Read:
+    - 1
+    - 2
+graph:
+  - id: A.1.1
+    baseSnapshot: A.1.1
+    aclSnapshot: A.1.1
+  - id: A.1.2
+    baseSnapshot: A.1.1
+    aclHeads: [B.1.1]
+    treeHeads: [B.1.1]
+  - id: B.1.1
+    baseSnapshot: A.1.1
+    aclHeads: [A.1.1]
+    treeHeads: [A.1.1]
+  - id: B.1.2
+    baseSnapshot: A.1.1
+    aclHeads: [B.1.1]
+    treeHeads: [B.1.1]
+  - id: A.1.3
+    baseSnapshot: A.1.1
+    aclHeads: [A.1.2]
+    treeHeads: [A.1.2]
+orphans:
+  - "A.1.3"
+  - "B.1.2"
+header:
+  firstChangeId: A.1.1
+  isWorkspace: false

testutils/yamltests/userremoveexample.yml

@@ -0,0 +1,110 @@
+thread:
+  author: A
+changes:
+  - id: A.1.1
+    identity: A
+    aclSnapshot:
+      userStates:
+        - identity: A
+          encryptionKey: key.Enc.A
+          encryptedReadKeys: [key.Read.1]
+          permission: admin
+    snapshot:
+      text: "some text"
+    aclChanges:
+      - userAdd:
+          identity: A
+          permission: admin
+          encryptionKey: key.Enc.A
+          encryptedReadKeys: [key.Read.1]
+    changes:
+      - textAppend:
+          text: "some text"
+    readKey: key.Read.1
+  - id: A.1.2
+    identity: A
+    aclChanges:
+      - userInvite:
+          acceptKey: key.Sign.Onetime1
+          encryptionKey: key.Enc.Onetime1
+          encryptedReadKeys: [key.Read.1]
+          permissions: writer
+          inviteId: A.1.2
+    readKey: key.Read.1
+  - id: A.1.3
+    identity: A
+    aclChanges:
+      - userRemove:
+          removedIdentity: B
+          newReadKey: key.Read.2
+          identitiesLeft: [A]
+    readKey: key.Read.2
+  - id: A.1.4
+    identity: A
+    changes:
+      - textAppend:
+          text: "first"
+    readKey: key.Read.2
+  - id: B.1.1
+    identity: B
+    aclChanges:
+      - userJoin:
+          identity: B
+          encryptionKey: key.Enc.B
+          acceptSignature: key.Sign.Onetime1
+          inviteId: A.1.2
+          encryptedReadKeys: [key.Read.1]
+    readKey: key.Read.1
+  - id: B.1.2
+    identity: B
+    changes:
+      - textAppend:
+          text: "second"
+    readKey: key.Read.1
+keys:
+  Enc:
+    - A
+    - B
+    - Onetime1
+  Sign:
+    - A
+    - B
+    - Onetime1
+  Read:
+    - 1
+    - 2
+graph:
+  - id: A.1.1
+    baseSnapshot: A.1.1
+    aclSnapshot: A.1.1
+  - id: A.1.2
+    baseSnapshot: A.1.1
+    aclSnapshot: A.1.1
+    aclHeads: [A.1.1]
+    treeHeads: [A.1.1]
+  - id: B.1.1
+    baseSnapshot: A.1.1
+    aclSnapshot: A.1.1
+    aclHeads: [A.1.2]
+    treeHeads: [A.1.2]
+  - id: B.1.2
+    baseSnapshot: A.1.1
+    aclSnapshot: A.1.1
+    aclHeads: [B.1.1]
+    treeHeads: [B.1.1]
+  - id: A.1.3
+    baseSnapshot: A.1.1
+    aclSnapshot: A.1.1
+    aclHeads: [B.1.1]
+    treeHeads: [B.1.1]
+  - id: A.1.4
+    baseSnapshot: A.1.1
+    aclSnapshot: A.1.1
+    aclHeads: [A.1.3]
+    treeHeads: [A.1.3]
+orphans:
+  - "A.1.4"
+  - "B.1.2"
+header:
+  firstChangeId: A.1.1
+  isWorkspace: false

testutils/yamltests/validsnapshotexample.yml

@@ -0,0 +1,133 @@
+thread:
+  author: A
+changes:
+  - id: A.1.1
+    identity: A
+    aclSnapshot:
+      userStates:
+        - identity: A
+          encryptionKey: key.Enc.A
+          encryptedReadKeys: [key.Read.1]
+          permission: admin
+        - identity: B
+          encryptionKey: key.Enc.B
+          encryptedReadKeys: [key.Read.1]
+          permission: admin
+    snapshot:
+      text: "some text"
+    aclChanges:
+      - userAdd:
+          identity: A
+          permission: admin
+          encryptionKey: key.Enc.A
+          encryptedReadKeys: [key.Read.1]
+      - userAdd:
+          identity: B
+          permission: admin
+          encryptionKey: key.Enc.B
+          encryptedReadKeys: [key.Read.1]
+    readKey: key.Read.1
+    changes:
+      - textAppend:
+          text: "some text"
+  - id: A.1.2
+    identity: A
+    aclSnapshot:
+      userStates:
+        - identity: A
+          encryptionKey: key.Enc.A
+          encryptedReadKeys: [key.Read.1]
+          permission: admin
+        - identity: B
+          encryptionKey: key.Enc.B
+          encryptedReadKeys: [key.Read.1]
+          permission: admin
+        - identity: C
+          encryptionKey: key.Enc.C
+          encryptedReadKeys: [ key.Read.1 ]
+          permission: admin
+        - identity: D
+          encryptionKey: key.Enc.D
+          encryptedReadKeys: [ key.Read.1 ]
+          permission: admin
+    snapshot:
+      text: "some text"
+    aclChanges:
+      - userAdd:
+          identity: D
+          permission: admin
+          encryptionKey: key.Enc.D
+          encryptedReadKeys: [key.Read.1]
+    readKey: key.Read.1
+  - id: A.1.3
+    identity: A
+    aclChanges:
+      - userAdd:
+          identity: E
+          permission: admin
+          encryptionKey: key.Enc.E
+          encryptedReadKeys: [key.Read.1]
+    readKey: key.Read.1
+  - id: B.1.1
+    identity: B
+    aclChanges:
+      - userAdd:
+          identity: C
+          permission: admin
+          encryptionKey: key.Enc.C
+          encryptedReadKeys: [ key.Read.1 ]
+    readKey: key.Read.1
+  - id: B.1.2
+    identity: B
+    aclChanges:
+      - userAdd:
+          identity: F
+          permission: admin
+          encryptionKey: key.Enc.F
+          encryptedReadKeys: [ key.Read.1 ]
+    readKey: key.Read.1
+keys:
+  Enc:
+    - A
+    - B
+    - C
+    - D
+    - E
+    - F
+  Sign:
+    - A
+    - B
+    - C
+    - D
+    - E
+    - F
+  Read:
+    - 1
+    - 2
+graph:
+  - id: A.1.1
+    baseSnapshot: A.1.1
+    aclSnapshot: A.1.1
+  - id: A.1.2
+    baseSnapshot: A.1.1
+    aclHeads: [B.1.1]
+    treeHeads: [B.1.1]
+  - id: B.1.1
+    baseSnapshot: A.1.1
+    aclHeads: [A.1.1]
+    treeHeads: [A.1.1]
+  - id: B.1.2
+    baseSnapshot: A.1.2
+    aclHeads: [A.1.2]
+    treeHeads: [A.1.2]
+  - id: A.1.3
+    baseSnapshot: A.1.2
+    aclHeads: [A.1.2]
+    treeHeads: [A.1.2]
+orphans:
+  - "A.1.3"
+  - "B.1.2"
+header:
+  firstChangeId: A.1.1
+  isWorkspace: false
+

thread/inmemory.go

@@ -0,0 +1,132 @@
+package thread
+
+import (
+	"context"
+	"fmt"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/aclchanges"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/thread/pb"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/cid"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice"
+	"github.com/gogo/protobuf/proto"
+	"sync"
+)
+
+type inMemoryThread struct {
+	id      string
+	header  *pb.ThreadHeader
+	heads   []string
+	orphans []string
+	changes map[string]*RawChange
+
+	sync.RWMutex
+}
+
+func NewInMemoryThread(firstChange *RawChange) (Thread, error) {
+	header := &pb.ThreadHeader{
+		FirstChangeId: firstChange.Id,
+		IsWorkspace:   false,
+	}
+	marshalledHeader, err := proto.Marshal(header)
+	if err != nil {
+		return nil, err
+	}
+	threadId, err := cid.NewCIDFromBytes(marshalledHeader)
+	if err != nil {
+		return nil, err
+	}
+
+	changes := make(map[string]*RawChange)
+	changes[firstChange.Id] = firstChange
+
+	return &inMemoryThread{
+		id:      threadId,
+		header:  header,
+		heads:   []string{firstChange.Id},
+		orphans: nil,
+		changes: changes,
+		RWMutex: sync.RWMutex{},
+	}, nil
+}
+
+func (t *inMemoryThread) ID() string {
+	t.RLock()
+	defer t.RUnlock()
+	return t.id
+}
+
+func (t *inMemoryThread) Header() *pb.ThreadHeader {
+	t.RLock()
+	defer t.RUnlock()
+	return t.header
+}
+
+func (t *inMemoryThread) Heads() []string {
+	t.RLock()
+	defer t.RUnlock()
+	return t.heads
+}
+
+func (t *inMemoryThread) Orphans() []string {
+	t.RLock()
+	defer t.RUnlock()
+	return t.orphans
+}
+
+func (t *inMemoryThread) SetHeads(heads []string) {
+	t.Lock()
+	defer t.Unlock()
+	t.heads = t.heads[:0]
+
+	for _, h := range heads {
+		t.heads = append(t.heads, h)
+	}
+}
+
+func (t *inMemoryThread) RemoveOrphans(orphans ...string) {
+	t.Lock()
+	defer t.Unlock()
+	t.orphans = slice.Difference(t.orphans, orphans)
+}
+
+func (t *inMemoryThread) AddOrphans(orphans ...string) {
+	t.Lock()
+	defer t.Unlock()
+	t.orphans = append(t.orphans, orphans...)
+}
+
+func (t *inMemoryThread) AddRawChange(change *RawChange) error {
+	t.Lock()
+	defer t.Unlock()
+	// TODO: better to do deep copy
+	t.changes[change.Id] = change
+	return nil
+}
+
+func (t *inMemoryThread) AddChange(change aclchanges.Change) error {
+	t.Lock()
+	defer t.Unlock()
+	signature := change.Signature()
+	id := change.CID()
+	aclChange := change.ProtoChange()
+
+	fullMarshalledChange, err := proto.Marshal(aclChange)
+	if err != nil {
+		return err
+	}
+	rawChange := &RawChange{
+		Payload:   fullMarshalledChange,
+		Signature: signature,
+		Id:        id,
+	}
+	t.changes[id] = rawChange
+	return nil
+}
+
+func (t *inMemoryThread) GetChange(ctx context.Context, changeId string) (*RawChange, error) {
+	t.RLock()
+	defer t.RUnlock()
+	if res, exists := t.changes[changeId]; exists {
+		return res, nil
+	}
+	return nil, fmt.Errorf("could not get change with id: %s", changeId)
+}

thread/models.go

@@ -0,0 +1,32 @@
+package thread
+
+import (
+	"context"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/thread/pb"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/aclchanges"
+)
+
+// TODO: change methods to have errors as a return parameter, because we will be dealing with a real database
+type Thread interface {
+	ID() string
+
+	Header() *pb.ThreadHeader
+	Heads() []string
+	Orphans() []string
+	SetHeads(heads []string)
+	RemoveOrphans(orphan ...string)
+	AddOrphans(orphan ...string)
+
+	AddRawChange(change *RawChange) error
+	AddChange(change aclchanges.Change) error
+
+	// TODO: have methods with raw changes also
+	GetChange(ctx context.Context, recordID string) (*RawChange, error)
+}
+
+type RawChange struct {
+	Payload   []byte
+	Signature []byte
+	Id        string
+}

thread/pb/protos/thread.proto

@@ -0,0 +1,9 @@
+syntax = "proto3";
+package anytype;
+option go_package = "pb";
+
+message ThreadHeader {
+  string firstChangeId = 1;
+  bool isWorkspace = 2;
+  // TODO: add user identity, signature and nano timestamp
+}

thread/pb/thread.pb.go

@@ -0,0 +1,355 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: thread/pb/protos/thread.proto
+
+package pb
+
+import (
+	fmt "fmt"
+	proto "github.com/gogo/protobuf/proto"
+	io "io"
+	math "math"
+	math_bits "math/bits"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+
+type ThreadHeader struct {
+	FirstChangeId string `protobuf:"bytes,1,opt,name=firstChangeId,proto3" json:"firstChangeId,omitempty"`
+	IsWorkspace   bool   `protobuf:"varint,2,opt,name=isWorkspace,proto3" json:"isWorkspace,omitempty"`
+}
+
+func (m *ThreadHeader) Reset()         { *m = ThreadHeader{} }
+func (m *ThreadHeader) String() string { return proto.CompactTextString(m) }
+func (*ThreadHeader) ProtoMessage()    {}
+func (*ThreadHeader) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b228ffbfd554b168, []int{0}
+}
+func (m *ThreadHeader) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ThreadHeader) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ThreadHeader.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ThreadHeader) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ThreadHeader.Merge(m, src)
+}
+func (m *ThreadHeader) XXX_Size() int {
+	return m.Size()
+}
+func (m *ThreadHeader) XXX_DiscardUnknown() {
+	xxx_messageInfo_ThreadHeader.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ThreadHeader proto.InternalMessageInfo
+
+func (m *ThreadHeader) GetFirstChangeId() string {
+	if m != nil {
+		return m.FirstChangeId
+	}
+	return ""
+}
+
+func (m *ThreadHeader) GetIsWorkspace() bool {
+	if m != nil {
+		return m.IsWorkspace
+	}
+	return false
+}
+
+func init() {
+	proto.RegisterType((*ThreadHeader)(nil), "anytype.ThreadHeader")
+}
+
+func init() { proto.RegisterFile("thread/pb/protos/thread.proto", fileDescriptor_b228ffbfd554b168) }
+
+var fileDescriptor_b228ffbfd554b168 = []byte{
+	// 156 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x2d, 0xc9, 0x28, 0x4a,
+	0x4d, 0x4c, 0xd1, 0x2f, 0x48, 0xd2, 0x2f, 0x28, 0xca, 0x2f, 0xc9, 0x2f, 0xd6, 0x87, 0x08, 0xe8,
+	0x81, 0x79, 0x42, 0xec, 0x89, 0x79, 0x95, 0x25, 0x95, 0x05, 0xa9, 0x4a, 0x61, 0x5c, 0x3c, 0x21,
+	0x60, 0x09, 0x8f, 0xd4, 0xc4, 0x94, 0xd4, 0x22, 0x21, 0x15, 0x2e, 0xde, 0xb4, 0xcc, 0xa2, 0xe2,
+	0x12, 0xe7, 0x8c, 0xc4, 0xbc, 0xf4, 0x54, 0xcf, 0x14, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xce, 0x20,
+	0x54, 0x41, 0x21, 0x05, 0x2e, 0xee, 0xcc, 0xe2, 0xf0, 0xfc, 0xa2, 0xec, 0xe2, 0x82, 0xc4, 0xe4,
+	0x54, 0x09, 0x26, 0x05, 0x46, 0x0d, 0x8e, 0x20, 0x64, 0x21, 0x27, 0x99, 0x13, 0x8f, 0xe4, 0x18,
+	0x2f, 0x3c, 0x92, 0x63, 0x7c, 0xf0, 0x48, 0x8e, 0x71, 0xc2, 0x63, 0x39, 0x86, 0x0b, 0x8f, 0xe5,
+	0x18, 0x6e, 0x3c, 0x96, 0x63, 0x88, 0x62, 0x2a, 0x48, 0x4a, 0x62, 0x03, 0xbb, 0xc2, 0x18, 0x10,
+	0x00, 0x00, 0xff, 0xff, 0x2a, 0xae, 0x9d, 0xc2, 0xa6, 0x00, 0x00, 0x00,
+}
+
+func (m *ThreadHeader) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ThreadHeader) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ThreadHeader) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.IsWorkspace {
+		i--
+		if m.IsWorkspace {
+			dAtA[i] = 1
+		} else {
+			dAtA[i] = 0
+		}
+		i--
+		dAtA[i] = 0x10
+	}
+	if len(m.FirstChangeId) > 0 {
+		i -= len(m.FirstChangeId)
+		copy(dAtA[i:], m.FirstChangeId)
+		i = encodeVarintThread(dAtA, i, uint64(len(m.FirstChangeId)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func encodeVarintThread(dAtA []byte, offset int, v uint64) int {
+	offset -= sovThread(v)
+	base := offset
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return base
+}
+func (m *ThreadHeader) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.FirstChangeId)
+	if l > 0 {
+		n += 1 + l + sovThread(uint64(l))
+	}
+	if m.IsWorkspace {
+		n += 2
+	}
+	return n
+}
+
+func sovThread(x uint64) (n int) {
+	return (math_bits.Len64(x|1) + 6) / 7
+}
+func sozThread(x uint64) (n int) {
+	return sovThread(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *ThreadHeader) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowThread
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ThreadHeader: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ThreadHeader: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field FirstChangeId", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowThread
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthThread
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthThread
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.FirstChangeId = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field IsWorkspace", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowThread
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				v |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.IsWorkspace = bool(v != 0)
+		default:
+			iNdEx = preIndex
+			skippy, err := skipThread(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthThread
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipThread(dAtA []byte) (n int, err error) {
+	l := len(dAtA)
+	iNdEx := 0
+	depth := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowThread
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowThread
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if dAtA[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+		case 1:
+			iNdEx += 8
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowThread
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if length < 0 {
+				return 0, ErrInvalidLengthThread
+			}
+			iNdEx += length
+		case 3:
+			depth++
+		case 4:
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupThread
+			}
+			depth--
+		case 5:
+			iNdEx += 4
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthThread
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
+	}
+	return 0, io.ErrUnexpectedEOF
+}
+
+var (
+	ErrInvalidLengthThread        = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowThread          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupThread = fmt.Errorf("proto: unexpected end of group")
+)

thread/threadid.go

@@ -0,0 +1,72 @@
+package thread
+
+import (
+	"crypto/rand"
+	"encoding/binary"
+	"fmt"
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
+	"hash/fnv"
+
+	"github.com/textileio/go-threads/core/thread"
+)
+
+func CreateACLThreadID(k keys.SigningPubKey, docType uint16) (thread.ID, error) {
+	rndlen := 32
+	buf := make([]byte, 8+rndlen)
+
+	// adding random bytes in the end
+	_, err := rand.Read(buf[8 : 8+rndlen])
+	if err != nil {
+		panic("random read failed")
+	}
+
+	keyBytes, err := k.Bytes()
+	if err != nil {
+		return thread.Undef, err
+	}
+
+	hasher := fnv.New64()
+	hasher.Write(keyBytes)
+	res := hasher.Sum64()
+
+	// putting hash of the pubkey in the beginning
+	binary.LittleEndian.PutUint64(buf[:8], res)
+
+	return threadIDFromBytes(docType, buf)
+}
+
+func VerifyACLThreadID(k keys.SigningPubKey, threadId thread.ID) (bool, error) {
+	bytes := threadId.Bytes()
+	pubKeyBytes := threadId.Bytes()[len(bytes)-40 : len(bytes)-32]
+	hash := binary.LittleEndian.Uint64(pubKeyBytes)
+
+	keyBytes, err := k.Bytes()
+	if err != nil {
+		return false, err
+	}
+
+	hasher := fnv.New64()
+	hasher.Write(keyBytes)
+	realHash := hasher.Sum64()
+
+	return hash == realHash, nil
+}
+
+func threadIDFromBytes(
+	docType uint16,
+	b []byte) (thread.ID, error) {
+	blen := len(b)
+
+	// two 8 bytes (max) numbers plus num
+	buf := make([]byte, 2*binary.MaxVarintLen64+blen)
+	n := binary.PutUvarint(buf, thread.V1)
+	n += binary.PutUvarint(buf[n:], uint64(thread.AccessControlled))
+	n += binary.PutUvarint(buf[n:], uint64(docType))
+
+	cn := copy(buf[n:], b)
+	if cn != blen {
+		return thread.Undef, fmt.Errorf("copy length is inconsistent")
+	}
+
+	return thread.Cast(buf[:n+blen])
+}

thread/threadid_test.go

@@ -0,0 +1,27 @@
+package thread
+
+import (
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
+	"testing"
+)
+
+func TestCreateACLThreadIDVerify(t *testing.T) {
+	_, pubKey, err := keys.GenerateRandomEd25519KeyPair()
+	if err != nil {
+		t.Fatalf("should not return error after generating key pair: %v", err)
+	}
+
+	thread, err := CreateACLThreadID(pubKey, 1)
+	if err != nil {
+		t.Fatalf("should not return error after generating thread: %v", err)
+	}
+
+	verified, err := VerifyACLThreadID(pubKey, thread)
+	if err != nil {
+		t.Fatalf("verification should not return error: %v", err)
+	}
+
+	if !verified {
+		t.Fatalf("the thread should be verified")
+	}
+}

util/cid/cid.go

@@ -0,0 +1,22 @@
+package cid
+
+import (
+	"github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+)
+
+func NewCIDFromBytes(data []byte) (string, error) {
+	hash, err := mh.Sum(data, mh.SHA2_256, -1)
+	if err != nil {
+		return "", err
+	}
+	return cid.NewCidV1(cid.DagCBOR, hash).String(), nil
+}
+
+func VerifyCID(data []byte, id string) bool {
+	hash, err := mh.Sum(data, mh.SHA2_256, -1)
+	if err != nil {
+		return false
+	}
+	return cid.NewCidV1(cid.DagCBOR, hash).String() == id
+}

util/crc16/crc16.go

@@ -0,0 +1,122 @@
+// Package crc16 is implementation according to CCITT standards.
+//
+// Note by @antirez: this is actually the XMODEM CRC 16 algorithm, using the
+// following parameters:
+//
+// Name                       : "XMODEM", also known as "ZMODEM", "CRC-16/ACORN"
+// Width                      : 16 bit
+// Poly                       : 1021 (That is actually x^16 + x^12 + x^5 + 1)
+// Initialization             : 0000
+// Reflect Input byte         : False
+// Reflect Output CRC         : False
+// Xor constant to output CRC : 0000
+// Output for "123456789"     : 31C3
+//
+// ported from the c++ code in the stellar-core codebase
+// (https://github.com/stellar/stellar-core).  The code is licensed
+// as:
+/*
+ * Copyright 2001-2010 Georges Menie (www.menie.org)
+ * Copyright 2010-2012 Salvatore Sanfilippo (adapted to Redis coding style)
+ * Copyright 2015 Stellar Development Foundation (ported to go)
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the University of California, Berkeley nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package crc16
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+)
+
+// ErrInvalidChecksum is returned when Validate determines either the checksum
+// or the payload has been corrupted
+var ErrInvalidChecksum = fmt.Errorf("invalid checksum")
+
+var crc16tab = [256]uint16{
+	0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
+	0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
+	0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
+	0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
+	0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
+	0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
+	0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
+	0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
+	0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
+	0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
+	0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
+	0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
+	0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
+	0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
+	0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
+	0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
+	0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
+	0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
+	0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
+	0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
+	0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
+	0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
+	0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
+	0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
+	0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
+	0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
+	0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
+	0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
+	0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
+	0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
+	0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
+	0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0,
+}
+
+// Checksum returns the 2-byte checksum for the provided data
+func Checksum(data []byte) []byte {
+	var crc uint16
+	var out bytes.Buffer
+	for _, b := range data {
+		crc = ((crc << 8) & 0xffff) ^ crc16tab[((crc>>8)^uint16(b))&0x00FF]
+	}
+
+	err := binary.Write(&out, binary.LittleEndian, crc)
+	if err != nil {
+		panic(err)
+	}
+
+	return out.Bytes()
+}
+
+// Validate returns an error if the provided checksum does not match
+// the calculated checksum of the provided data
+func Validate(data []byte, expected []byte) error {
+
+	actual := Checksum(data)
+
+	// validate the provided checksum against the calculated
+	if !bytes.Equal(actual, expected) {
+		return ErrInvalidChecksum
+	}
+
+	return nil
+}

util/keys/keys.go

@@ -0,0 +1,256 @@
+package keys
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha512"
+	"crypto/subtle"
+	"crypto/x509"
+	"errors"
+	"io"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/strkey"
+	"github.com/libp2p/go-libp2p-core/crypto"
+	crypto_pb "github.com/libp2p/go-libp2p-core/crypto/pb"
+)
+
+type SigningPubKey crypto.PubKey
+type SigningPrivKey crypto.PrivKey
+
+var MinRsaKeyBits = 2048
+
+var ErrKeyLengthTooSmall = errors.New("error key length too small")
+
+type Key interface {
+	Equals(Key) bool
+
+	Raw() ([]byte, error)
+}
+
+type EncryptionPrivKey interface {
+	Key
+
+	Decrypt([]byte) ([]byte, error)
+	GetPublic() EncryptionPubKey
+}
+
+type EncryptionPubKey interface {
+	Key
+
+	Encrypt(data []byte) ([]byte, error)
+}
+
+type EncryptionRsaPrivKey struct {
+	privKey rsa.PrivateKey
+}
+
+type EncryptionRsaPubKey struct {
+	pubKey rsa.PublicKey
+}
+
+func (e *EncryptionRsaPubKey) Equals(key Key) bool {
+	other, ok := (key).(*EncryptionRsaPubKey)
+	if !ok {
+		return keyEquals(e, key)
+	}
+
+	return e.pubKey.N.Cmp(other.pubKey.N) == 0 && e.pubKey.E == other.pubKey.E
+}
+
+func (e *EncryptionRsaPubKey) Raw() ([]byte, error) {
+	return x509.MarshalPKIXPublicKey(&e.pubKey)
+}
+
+func (e *EncryptionRsaPubKey) Encrypt(data []byte) ([]byte, error) {
+	hash := sha512.New()
+	return rsa.EncryptOAEP(hash, rand.Reader, &e.pubKey, data, nil)
+}
+
+func (e *EncryptionRsaPrivKey) Equals(key Key) bool {
+	other, ok := (key).(*EncryptionRsaPrivKey)
+	if !ok {
+		return keyEquals(e, key)
+	}
+
+	return e.privKey.N.Cmp(other.privKey.N) == 0 && e.privKey.E == other.privKey.E
+}
+
+func (e *EncryptionRsaPrivKey) Raw() ([]byte, error) {
+	b := x509.MarshalPKCS1PrivateKey(&e.privKey)
+	return b, nil
+}
+
+func (e *EncryptionRsaPrivKey) Decrypt(bytes []byte) ([]byte, error) {
+	hash := sha512.New()
+	return rsa.DecryptOAEP(hash, rand.Reader, &e.privKey, bytes, nil)
+}
+
+func (e *EncryptionRsaPrivKey) GetPublic() EncryptionPubKey {
+	return &EncryptionRsaPubKey{pubKey: e.privKey.PublicKey}
+}
+
+func GenerateRandomRSAKeyPair(bits int) (EncryptionPrivKey, EncryptionPubKey, error) {
+	return GenerateRSAKeyPair(bits, rand.Reader)
+}
+
+func GenerateRSAKeyPair(bits int, src io.Reader) (EncryptionPrivKey, EncryptionPubKey, error) {
+	if bits < MinRsaKeyBits {
+		return nil, nil, ErrKeyLengthTooSmall
+	}
+	priv, err := rsa.GenerateKey(src, bits)
+	if err != nil {
+		return nil, nil, err
+	}
+	pk := priv.PublicKey
+	return &EncryptionRsaPrivKey{privKey: *priv}, &EncryptionRsaPubKey{pubKey: pk}, nil
+}
+
+func NewEncryptionRsaPrivKeyFromBytes(bytes []byte) (EncryptionPrivKey, error) {
+	sk, err := x509.ParsePKCS1PrivateKey(bytes)
+	if err != nil {
+		return nil, err
+	}
+	if sk.N.BitLen() < MinRsaKeyBits {
+		return nil, ErrKeyLengthTooSmall
+	}
+	return &EncryptionRsaPrivKey{privKey: *sk}, nil
+}
+
+func NewEncryptionRsaPubKeyFromBytes(bytes []byte) (EncryptionPubKey, error) {
+	pub, err := x509.ParsePKIXPublicKey(bytes)
+	if err != nil {
+		return nil, err
+	}
+	pk, ok := pub.(*rsa.PublicKey)
+	if !ok {
+		return nil, errors.New("not actually an rsa public key")
+	}
+	if pk.N.BitLen() < MinRsaKeyBits {
+		return nil, ErrKeyLengthTooSmall
+	}
+
+	return &EncryptionRsaPubKey{pubKey: *pk}, nil
+}
+
+func NewSigningEd25519PubKeyFromBytes(bytes []byte) (SigningPubKey, error) {
+	return crypto.UnmarshalEd25519PublicKey(bytes)
+}
+
+func GenerateRandomEd25519KeyPair() (SigningPrivKey, SigningPubKey, error) {
+	return crypto.GenerateEd25519Key(rand.Reader)
+}
+
+func keyEquals(k1, k2 Key) bool {
+	a, err := k1.Raw()
+	if err != nil {
+		return false
+	}
+	b, err := k2.Raw()
+	if err != nil {
+		return false
+	}
+	return subtle.ConstantTimeCompare(a, b) == 1
+}
+
+type Ed25519SigningPubKeyDecoder struct{}
+
+func NewEd25519Decoder() *Ed25519SigningPubKeyDecoder {
+	return &Ed25519SigningPubKeyDecoder{}
+}
+
+func (e *Ed25519SigningPubKeyDecoder) DecodeFromBytes(bytes []byte) (SigningPubKey, error) {
+	return NewSigningEd25519PubKeyFromBytes(bytes)
+}
+
+func (e *Ed25519SigningPubKeyDecoder) DecodeFromString(identity string) (SigningPubKey, error) {
+	pubKeyRaw, err := strkey.Decode(0x5b, identity)
+	if err != nil {
+		return nil, err
+	}
+
+	return e.DecodeFromBytes(pubKeyRaw)
+}
+
+func (e *Ed25519SigningPubKeyDecoder) DecodeFromStringIntoBytes(identity string) ([]byte, error) {
+	return strkey.Decode(0x5b, identity)
+}
+
+func (e *Ed25519SigningPubKeyDecoder) EncodeToString(pubkey crypto.PubKey) (string, error) {
+	raw, err := pubkey.Raw()
+	if err != nil {
+		return "", err
+	}
+	return strkey.Encode(0x5b, raw)
+}
+
+type SigningPubKeyDecoder interface {
+	DecodeFromBytes(bytes []byte) (SigningPubKey, error)
+	DecodeFromString(identity string) (SigningPubKey, error)
+	DecodeFromStringIntoBytes(identity string) ([]byte, error)
+}
+
+// Below keys are required for testing and mocking purposes
+
+type EmptyRecorderEncryptionKey struct {
+	recordedEncrypted [][]byte
+	recordedDecrypted [][]byte
+}
+
+func (f *EmptyRecorderEncryptionKey) Equals(key Key) bool {
+	return true
+}
+
+func (f *EmptyRecorderEncryptionKey) Raw() ([]byte, error) {
+	panic("can't get bytes from this key")
+}
+
+func (f *EmptyRecorderEncryptionKey) GetPublic() EncryptionPubKey {
+	panic("this key doesn't have a public key")
+}
+
+func (f *EmptyRecorderEncryptionKey) Encrypt(msg []byte) ([]byte, error) {
+	f.recordedEncrypted = append(f.recordedEncrypted, msg)
+	return msg, nil
+}
+
+func (f *EmptyRecorderEncryptionKey) Decrypt(msg []byte) ([]byte, error) {
+	f.recordedDecrypted = append(f.recordedDecrypted, msg)
+	return msg, nil
+}
+
+type SignatureVerificationPayload struct {
+	message   []byte
+	signature []byte
+}
+
+type EmptyRecorderVerificationKey struct {
+	verifications []SignatureVerificationPayload
+}
+
+func (e *EmptyRecorderVerificationKey) Bytes() ([]byte, error) {
+	panic("can't get bytes from this key")
+}
+
+func (e *EmptyRecorderVerificationKey) Equals(key crypto.Key) bool {
+	return true
+}
+
+func (e *EmptyRecorderVerificationKey) Raw() ([]byte, error) {
+	panic("can't get bytes from this key")
+}
+
+func (e *EmptyRecorderVerificationKey) Type() crypto_pb.KeyType {
+	panic("can't get type from this key")
+}
+
+func (e *EmptyRecorderVerificationKey) Verify(data []byte, sig []byte) (bool, error) {
+	e.verifications = append(e.verifications, SignatureVerificationPayload{
+		message:   data,
+		signature: sig,
+	})
+	return true, nil
+}
+
+func NewMockSigningPubKeyFromBytes(bytes []byte) (SigningPubKey, error) {
+	return &EmptyRecorderVerificationKey{}, nil
+}

util/slice/slice.go

@@ -0,0 +1,120 @@
+package slice
+
+import (
+	"hash/fnv"
+	"math/rand"
+	"sort"
+)
+
+func DifferenceRemovedAdded(a, b []string) (removed []string, added []string) {
+	var amap = map[string]struct{}{}
+	var bmap = map[string]struct{}{}
+
+	for _, item := range a {
+		amap[item] = struct{}{}
+	}
+
+	for _, item := range b {
+		if _, exists := amap[item]; !exists {
+			added = append(added, item)
+		}
+		bmap[item] = struct{}{}
+	}
+
+	for _, item := range a {
+		if _, exists := bmap[item]; !exists {
+			removed = append(removed, item)
+		}
+	}
+	return
+}
+
+func FindPos(s []string, v string) int {
+	for i, sv := range s {
+		if sv == v {
+			return i
+		}
+	}
+	return -1
+}
+
+// Difference returns the elements in `a` that aren't in `b`.
+func Difference(a, b []string) []string {
+	var diff = make([]string, 0, len(a))
+	for _, a1 := range a {
+		if FindPos(b, a1) == -1 {
+			diff = append(diff, a1)
+		}
+	}
+	return diff
+}
+
+func Insert(s []string, pos int, v ...string) []string {
+	if len(s) <= pos {
+		return append(s, v...)
+	}
+	if pos == 0 {
+		return append(v, s[pos:]...)
+	}
+	return append(s[:pos], append(v, s[pos:]...)...)
+}
+
+// Remove reuses provided slice capacity. Provided s slice should not be used after without reassigning to the func return!
+func Remove(s []string, v string) []string {
+	var n int
+	for _, x := range s {
+		if x != v {
+			s[n] = x
+			n++
+		}
+	}
+	return s[:n]
+}
+
+func Filter(vals []string, cond func(string) bool) []string {
+	var result = make([]string, 0, len(vals))
+	for i := range vals {
+		if cond(vals[i]) {
+			result = append(result, vals[i])
+		}
+	}
+	return result
+}
+
+func GetRandomString(s []string, seed string) string {
+	rand.Seed(int64(hash(seed)))
+	return s[rand.Intn(len(s))]
+}
+
+func hash(s string) uint64 {
+	h := fnv.New64a()
+	h.Write([]byte(s))
+	return h.Sum64()
+}
+
+func SortedEquals(s1, s2 []string) bool {
+	if len(s1) != len(s2) {
+		return false
+	}
+	for i := range s1 {
+		if s1[i] != s2[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func UnsortedEquals(s1, s2 []string) bool {
+	if len(s1) != len(s2) {
+		return false
+	}
+
+	s1Sorted := make([]string, len(s1))
+	s2Sorted := make([]string, len(s2))
+	copy(s1Sorted, s1)
+	copy(s2Sorted, s2)
+	sort.Strings(s1Sorted)
+	sort.Strings(s2Sorted)
+
+	return SortedEquals(s1Sorted, s2Sorted)
+}

util/strkey/strkey.go

@@ -0,0 +1,118 @@
+package strkey
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+
+	"github.com/mr-tron/base58/base58"
+
+	"github.com/anytypeio/go-anytype-infrastructure-experiments/util/crc16"
+)
+
+// ErrInvalidVersionByte is returned when the version byte from a provided
+// strkey-encoded string is not one of the valid values.
+var ErrInvalidVersionByte = fmt.Errorf("invalid version byte")
+
+// VersionByte represents one of the possible prefix values for a StrKey base
+// string--the string the when encoded using base58 yields a final StrKey.
+type VersionByte byte
+
+// Decode decodes the provided StrKey into a raw value, checking the checksum
+// and ensuring the expected VersionByte (the version parameter) is the value
+// actually encoded into the provided src string.
+func Decode(expected VersionByte, src string) ([]byte, error) {
+	raw, err := decodeString(src)
+	if err != nil {
+		return nil, err
+	}
+
+	// decode into components
+	version := VersionByte(raw[0])
+	vp := raw[0 : len(raw)-2]
+	payload := raw[1 : len(raw)-2]
+	checksum := raw[len(raw)-2:]
+
+	// ensure version byte is expected
+	if version != expected {
+		return nil, ErrInvalidVersionByte
+	}
+
+	// ensure checksum is valid
+	if err := crc16.Validate(vp, checksum); err != nil {
+		return nil, err
+	}
+
+	// if we made it through the gaunlet, return the decoded value
+	return payload, nil
+}
+
+// MustDecode is like Decode, but panics on error
+func MustDecode(expected VersionByte, src string) []byte {
+	d, err := Decode(expected, src)
+	if err != nil {
+		panic(err)
+	}
+	return d
+}
+
+// Encode encodes the provided data to a StrKey, using the provided version
+// byte.
+func Encode(version VersionByte, src []byte) (string, error) {
+	var raw bytes.Buffer
+
+	// write version byte
+	if err := binary.Write(&raw, binary.LittleEndian, version); err != nil {
+		return "", err
+	}
+
+	// write payload
+	if _, err := raw.Write(src); err != nil {
+		return "", err
+	}
+
+	// calculate and write checksum
+	checksum := crc16.Checksum(raw.Bytes())
+	if _, err := raw.Write(checksum); err != nil {
+		return "", err
+	}
+
+	result := base58.FastBase58Encoding(raw.Bytes())
+	return result, nil
+}
+
+// MustEncode is like Encode, but panics on error
+func MustEncode(version VersionByte, src []byte) string {
+	e, err := Encode(version, src)
+	if err != nil {
+		panic(err)
+	}
+	return e
+}
+
+// Version extracts and returns the version byte from the provided source
+// string.
+func Version(src string) (VersionByte, error) {
+	raw, err := decodeString(src)
+	if err != nil {
+		return VersionByte(0), err
+	}
+
+	return VersionByte(raw[0]), nil
+}
+
+// decodeString decodes a base58 string into the raw bytes, and ensures it could
+// potentially be strkey encoded (i.e. it has both a version byte and a
+// checksum, neither of which are explicitly checked by this func)
+func decodeString(src string) ([]byte, error) {
+	raw, err := base58.FastBase58Decoding(src)
+	if err != nil {
+		return nil, fmt.Errorf("base58 decode failed: %s", err)
+	}
+
+	if len(raw) < 3 {
+		return nil, fmt.Errorf("encoded value is %d bytes; minimum valid length is 3", len(raw))
+	}
+
+	return raw, nil
+}