benjamyn/any-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixing several copy issues and other stuff in doc tree

Browse Source
This commit is contained in:
mcrakhman 2022-09-05 12:35:54 +02:00
parent 90cee00319
commit 9a1dc89a99
No known key found for this signature in database
GPG Key ID: DED12CFEF5B8396B
3 changed files with 65 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
pkg/acl/tree/change.go
View File

@ -1,6 +1,7 @@
package tree package tree
import ( import (
"errors"
"fmt" "fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/proto"
@ -8,6 +9,8 @@ import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/symmetric" "github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/symmetric"
) )
var ErrIncorrectSignature = errors.New("change has incorrect signature")
type ChangeContent struct { type ChangeContent struct {
ChangesData proto.Marshaler ChangesData proto.Marshaler
ACLData *aclpb.ACLChangeACLData ACLData *aclpb.ACLChangeACLData
@ -21,8 +24,8 @@ type Change struct {
Id string Id string
SnapshotId string SnapshotId string
IsSnapshot bool IsSnapshot bool
DecryptedChange []byte // TODO: check if we need it DecryptedChange []byte // TODO: check if we need it
ParsedModel interface{} ParsedModel interface{} // TODO: check if we need it
// iterator helpers // iterator helpers
visited bool visited bool
@ -57,8 +60,7 @@ func NewChangeFromRaw(rawChange *aclpb.RawChange) (*Change, error) {
return nil, err return nil, err
} }
ch := NewChange(rawChange.Id, unmarshalled) ch := NewChange(rawChange.Id, unmarshalled, rawChange.Signature)
ch.Sign = rawChange.Signature
return ch, nil return ch, nil
} }
@ -66,7 +68,7 @@ func NewVerifiedChangeFromRaw(
rawChange *aclpb.RawChange, rawChange *aclpb.RawChange,
kch *keychain) (*Change, error) { kch *keychain) (*Change, error) {
unmarshalled := &aclpb.Change{} unmarshalled := &aclpb.Change{}
err := proto.Unmarshal(rawChange.Payload, unmarshalled) ch, err := NewChangeFromRaw(rawChange)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -81,13 +83,13 @@ func NewVerifiedChangeFromRaw(
return nil, err return nil, err
} }
if !res { if !res {
return nil, fmt.Errorf("change has incorrect signature") return nil, ErrIncorrectSignature
} }
return NewChange(rawChange.Id, unmarshalled), nil return ch, nil
} }
func NewChange(id string, ch *aclpb.Change) *Change { func NewChange(id string, ch *aclpb.Change, signature []byte) *Change {
return &Change{ return &Change{
Next: nil, Next: nil,
PreviousIds: ch.TreeHeadIds, PreviousIds: ch.TreeHeadIds,
@ -95,6 +97,7 @@ func NewChange(id string, ch *aclpb.Change) *Change {
Content: ch, Content: ch,
SnapshotId: ch.SnapshotBaseId, SnapshotId: ch.SnapshotBaseId,
IsSnapshot: ch.IsSnapshot, IsSnapshot: ch.IsSnapshot,
Sign: signature,
} }
} }

99
pkg/acl/tree/doctree.go
View File

@ -25,8 +25,10 @@ type RWLocker interface {
RUnlock() RUnlock()
} }
var ErrHasInvalidChanges = errors.New("the change is invalid") var (
var ErrNoCommonSnapshot = errors.New("trees doesn't have a common snapshot") ErrHasInvalidChanges = errors.New("the change is invalid")
ErrNoCommonSnapshot = errors.New("trees doesn't have a common snapshot")
)
type AddResultSummary int type AddResultSummary int
@ -40,7 +42,7 @@ type AddResult struct {
OldHeads []string OldHeads []string
Heads []string Heads []string
Added []*aclpb.RawChange Added []*aclpb.RawChange
// TODO: add summary for changes
Summary AddResultSummary Summary AddResultSummary
} }
@ -64,10 +66,12 @@ type docTree struct {
kch *keychain kch *keychain
// buffers // buffers
difSnapshotBuf []*aclpb.RawChange difSnapshotBuf []*aclpb.RawChange
tmpChangesBuf []*Change tmpChangesBuf []*Change
newSnapshots []*Change newSnapshotsBuf []*Change
notSeenIdxBuf []int notSeenIdxBuf []int
snapshotPath []string
sync.RWMutex sync.RWMutex
} }
@ -128,7 +132,7 @@ func (d *docTree) rebuildFromStorage(aclList list.ACLList, newChanges []*Change)
d.tree, err = d.treeBuilder.Build(newChanges) d.tree, err = d.treeBuilder.Build(newChanges)
if err != nil { if err != nil {
return err return
} }
// during building the tree we may have marked some changes as possible roots, // during building the tree we may have marked some changes as possible roots,
@ -198,9 +202,8 @@ func (d *docTree) AddContent(ctx context.Context, aclList list.ACLList, content
return nil, err return nil, err
} }
docChange := NewChange(id, aclChange) docChange := NewChange(id, aclChange, signature)
docChange.ParsedModel = content docChange.ParsedModel = content
docChange.Sign = signature
if content.IsSnapshot { if content.IsSnapshot {
// clearing tree, because we already fixed everything in the last snapshot // clearing tree, because we already fixed everything in the last snapshot
@ -269,10 +272,11 @@ func (d *docTree) addRawChanges(ctx context.Context, aclList list.ACLList, rawCh
d.tmpChangesBuf = d.tmpChangesBuf[:0] d.tmpChangesBuf = d.tmpChangesBuf[:0]
d.notSeenIdxBuf = d.notSeenIdxBuf[:0] d.notSeenIdxBuf = d.notSeenIdxBuf[:0]
d.difSnapshotBuf = d.difSnapshotBuf[:0] d.difSnapshotBuf = d.difSnapshotBuf[:0]
d.newSnapshots = d.newSnapshots[:0] d.newSnapshotsBuf = d.newSnapshotsBuf[:0]
prevHeads := d.tree.Heads() // this will be returned to client so we shouldn't use buffer here
// TODO: if we can use new snapshot -> update tree, check if some snapshot, dfsPrev? prevHeadsCopy := make([]string, 0, len(d.tree.Heads()))
copy(prevHeadsCopy, d.tree.Heads())
// filtering changes, verifying and unmarshalling them // filtering changes, verifying and unmarshalling them
for idx, ch := range rawChanges { for idx, ch := range rawChanges {
@ -287,7 +291,7 @@ func (d *docTree) addRawChanges(ctx context.Context, aclList list.ACLList, rawCh
} }
if change.IsSnapshot { if change.IsSnapshot {
d.newSnapshots = append(d.newSnapshots, change) d.newSnapshotsBuf = append(d.newSnapshotsBuf, change)
} }
d.tmpChangesBuf = append(d.tmpChangesBuf, change) d.tmpChangesBuf = append(d.tmpChangesBuf, change)
d.notSeenIdxBuf = append(d.notSeenIdxBuf, idx) d.notSeenIdxBuf = append(d.notSeenIdxBuf, idx)
@ -296,13 +300,19 @@ func (d *docTree) addRawChanges(ctx context.Context, aclList list.ACLList, rawCh
// if no new changes, then returning // if no new changes, then returning
if len(d.notSeenIdxBuf) == 0 { if len(d.notSeenIdxBuf) == 0 {
addResult = AddResult{ addResult = AddResult{
OldHeads: prevHeads, OldHeads: prevHeadsCopy,
Heads: prevHeads, Heads: prevHeadsCopy,
Summary: AddResultSummaryNothing, Summary: AddResultSummaryNothing,
} }
return return
} }
headsCopy := func() []string {
newHeads := make([]string, 0, len(d.tree.Heads()))
copy(newHeads, d.tree.Heads())
return newHeads
}
// returns changes that we added to the tree // returns changes that we added to the tree
getAddedChanges := func() []*aclpb.RawChange { getAddedChanges := func() []*aclpb.RawChange {
var added []*aclpb.RawChange var added []*aclpb.RawChange
@ -330,7 +340,7 @@ func (d *docTree) addRawChanges(ctx context.Context, aclList list.ACLList, rawCh
if ch.SnapshotId == d.tree.RootId() { if ch.SnapshotId == d.tree.RootId() {
return false return false
} }
for _, sn := range d.newSnapshots { for _, sn := range d.newSnapshotsBuf {
// if change refers to newly received snapshot // if change refers to newly received snapshot
if ch.SnapshotId == sn.Id { if ch.SnapshotId == sn.Id {
return false return false
@ -350,8 +360,8 @@ func (d *docTree) addRawChanges(ctx context.Context, aclList list.ACLList, rawCh
} }
addResult = AddResult{ addResult = AddResult{
OldHeads: prevHeads, OldHeads: prevHeadsCopy,
Heads: d.tree.Heads(), Heads: headsCopy(),
Added: getAddedChanges(), Added: getAddedChanges(),
Summary: AddResultSummaryRebuild, Summary: AddResultSummaryRebuild,
} }
@ -364,8 +374,8 @@ func (d *docTree) addRawChanges(ctx context.Context, aclList list.ACLList, rawCh
switch mode { switch mode {
case Nothing: case Nothing:
addResult = AddResult{ addResult = AddResult{
OldHeads: prevHeads, OldHeads: prevHeadsCopy,
Heads: prevHeads, Heads: prevHeadsCopy,
Summary: AddResultSummaryNothing, Summary: AddResultSummaryNothing,
} }
return return
@ -381,8 +391,8 @@ func (d *docTree) addRawChanges(ctx context.Context, aclList list.ACLList, rawCh
} }
addResult = AddResult{ addResult = AddResult{
OldHeads: prevHeads, OldHeads: prevHeadsCopy,
Heads: d.tree.Heads(), Heads: headsCopy(),
Added: getAddedChanges(), Added: getAddedChanges(),
Summary: AddResultSummaryAppend, Summary: AddResultSummaryAppend,
} }
@ -417,7 +427,9 @@ func (d *docTree) Close() error {
} }
func (d *docTree) SnapshotPath() []string { func (d *docTree) SnapshotPath() []string {
// TODO: think about caching this if d.snapshotPathIsActual() {
return d.snapshotPath
}
var path []string var path []string
// TODO: think that the user may have not all of the snapshots locally // TODO: think that the user may have not all of the snapshots locally
@ -430,20 +442,22 @@ func (d *docTree) SnapshotPath() []string {
path = append(path, currentSnapshotId) path = append(path, currentSnapshotId)
currentSnapshotId = sn.SnapshotId currentSnapshotId = sn.SnapshotId
} }
d.snapshotPath = path
return path return path
} }
func (d *docTree) ChangesAfterCommonSnapshot(theirPath []string) ([]*aclpb.RawChange, error) { func (d *docTree) ChangesAfterCommonSnapshot(theirPath []string) ([]*aclpb.RawChange, error) {
var ( var (
isNewDocument = len(theirPath) == 0 needFullDocument = len(theirPath) == 0
ourPath = d.SnapshotPath() ourPath = d.SnapshotPath()
// by default returning everything we have // by default returning everything we have
commonSnapshot = ourPath[len(ourPath)-1] commonSnapshot = ourPath[len(ourPath)-1]
err error err error
) )
// if this is non-empty request // if this is non-empty request
if !isNewDocument { if !needFullDocument {
commonSnapshot, err = commonSnapshotForTwoPaths(ourPath, theirPath) commonSnapshot, err = commonSnapshotForTwoPaths(ourPath, theirPath)
if err != nil { if err != nil {
return nil, err return nil, err
@ -457,17 +471,13 @@ func (d *docTree) ChangesAfterCommonSnapshot(theirPath []string) ([]*aclpb.RawCh
Debug("getting all changes from common snapshot") Debug("getting all changes from common snapshot")
if commonSnapshot == d.tree.RootId() { if commonSnapshot == d.tree.RootId() {
return d.getChangesFromTree(isNewDocument) return d.getChangesFromTree()
} else { } else {
return d.getChangesFromDB(commonSnapshot, isNewDocument) return d.getChangesFromDB(commonSnapshot, needFullDocument)
} }
} }
func (d *docTree) getChangesFromTree(isNewDocument bool) (marshalledChanges []*aclpb.RawChange, err error) { func (d *docTree) getChangesFromTree() (rawChanges []*aclpb.RawChange, err error) {
if !isNewDocument {
// ignoring root change
d.tree.Root().visited = true
}
d.tree.dfsPrev(d.tree.HeadsChanges(), func(ch *Change) bool { d.tree.dfsPrev(d.tree.HeadsChanges(), func(ch *Change) bool {
var marshalled []byte var marshalled []byte
marshalled, err = ch.Content.Marshal() marshalled, err = ch.Content.Marshal()
@ -479,18 +489,14 @@ func (d *docTree) getChangesFromTree(isNewDocument bool) (marshalledChanges []*a
Signature: ch.Signature(), Signature: ch.Signature(),
Id: ch.Id, Id: ch.Id,
} }
marshalledChanges = append(marshalledChanges, raw) rawChanges = append(rawChanges, raw)
return true return true
}, func(changes []*Change) {}) }, func(changes []*Change) {})
if err != nil {
return nil, err
}
return return
} }
func (d *docTree) getChangesFromDB(commonSnapshot string, isNewDocument bool) (marshalledChanges []*aclpb.RawChange, err error) { func (d *docTree) getChangesFromDB(commonSnapshot string, needStartSnapshot bool) (rawChanges []*aclpb.RawChange, err error) {
var rawChanges []*aclpb.RawChange
load := func(id string) (*Change, error) { load := func(id string) (*Change, error) {
raw, err := d.treeStorage.GetRawChange(context.Background(), id) raw, err := d.treeStorage.GetRawChange(context.Background(), id)
if err != nil { if err != nil {
@ -508,18 +514,19 @@ func (d *docTree) getChangesFromDB(commonSnapshot string, isNewDocument bool) (m
_, err = d.treeBuilder.dfs(d.tree.Heads(), commonSnapshot, load) _, err = d.treeBuilder.dfs(d.tree.Heads(), commonSnapshot, load)
if err != nil { if err != nil {
return nil, err return
} }
if isNewDocument { if needStartSnapshot {
// adding snapshot to raw changes // adding snapshot to raw changes
_, err = load(commonSnapshot) _, err = load(commonSnapshot)
if err != nil {
return nil, err
}
} }
return rawChanges, nil return
}
func (d *docTree) snapshotPathIsActual() bool {
return len(d.snapshotPath) != 0 && d.snapshotPath[len(d.snapshotPath)-1] == d.tree.RootId()
} }
func (d *docTree) DebugDump() (string, error) { func (d *docTree) DebugDump() (string, error) {

4
pkg/acl/tree/treebuilder.go
View File

@ -6,8 +6,6 @@ import (
"fmt" "fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app/logger" "github.com/anytypeio/go-anytype-infrastructure-experiments/app/logger"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage" "github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice" "github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice"
"go.uber.org/zap" "go.uber.org/zap"
"time" "time"
@ -133,7 +131,7 @@ func (tb *treeBuilder) loadChange(id string) (ch *Change, err error) {
return nil, err return nil, err
} }
ch, err = NewVerifiedChangeFromRaw(change, tb.identityKeys, tb.signingPubKeyDecoder) ch, err = NewVerifiedChangeFromRaw(change, tb.kch)
if err != nil { if err != nil {
return nil, err return nil, err
} }