diff --git a/go.mod b/go.mod index 8594463c..65b61f86 100644 --- a/go.mod +++ b/go.mod @@ -4,22 +4,26 @@ go 1.18 require ( github.com/awalterschulze/gographviz v0.0.0-20190522210029-fa59802746ab - github.com/cheggaaa/mb v1.0.3 + github.com/cespare/xxhash v1.1.0 github.com/goccy/go-graphviz v0.0.9 github.com/gogo/protobuf v1.3.2 + github.com/huandu/skiplist v1.2.0 github.com/ipfs/go-cid v0.1.0 github.com/libp2p/go-libp2p v0.20.3 github.com/libp2p/go-libp2p-core v0.16.1 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multibase v0.0.3 github.com/multiformats/go-multihash v0.1.0 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.8.0 + github.com/zeebo/blake3 v0.2.3 go.uber.org/zap v1.21.0 + gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 gopkg.in/yaml.v3 v3.0.1 storj.io/drpc v0.0.32 ) require ( + github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/btcsuite/btcd v0.22.1 // indirect github.com/btcsuite/btcd/btcec/v2 v2.1.3 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect @@ -27,7 +31,7 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/fogleman/gg v1.3.0 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect - github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/klauspost/cpuid/v2 v2.0.12 // indirect github.com/libp2p/go-buffer-pool v0.0.2 // indirect github.com/libp2p/go-openssl v0.0.7 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect @@ -48,7 +52,6 @@ require ( golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect lukechampine.com/blake3 v1.1.6 // indirect ) diff --git a/go.sum b/go.sum index 12ac3e57..6c6a5a5f 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,6 @@ +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= +github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/awalterschulze/gographviz v0.0.0-20190522210029-fa59802746ab h1:+cdNqtOJWjvepyhxy23G7z7vmpYCoC65AP0nqi1f53s= github.com/awalterschulze/gographviz v0.0.0-20190522210029-fa59802746ab/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -8,8 +11,8 @@ github.com/btcsuite/btcd/btcec/v2 v2.1.3 h1:xM/n3yIhHAhHy04z4i43C8p4ehixJZMsnrVJ github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= -github.com/cheggaaa/mb v1.0.3 h1:03ksWum+6kHclB+kjwKMaBtgl5gtNYUwNpxsHQciKe8= -github.com/cheggaaa/mb v1.0.3/go.mod h1:NUl0GBtFLlfg2o6iZwxzcG7Lslc2wV/ADTFbLXtVPE4= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA= github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -27,6 +30,10 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c= +github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U= +github.com/huandu/skiplist v1.2.0 h1:gox56QD77HzSC0w+Ws3MH3iie755GBJU1OER3h5VsYw= +github.com/huandu/skiplist v1.2.0/go.mod h1:7v3iFjLcSAzO4fN5B8dvebvo/qsfumiLiDXMrPiHF9w= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.1.0 h1:YN33LQulcRHjfom/i25yoOZR4Telp1Hr/2RU3d0PnC0= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= @@ -34,8 +41,9 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -85,18 +93,28 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= +github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= github.com/zeebo/errs v1.2.2 h1:5NFypMTuSdoySVTqlNs1dEoU21QVamMQJxW/Fii5O7g= github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -164,6 +182,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/util/ldiff/diff.go b/util/ldiff/diff.go new file mode 100644 index 00000000..29fa04f7 --- /dev/null +++ b/util/ldiff/diff.go @@ -0,0 +1,288 @@ +// Package ldiff provides a container of elements with fixed id and changeable content. +// Diff can calculate the difference with another diff container (you can make it remote) with minimum hops and traffic. +package ldiff + +import ( + "bytes" + "context" + "errors" + "fmt" + "github.com/cespare/xxhash" + "github.com/huandu/skiplist" + "github.com/zeebo/blake3" + "math" + "sync" +) + +// New creates new Diff container +// +// divideFactor - means how many hashes you want to ask for once +// +// it must be 2 or greater +// normal value usually between 4 and 64 +// +// compareThreshold - means the maximum count of elements remote diff will send directly +// +// if elements under range will be more - remote diff will send only hash +// it must be 1 or greater +// normal value between 8 and 64 +// +// Less threshold and divideFactor - less traffic but more requests +func New(divideFactor, compareThreshold int) Diff { + if divideFactor < 2 { + divideFactor = 2 + } + if compareThreshold < 1 { + compareThreshold = 1 + } + d := &diff{ + divideFactor: divideFactor, + compareThreshold: compareThreshold, + } + d.sl = skiplist.New(d) + return d +} + +var hashersPool = &sync.Pool{ + New: func() any { + return blake3.New() + }, +} + +var ErrElementNotFound = errors.New("element not found") + +// Element of data +type Element struct { + Id string + Head string +} + +// Range request to get RangeResult +type Range struct { + From, To uint64 + Limit int +} + +// RangeResult response for Range +type RangeResult struct { + Hash []byte + Elements []Element + Count int +} + +type element struct { + Element + hash uint64 +} + +// Diff contains elements and can compare it with Remote diff +type Diff interface { + Remote + // Set adds or update element in container + Set(e Element) + // RemoveId removes element by id + RemoveId(id string) error + // Diff makes diff with remote container + Diff(ctx context.Context, dl Remote) (newIds, changedIds, removedIds []string, err error) +} + +// Remote interface for using in the Diff +type Remote interface { + // Ranges calculates given ranges and return results + Ranges(ctx context.Context, ranges []Range, resBuf []RangeResult) (results []RangeResult, err error) +} + +type diff struct { + sl *skiplist.SkipList + divideFactor int + compareThreshold int + mu sync.RWMutex +} + +// Compare implements skiplist interface +func (d *diff) Compare(lhs, rhs interface{}) int { + lhe := lhs.(*element) + rhe := rhs.(*element) + if lhe.Id == rhe.Id { + return 0 + } + if lhe.hash > rhe.hash { + return 1 + } else if lhe.hash < rhe.hash { + return -1 + } + if lhe.Id > rhe.Id { + return 1 + } else { + return -1 + } +} + +// CalcScore implements skiplist interface +func (d *diff) CalcScore(key interface{}) float64 { + return 0 +} + +// Set adds or update element in container +func (d *diff) Set(e Element) { + d.mu.Lock() + defer d.mu.Unlock() + el := &element{Element: e, hash: xxhash.Sum64([]byte(e.Id))} + d.sl.Remove(el) + d.sl.Set(el, nil) +} + +// RemoveId removes element by id +func (d *diff) RemoveId(id string) error { + d.mu.Lock() + defer d.mu.Unlock() + el := &element{Element: Element{ + Id: id, + }, hash: xxhash.Sum64([]byte(id))} + if d.sl.Remove(el) == nil { + return ErrElementNotFound + } + return nil +} + +func (d *diff) getRange(r Range) (rr RangeResult) { + hasher := hashersPool.Get().(*blake3.Hasher) + defer hashersPool.Put(hasher) + hasher.Reset() + + el := d.sl.Find(&element{hash: r.From}) + rr.Elements = make([]Element, 0, r.Limit) + var overfill bool + for el != nil && el.Key().(*element).hash <= r.To { + elem := el.Key().(*element).Element + el = el.Next() + + hasher.WriteString(elem.Id) + hasher.WriteString(elem.Head) + rr.Count++ + if !overfill { + if len(rr.Elements) < r.Limit { + rr.Elements = append(rr.Elements, elem) + } + if len(rr.Elements) == r.Limit && el != nil { + overfill = true + } + } + } + if overfill { + rr.Elements = nil + } + rr.Hash = hasher.Sum(nil) + return +} + +// Ranges calculates given ranges and return results +func (d *diff) Ranges(ctx context.Context, ranges []Range, resBuf []RangeResult) (results []RangeResult, err error) { + d.mu.RLock() + defer d.mu.RUnlock() + results = resBuf[:0] + for _, r := range ranges { + results = append(results, d.getRange(r)) + } + return +} + +type diffCtx struct { + newIds, changedIds, removedIds []string + + toSend, prepare []Range + myRes, otherRes []RangeResult +} + +var errMismatched = errors.New("query and results mismatched") + +// Diff makes diff with remote container +func (d *diff) Diff(ctx context.Context, dl Remote) (newIds, changedIds, removedIds []string, err error) { + d.mu.RLock() + defer d.mu.RUnlock() + + dctx := &diffCtx{} + dctx.toSend = append(dctx.toSend, Range{ + From: 0, + To: math.MaxUint64, + Limit: d.compareThreshold, + }) + for len(dctx.toSend) > 0 { + fmt.Println("fill ranges:", len(dctx.toSend)) + if dctx.myRes, err = d.Ranges(ctx, dctx.toSend, dctx.myRes); err != nil { + return + } + if dctx.otherRes, err = dl.Ranges(ctx, dctx.toSend, dctx.otherRes); err != nil { + return + } + if len(dctx.otherRes) != len(dctx.toSend) || len(dctx.myRes) != len(dctx.toSend) { + err = errMismatched + return + } + for i, r := range dctx.toSend { + d.compareResults(dctx, r, dctx.myRes[i], dctx.otherRes[i]) + } + dctx.toSend, dctx.prepare = dctx.prepare, dctx.toSend + dctx.prepare = dctx.prepare[:0] + } + return dctx.newIds, dctx.changedIds, dctx.removedIds, nil +} + +func (d *diff) compareResults(dctx *diffCtx, r Range, myRes, otherRes RangeResult) { + // both hash equals - do nothing + if bytes.Equal(myRes.Hash, otherRes.Hash) { + return + } + + // both has elements + if len(myRes.Elements) == myRes.Count && len(otherRes.Elements) == otherRes.Count { + d.compareElements(dctx, myRes.Elements, otherRes.Elements) + return + } + + // make more queries + divideFactor := uint64(d.divideFactor) + perRange := (r.To - r.From) / divideFactor + align := ((r.To-r.From)%divideFactor + 1) % divideFactor + if align == 0 { + perRange += 1 + } + var j = r.From + for i := 0; i < d.divideFactor; i++ { + if i == d.divideFactor-1 { + perRange += align + } + dctx.prepare = append(dctx.prepare, Range{From: j, To: j + perRange - 1, Limit: r.Limit}) + j += perRange + } + return +} + +func (d *diff) compareElements(dctx *diffCtx, my, other []Element) { + find := func(list []Element, targetEl Element) (has, eq bool) { + for _, el := range list { + if el.Id == targetEl.Id { + return true, el.Head == targetEl.Head + } + } + return false, false + } + + for _, el := range my { + has, eq := find(other, el) + if !has { + dctx.removedIds = append(dctx.removedIds, el.Id) + continue + } else { + if !eq { + dctx.changedIds = append(dctx.changedIds, el.Id) + } + } + } + + for _, el := range other { + if has, _ := find(my, el); !has { + dctx.newIds = append(dctx.newIds, el.Id) + } + } +} diff --git a/util/ldiff/diff_test.go b/util/ldiff/diff_test.go new file mode 100644 index 00000000..bf6f5e61 --- /dev/null +++ b/util/ldiff/diff_test.go @@ -0,0 +1,101 @@ +package ldiff + +import ( + "context" + "fmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/mgo.v2/bson" + "math" + "testing" +) + +func TestDiff_fillRange(t *testing.T) { + d := New(4, 4).(*diff) + for i := 0; i < 10; i++ { + el := Element{ + Id: fmt.Sprint(i), + Head: fmt.Sprint("h", i), + } + d.Set(el) + } + t.Log(d.sl.Len()) + + t.Run("elements", func(t *testing.T) { + r := Range{From: 0, To: math.MaxUint64, Limit: 10} + res := d.getRange(r) + assert.NotNil(t, res.Hash) + assert.Len(t, res.Elements, 10) + }) + t.Run("hash", func(t *testing.T) { + r := Range{From: 0, To: math.MaxUint64, Limit: 9} + res := d.getRange(r) + t.Log(len(res.Elements)) + assert.NotNil(t, res.Hash) + assert.Nil(t, res.Elements) + }) +} + +func TestDiff_Diff(t *testing.T) { + t.Run("basic", func(t *testing.T) { + d1 := New(16, 16) + d2 := New(16, 16) + for i := 0; i < 1000; i++ { + id := fmt.Sprint(i) + head := bson.NewObjectId().Hex() + d1.Set(Element{ + Id: id, + Head: head, + }) + d2.Set(Element{ + Id: id, + Head: head, + }) + } + + ctx := context.Background() + + newIds, changedIds, removedIds, err := d1.Diff(ctx, d2) + require.NoError(t, err) + assert.Len(t, newIds, 0) + assert.Len(t, changedIds, 0) + assert.Len(t, removedIds, 0) + + d2.Set(Element{ + Id: "newD1", + Head: "newD1", + }) + d2.Set(Element{ + Id: "1", + Head: "changed", + }) + require.NoError(t, d2.RemoveId("0")) + + newIds, changedIds, removedIds, err = d1.Diff(ctx, d2) + require.NoError(t, err) + assert.Len(t, newIds, 1) + assert.Len(t, changedIds, 1) + assert.Len(t, removedIds, 1) + }) +} + +func BenchmarkDiff_Ranges(b *testing.B) { + d := New(16, 16) + for i := 0; i < 10000; i++ { + id := fmt.Sprint(i) + head := bson.NewObjectId().Hex() + d.Set(Element{ + Id: id, + Head: head, + }) + } + ctx := context.Background() + b.ResetTimer() + b.ReportAllocs() + var resBuf []RangeResult + var ranges = []Range{{From: 0, To: math.MaxUint64, Limit: 10}} + for i := 0; i < b.N; i++ { + d.Ranges(ctx, ranges, resBuf) + resBuf = resBuf[:0] + } +}