From 90f0d0bb37649214f0774b8d92307e3e235137c1 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Thu, 5 Jan 2023 15:39:08 +0300 Subject: [PATCH 01/60] remove consensus dep --- commonspace/objectsync/streampool_test.go | 3 --- go.mod | 1 - go.sum | 2 -- 3 files changed, 6 deletions(-) diff --git a/commonspace/objectsync/streampool_test.go b/commonspace/objectsync/streampool_test.go index 052802d5..056d11de 100644 --- a/commonspace/objectsync/streampool_test.go +++ b/commonspace/objectsync/streampool_test.go @@ -5,7 +5,6 @@ import ( "github.com/anytypeio/any-sync/commonspace/spacesyncproto" "github.com/anytypeio/any-sync/net/peer" "github.com/anytypeio/any-sync/net/rpc/rpctest" - "github.com/anytypeio/go-anytype-infrastructure-experiments/consensus/consensusproto" "github.com/stretchr/testify/require" "testing" "time" @@ -13,8 +12,6 @@ import ( type testServer struct { stream chan spacesyncproto.DRPCSpaceSync_ObjectSyncStreamStream - addLog func(ctx context.Context, req *consensusproto.AddLogRequest) error - addRecord func(ctx context.Context, req *consensusproto.AddRecordRequest) error releaseStream chan error watchErrOnce bool } diff --git a/go.mod b/go.mod index f7a4ae16..c0bb5cab 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/anytypeio/any-sync go 1.19 require ( - github.com/anytypeio/go-anytype-infrastructure-experiments/consensus v0.0.0-20221107145605-92bdf7d57b48 github.com/anytypeio/go-chash v0.0.0-20220629194632-4ad1154fe232 github.com/awalterschulze/gographviz v2.0.3+incompatible github.com/cespare/xxhash v1.1.0 diff --git a/go.sum b/go.sum index c758548f..cfd087e3 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/anytypeio/go-anytype-infrastructure-experiments/consensus v0.0.0-20221107145605-92bdf7d57b48 h1:fCZ8db6yJBLPd6bP59zwfjuokL7SLbeTKrk3kBSy034= -github.com/anytypeio/go-anytype-infrastructure-experiments/consensus v0.0.0-20221107145605-92bdf7d57b48/go.mod h1:w0i62cRB2jVpjFb2CpPNj5J+ihKqqmBBG9X2+Odekjw= github.com/anytypeio/go-chash v0.0.0-20220629194632-4ad1154fe232 h1:kMPPZYmJgbs4AJfodbg2OCXg5cp+9LPAJcLZJqmcghk= github.com/anytypeio/go-chash v0.0.0-20220629194632-4ad1154fe232/go.mod h1:+PeHBAWp7gUh/yw6uAauKc5ku0w4cFNg6DUddGxoGq0= github.com/awalterschulze/gographviz v2.0.3+incompatible h1:9sVEXJBJLwGX7EQVhLm2elIKCm7P2YHFC8v6096G09E= From c88b31ca2f4ec4279e74ec6d3fa82e4a4f515fc7 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Thu, 5 Jan 2023 17:53:24 +0300 Subject: [PATCH 02/60] remove key.Key.Equals for more compatibility --- util/keys/key.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/util/keys/key.go b/util/keys/key.go index 150d0897..3f1c9f2e 100644 --- a/util/keys/key.go +++ b/util/keys/key.go @@ -3,8 +3,6 @@ package keys import "crypto/subtle" type Key interface { - Equals(Key) bool - Raw() ([]byte, error) } From f79d7f8eb497015a9448ee1362887f0939962fdd Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Thu, 5 Jan 2023 18:24:48 +0300 Subject: [PATCH 03/60] Revert "remove key.Key.Equals for more compatibility" This reverts commit 7f5df570631033b017e2d724f7cd118445834ec9. --- util/keys/key.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/keys/key.go b/util/keys/key.go index 3f1c9f2e..150d0897 100644 --- a/util/keys/key.go +++ b/util/keys/key.go @@ -3,6 +3,8 @@ package keys import "crypto/subtle" type Key interface { + Equals(Key) bool + Raw() ([]byte, error) } From b8b4f937d47043a7a9d7677b4d63381bdc66baf1 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 6 Jan 2023 21:38:48 +0300 Subject: [PATCH 04/60] CI coverage --- .github/workflows/ci.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..668c61b0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,36 @@ +on: + pull_request: + branches: + - master + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: '^1.19.0' + + - name: Vet + run: | + go vet ./... + - name: Unit tests + run: | + sudo go test ./... -coverprofile coverage.out -covermode count + sudo go tool cover -func coverage.out + - name: Quality Gate - Test coverage shall be above threshold + env: + TESTCOVERAGE_THRESHOLD: 0 + run: | + echo "Quality Gate: checking test coverage is above threshold ..." + echo "Threshold : $TESTCOVERAGE_THRESHOLD %" + totalCoverage=`go tool cover -func=coverage.out | grep total | grep -Eo '[0-9]+\.[0-9]+'` + echo "Current test coverage : $totalCoverage %" + if (( $(echo "$totalCoverage $TESTCOVERAGE_THRESHOLD" | awk '{print ($1 > $2)}') )); then + echo "OK" + else + echo "Current test coverage is below threshold. Please add more unit tests or adjust threshold to a lower value." + echo "Failed" + exit 1 + fi From dbfc62eef5f64f4ec9d14516f7baf70d9e8d7137 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 6 Jan 2023 21:39:36 +0300 Subject: [PATCH 05/60] crypro: DeriveRSA --- go.mod | 2 +- go.sum | 4 + util/keys/asymmetric/encryptionkey/rsa.go | 155 ++++++++++++++++++ .../keys/asymmetric/encryptionkey/rsa_test.go | 24 +++ 4 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 util/keys/asymmetric/encryptionkey/rsa_test.go diff --git a/go.mod b/go.mod index c0bb5cab..12b06522 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/zeebo/errs v1.3.0 go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b + golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 gopkg.in/yaml.v3 v3.0.1 storj.io/drpc v0.0.32 diff --git a/go.sum b/go.sum index cfd087e3..0e52e5ce 100644 --- a/go.sum +++ b/go.sum @@ -505,6 +505,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b h1:SCE/18RnFsLrjydh/R/s5EVvHoZprqEQUuoxK8q2Pc4= golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3 h1:fJwx88sMf5RXwDwziL0/Mn9Wqs+efMSo/RYcL+37W9c= +golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg= @@ -529,6 +531,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -697,6 +700,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/util/keys/asymmetric/encryptionkey/rsa.go b/util/keys/asymmetric/encryptionkey/rsa.go index 467a22e4..80fc599a 100644 --- a/util/keys/asymmetric/encryptionkey/rsa.go +++ b/util/keys/asymmetric/encryptionkey/rsa.go @@ -8,9 +8,16 @@ import ( "crypto/x509" "errors" "github.com/anytypeio/any-sync/util/keys" + "github.com/cespare/xxhash" + mrand "golang.org/x/exp/rand" "io" + "math" + "math/big" ) +var bigZero = big.NewInt(0) +var bigOne = big.NewInt(1) + var MinRsaKeyBits = 2048 var ErrKeyLengthTooSmall = errors.New("error key length too small") @@ -80,6 +87,19 @@ func GenerateRSAKeyPair(bits int, src io.Reader) (PrivKey, PubKey, error) { return &EncryptionRsaPrivKey{privKey: *priv}, &EncryptionRsaPubKey{pubKey: pk}, nil } +func DeriveRSAKePair(bits int, seed []byte) (PrivKey, PubKey, error) { + if bits < MinRsaKeyBits { + return nil, nil, ErrKeyLengthTooSmall + } + seed64 := xxhash.Sum64(seed) + priv, err := rsaGenerateMultiPrimeKey(mrand.New(mrand.NewSource(seed64)), 2, bits) + if err != nil { + return nil, nil, err + } + pk := priv.PublicKey + return &EncryptionRsaPrivKey{privKey: *priv}, &EncryptionRsaPubKey{pubKey: pk}, nil +} + func NewEncryptionRsaPrivKeyFromBytes(bytes []byte) (PrivKey, error) { sk, err := x509.ParsePKCS1PrivateKey(bytes) if err != nil { @@ -118,3 +138,138 @@ func keyEquals(k1, k2 keys.Key) bool { } return subtle.ConstantTimeCompare(a, b) == 1 } + +// generateMultiPrimeKey is a copied original rsa.GenerateMultiPrimeKey but without randutil.MaybeReadByte calls +func rsaGenerateMultiPrimeKey(random io.Reader, nprimes int, bits int) (*rsa.PrivateKey, error) { + + priv := new(rsa.PrivateKey) + priv.E = 65537 + + if nprimes < 2 { + return nil, errors.New("crypto/rsa: GenerateMultiPrimeKey: nprimes must be >= 2") + } + + if bits < 64 { + primeLimit := float64(uint64(1) << uint(bits/nprimes)) + // pi approximates the number of primes less than primeLimit + pi := primeLimit / (math.Log(primeLimit) - 1) + // Generated primes start with 11 (in binary) so we can only + // use a quarter of them. + pi /= 4 + // Use a factor of two to ensure that key generation terminates + // in a reasonable amount of time. + pi /= 2 + if pi <= float64(nprimes) { + return nil, errors.New("crypto/rsa: too few primes of given length to generate an RSA key") + } + } + + primes := make([]*big.Int, nprimes) + +NextSetOfPrimes: + for { + todo := bits + // crypto/rand should set the top two bits in each prime. + // Thus each prime has the form + // p_i = 2^bitlen(p_i) × 0.11... (in base 2). + // And the product is: + // P = 2^todo × α + // where α is the product of nprimes numbers of the form 0.11... + // + // If α < 1/2 (which can happen for nprimes > 2), we need to + // shift todo to compensate for lost bits: the mean value of 0.11... + // is 7/8, so todo + shift - nprimes * log2(7/8) ~= bits - 1/2 + // will give good results. + if nprimes >= 7 { + todo += (nprimes - 2) / 5 + } + for i := 0; i < nprimes; i++ { + var err error + primes[i], err = randPrime(random, todo/(nprimes-i)) + if err != nil { + return nil, err + } + todo -= primes[i].BitLen() + } + + // Make sure that primes is pairwise unequal. + for i, prime := range primes { + for j := 0; j < i; j++ { + if prime.Cmp(primes[j]) == 0 { + continue NextSetOfPrimes + } + } + } + + n := new(big.Int).Set(bigOne) + totient := new(big.Int).Set(bigOne) + pminus1 := new(big.Int) + for _, prime := range primes { + n.Mul(n, prime) + pminus1.Sub(prime, bigOne) + totient.Mul(totient, pminus1) + } + if n.BitLen() != bits { + // This should never happen for nprimes == 2 because + // crypto/rand should set the top two bits in each prime. + // For nprimes > 2 we hope it does not happen often. + continue NextSetOfPrimes + } + + priv.D = new(big.Int) + e := big.NewInt(int64(priv.E)) + ok := priv.D.ModInverse(e, totient) + + if ok != nil { + priv.Primes = primes + priv.N = n + break + } + } + + priv.Precompute() + return priv, nil +} + +func randPrime(rand io.Reader, bits int) (*big.Int, error) { + if bits < 2 { + return nil, errors.New("crypto/rand: prime size must be at least 2-bit") + } + + b := uint(bits % 8) + if b == 0 { + b = 8 + } + + bytes := make([]byte, (bits+7)/8) + p := new(big.Int) + + for { + if _, err := io.ReadFull(rand, bytes); err != nil { + return nil, err + } + + // Clear bits in the first byte to make sure the candidate has a size <= bits. + bytes[0] &= uint8(int(1<= 2 { + bytes[0] |= 3 << (b - 2) + } else { + // Here b==1, because b cannot be zero. + bytes[0] |= 1 + if len(bytes) > 1 { + bytes[1] |= 0x80 + } + } + // Make the value odd since an even number this large certainly isn't prime. + bytes[len(bytes)-1] |= 1 + + p.SetBytes(bytes) + if p.ProbablyPrime(20) { + return p, nil + } + } +} diff --git a/util/keys/asymmetric/encryptionkey/rsa_test.go b/util/keys/asymmetric/encryptionkey/rsa_test.go new file mode 100644 index 00000000..55b2bfd8 --- /dev/null +++ b/util/keys/asymmetric/encryptionkey/rsa_test.go @@ -0,0 +1,24 @@ +package encryptionkey + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" +) + +func TestDeriveRSAKePair(t *testing.T) { + privKey1, _, err := DeriveRSAKePair(4096, []byte("test seed")) + require.NoError(t, err) + + privKey2, _, err := DeriveRSAKePair(4096, []byte("test seed")) + require.NoError(t, err) + data := []byte("test data") + + encryped, err := privKey1.GetPublic().Encrypt(data) + require.NoError(t, err) + + decrypted, err := privKey2.Decrypt(encryped) + require.NoError(t, err) + + assert.Equal(t, data, decrypted) +} From 248a1122365f6cea24f28dea08687d25fa4f214a Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 6 Jan 2023 21:44:28 +0300 Subject: [PATCH 06/60] fix CI --- .github/workflows/{ci.yml => coverage.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{ci.yml => coverage.yml} (98%) diff --git a/.github/workflows/ci.yml b/.github/workflows/coverage.yml similarity index 98% rename from .github/workflows/ci.yml rename to .github/workflows/coverage.yml index 668c61b0..ba9f6cc5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/coverage.yml @@ -1,7 +1,7 @@ on: pull_request: branches: - - master + - main jobs: test: From dfd2b0e8b9362c80287071aff98c0e41e82cbf1a Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 6 Jan 2023 20:55:10 +0100 Subject: [PATCH 07/60] Use account service in secureservice instead of config --- net/secureservice/secureservice.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/net/secureservice/secureservice.go b/net/secureservice/secureservice.go index 814df847..d83d1c94 100644 --- a/net/secureservice/secureservice.go +++ b/net/secureservice/secureservice.go @@ -5,7 +5,6 @@ import ( commonaccount "github.com/anytypeio/any-sync/accountservice" "github.com/anytypeio/any-sync/app" "github.com/anytypeio/any-sync/app/logger" - "github.com/anytypeio/any-sync/util/keys" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/sec" libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls" @@ -35,16 +34,16 @@ type secureService struct { } func (s *secureService) Init(a *app.App) (err error) { - account := a.MustComponent("config").(commonaccount.ConfigGetter).GetAccount() - pkb, err := keys.DecodeBytesFromString(account.PeerKey) + account := a.MustComponent(commonaccount.CName).(commonaccount.Service) + peerKey, err := account.Account().PeerKey.Raw() if err != nil { return } - if s.key, err = crypto.UnmarshalEd25519PrivateKey(pkb); err != nil { + if s.key, err = crypto.UnmarshalEd25519PrivateKey(peerKey); err != nil { return } - log.Info("secure service init", zap.String("peerId", account.PeerId)) + log.Info("secure service init", zap.String("peerId", account.Account().PeerId)) return nil } From 207e2dce526837f4ec02c5daa7b09cccbd2a492d Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Sat, 7 Jan 2023 00:11:27 +0100 Subject: [PATCH 08/60] Update libp2p ver --- go.mod | 19 +++++++----- go.sum | 47 +++++++++++++++++------------- net/secureservice/secureservice.go | 2 +- net/secureservice/tlslistener.go | 2 +- 4 files changed, 40 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index 12b06522..3f74ae94 100644 --- a/go.mod +++ b/go.mod @@ -20,11 +20,11 @@ require ( github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-merkledag v0.8.1 github.com/ipfs/go-unixfs v0.4.1 - github.com/libp2p/go-libp2p v0.23.2 + github.com/libp2p/go-libp2p v0.24.1 github.com/minio/sha256-simd v1.0.0 github.com/multiformats/go-multibase v0.1.1 github.com/multiformats/go-multihash v0.2.1 - github.com/prometheus/client_golang v1.13.0 + github.com/prometheus/client_golang v1.14.0 github.com/stretchr/testify v1.8.1 github.com/zeebo/blake3 v0.2.3 github.com/zeebo/errs v1.3.0 @@ -39,8 +39,9 @@ require ( require ( github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect github.com/go-logr/logr v1.2.3 // indirect @@ -64,24 +65,26 @@ require ( github.com/ipfs/go-verifcid v0.0.2 // indirect github.com/ipld/go-codec-dagpb v1.5.0 // indirect github.com/ipld/go-ipld-prime v0.19.0 // indirect + github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/klauspost/cpuid/v2 v2.2.2 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-openssl v0.1.0 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-pointer v0.0.1 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr v0.7.0 // indirect - github.com/multiformats/go-multicodec v0.6.0 // indirect + github.com/multiformats/go-multiaddr v0.8.0 // indirect + github.com/multiformats/go-multicodec v0.7.0 // indirect + github.com/multiformats/go-multistream v0.3.3 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect @@ -93,7 +96,7 @@ require ( go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.4.0 // indirect golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.3.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index 0e52e5ce..603aa4e9 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,9 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/mb/v3 v3.0.0 h1:+FkV4fAefQfJSsfMtWC9cnSrVYKd3TXcerPTwRuWWfE= github.com/cheggaaa/mb/v3 v3.0.0/go.mod h1:zCt2QeYukhd/g0bIdNqF+b/kKz1hnLFNDkP49qN5kqI= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -72,6 +73,8 @@ github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= @@ -258,6 +261,8 @@ github.com/ipld/go-ipld-prime v0.19.0 h1:5axC7rJmPc17Emw6TelxGwnzALk0PdupZ2oj2ro github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= +github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -296,16 +301,17 @@ github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoR github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= -github.com/libp2p/go-libp2p v0.23.2 h1:yqyTeKQJyofWXxEv/eEVUvOrGdt/9x+0PIQ4N1kaxmE= -github.com/libp2p/go-libp2p v0.23.2/go.mod h1:s9DEa5NLR4g+LZS+md5uGU4emjMWFiqkZr6hBTY8UxI= +github.com/libp2p/go-libp2p v0.24.1 h1:+lS4fqj7RF9egcPq9Yo3iqdRTcDMApzoBbQMhxtwOVw= +github.com/libp2p/go-libp2p v0.24.1/go.mod h1:5LJqbrqFsUzWrq70JHCYqjATlX4ey8Klpct3OEe8hSI= github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-msgio v0.2.0 h1:W6shmB+FeynDrUVl2dgFQvzfBZcXiyqY4VmpQLu9FqU= github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= -github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4nWRE= +github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= +github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rBfZQ= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -313,8 +319,9 @@ github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peK github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= @@ -337,16 +344,16 @@ github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYg github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multiaddr v0.7.0 h1:gskHcdaCyPtp9XskVwtvEeQOG465sCohbQIirSyqxrc= -github.com/multiformats/go-multiaddr v0.7.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= +github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= -github.com/multiformats/go-multicodec v0.6.0 h1:KhH2kSuCARyuJraYMFxrNO3DqIaYhOdS039kbhgVwpE= -github.com/multiformats/go-multicodec v0.6.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= +github.com/multiformats/go-multicodec v0.7.0 h1:rTUjGOwjlhGHbEMbPoSUJowG1spZTVsITRANCjKTUAQ= +github.com/multiformats/go-multicodec v0.7.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= @@ -355,6 +362,7 @@ github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJ github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= github.com/multiformats/go-multistream v0.3.3 h1:d5PZpjwRgVlbwfdTDjife7XszfZd8KYWfROYFlGcR8o= +github.com/multiformats/go-multistream v0.3.3/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= @@ -381,13 +389,14 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= @@ -489,6 +498,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= @@ -503,8 +513,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b h1:SCE/18RnFsLrjydh/R/s5EVvHoZprqEQUuoxK8q2Pc4= -golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3 h1:fJwx88sMf5RXwDwziL0/Mn9Wqs+efMSo/RYcL+37W9c= golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -530,8 +538,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= +golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -585,8 +592,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -619,6 +626,7 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -699,8 +707,7 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= +golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/net/secureservice/secureservice.go b/net/secureservice/secureservice.go index d83d1c94..3c9bc068 100644 --- a/net/secureservice/secureservice.go +++ b/net/secureservice/secureservice.go @@ -61,7 +61,7 @@ func (s *secureService) BasicListener(lis net.Listener, timeoutMillis int) Conte } func (s *secureService) TLSConn(ctx context.Context, conn net.Conn) (sec.SecureConn, error) { - tr, err := libp2ptls.New(s.key) + tr, err := libp2ptls.New(libp2ptls.ID, s.key, nil) if err != nil { return nil, err } diff --git a/net/secureservice/tlslistener.go b/net/secureservice/tlslistener.go index 1fb6495f..ccf9da2d 100644 --- a/net/secureservice/tlslistener.go +++ b/net/secureservice/tlslistener.go @@ -23,7 +23,7 @@ type ContextListener interface { } func newTLSListener(key crypto.PrivKey, lis net.Listener, timeoutMillis int) ContextListener { - tr, _ := libp2ptls.New(key) + tr, _ := libp2ptls.New(libp2ptls.ID, key, nil) return &tlsListener{ tr: tr, Listener: lis, From b3194da23c067d83b33f284a70d7e947164eea6e Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Sat, 7 Jan 2023 13:26:46 +0300 Subject: [PATCH 09/60] fix CI --- .github/workflows/coverage.yml | 3 +-- Makefile | 8 ++++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ba9f6cc5..fb7f7797 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -17,8 +17,7 @@ jobs: go vet ./... - name: Unit tests run: | - sudo go test ./... -coverprofile coverage.out -covermode count - sudo go tool cover -func coverage.out + sudo make test-coverage - name: Quality Gate - Test coverage shall be above threshold env: TESTCOVERAGE_THRESHOLD: 0 diff --git a/Makefile b/Makefile index c327ed5d..1ffe698d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: proto test +.PHONY: proto test test-coverage export GOPRIVATE=github.com/anytypeio proto: @@ -17,4 +17,8 @@ proto: $(GOGO_START) protoc --gogofaster_out=$(PKGMAP):. --go-drpc_out=protolib=github.com/gogo/protobuf:. commonfile/fileproto/protos/*.proto test: - go test ./... --cover \ No newline at end of file + go test ./... --cover + +test-coverage: + go test ./... -coverprofile coverage.out -covermode count + go tool cover -func coverage.out From 62c4e80c361bb159041857ef8c7c836e911d9af4 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Sat, 7 Jan 2023 13:35:12 +0300 Subject: [PATCH 10/60] go vet --- .github/workflows/coverage.yml | 2 +- Makefile | 5 ++++- commonspace/object/tree/objecttree/objecttree.go | 1 - commonspace/settings/deletionstate/deletionstate.go | 2 +- commonspace/settings/idprovider_test.go | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index fb7f7797..85a6fff6 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -14,7 +14,7 @@ jobs: - name: Vet run: | - go vet ./... + make vet - name: Unit tests run: | sudo make test-coverage diff --git a/Makefile b/Makefile index 1ffe698d..53988ac1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: proto test test-coverage +.PHONY: proto test test-coverage vet export GOPRIVATE=github.com/anytypeio proto: @@ -16,6 +16,9 @@ proto: $(GOGO_START) protoc --gogofaster_out=$(PKGMAP):. --go-drpc_out=protolib=github.com/gogo/protobuf:. commonspace/spacesyncproto/protos/*.proto $(GOGO_START) protoc --gogofaster_out=$(PKGMAP):. --go-drpc_out=protolib=github.com/gogo/protobuf:. commonfile/fileproto/protos/*.proto +vet: + go vet ./... + test: go test ./... --cover diff --git a/commonspace/object/tree/objecttree/objecttree.go b/commonspace/object/tree/objecttree/objecttree.go index 7e14fcfc..3dc11cf8 100644 --- a/commonspace/object/tree/objecttree/objecttree.go +++ b/commonspace/object/tree/objecttree/objecttree.go @@ -395,7 +395,6 @@ func (ot *objectTree) addRawChanges(ctx context.Context, changesPayload RawChang } return } - return } func (ot *objectTree) createAddResult(oldHeads []string, mode Mode, treeChangesAdded []*Change, rawChanges []*treechangeproto.RawTreeChangeWithId) (addResult AddResult, err error) { diff --git a/commonspace/settings/deletionstate/deletionstate.go b/commonspace/settings/deletionstate/deletionstate.go index 984971ed..c0d700a8 100644 --- a/commonspace/settings/deletionstate/deletionstate.go +++ b/commonspace/settings/deletionstate/deletionstate.go @@ -133,7 +133,7 @@ func (st *deletionState) CreateDeleteChange(id string, isSnapshot bool) (res []b } change := &spacesyncproto.SettingsData{ Content: []*spacesyncproto.SpaceSettingsContent{ - {content}, + {Value: content}, }, Snapshot: nil, } diff --git a/commonspace/settings/idprovider_test.go b/commonspace/settings/idprovider_test.go index 9bf6c37d..66194ae4 100644 --- a/commonspace/settings/idprovider_test.go +++ b/commonspace/settings/idprovider_test.go @@ -55,7 +55,7 @@ func TestProvider_ProcessChange(t *testing.T) { ch := &objecttree.Change{} ch.Model = &spacesyncproto.SettingsData{ Content: []*spacesyncproto.SpaceSettingsContent{ - {&spacesyncproto.SpaceSettingsContent_ObjectDelete{ + {Value: &spacesyncproto.SpaceSettingsContent_ObjectDelete{ ObjectDelete: &spacesyncproto.ObjectDelete{Id: "id1"}, }}, }, From 048406edfd1c196b64d306b1c9e8c4524db3ac92 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Sat, 7 Jan 2023 12:41:13 +0100 Subject: [PATCH 11/60] Fix incorrect Graph method --- commonspace/object/tree/objecttree/treegraph.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commonspace/object/tree/objecttree/treegraph.go b/commonspace/object/tree/objecttree/treegraph.go index a4915d34..9bb9611e 100644 --- a/commonspace/object/tree/objecttree/treegraph.go +++ b/commonspace/object/tree/objecttree/treegraph.go @@ -6,6 +6,6 @@ package objecttree import "fmt" -func (t *Tree) Graph(parser DescriptionParser) (data []string, err error) { +func (t *Tree) Graph(parser DescriptionParser) (data string, err error) { return "", fmt.Errorf("not supported") } From b96654dd142affdc8bcef1fb44240f47e958839c Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 10 Jan 2023 17:54:04 +0100 Subject: [PATCH 12/60] Fix acl derivation --- commonspace/payloads.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/commonspace/payloads.go b/commonspace/payloads.go index 83fb7867..29e9568e 100644 --- a/commonspace/payloads.go +++ b/commonspace/payloads.go @@ -1,7 +1,7 @@ package commonspace import ( - aclrecordproto2 "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" + aclrecordproto "github.com/anytypeio/any-sync/commonspace/object/acl/aclrecordproto" "github.com/anytypeio/any-sync/commonspace/object/keychain" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/spacestorage" @@ -75,7 +75,7 @@ func storagePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload sp } // preparing acl - aclRoot := &aclrecordproto2.AclRoot{ + aclRoot := &aclrecordproto.AclRoot{ Identity: identity, EncryptionKey: encPubKey, SpaceId: spaceId, @@ -171,7 +171,7 @@ func storagePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload sp } // deriving and encrypting read key - readKey, err := aclrecordproto2.AclReadKeyDerive(signPrivKey, encPrivKey) + readKey, err := aclrecordproto.AclReadKeyDerive(signPrivKey, encPrivKey) if err != nil { return } @@ -181,17 +181,12 @@ func storagePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload sp return } readKeyHash := hasher.Sum64() - encReadKey, err := payload.EncryptionKey.GetPublic().Encrypt(readKey.Bytes()) - if err != nil { - return - } // preparing acl - aclRoot := &aclrecordproto2.AclRoot{ + aclRoot := &aclrecordproto.AclRoot{ Identity: identity, EncryptionKey: encPubKey, SpaceId: spaceId, - EncryptedReadKey: encReadKey, DerivationScheme: SpaceDerivationScheme, CurrentReadKeyHash: readKeyHash, } @@ -221,7 +216,7 @@ func storagePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload sp return } -func marshalAclRoot(aclRoot *aclrecordproto2.AclRoot, key signingkey.PrivKey) (rawWithId *aclrecordproto2.RawAclRecordWithId, err error) { +func marshalAclRoot(aclRoot *aclrecordproto.AclRoot, key signingkey.PrivKey) (rawWithId *aclrecordproto.RawAclRecordWithId, err error) { marshalledRoot, err := aclRoot.Marshal() if err != nil { return @@ -230,7 +225,7 @@ func marshalAclRoot(aclRoot *aclrecordproto2.AclRoot, key signingkey.PrivKey) (r if err != nil { return } - raw := &aclrecordproto2.RawAclRecord{ + raw := &aclrecordproto.RawAclRecord{ Payload: marshalledRoot, Signature: signature, } @@ -242,7 +237,7 @@ func marshalAclRoot(aclRoot *aclrecordproto2.AclRoot, key signingkey.PrivKey) (r if err != nil { return } - rawWithId = &aclrecordproto2.RawAclRecordWithId{ + rawWithId = &aclrecordproto.RawAclRecordWithId{ Payload: marshalledRaw, Id: aclHeadId, } From c565071dadd4c2300fde38040ec47c908f1d16b5 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Wed, 11 Jan 2023 14:15:20 +0300 Subject: [PATCH 13/60] build deps locally --- Makefile | 17 +++++++++++------ deps/.gitignore | 2 ++ 2 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 deps/.gitignore diff --git a/Makefile b/Makefile index 53988ac1..140356d4 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,20 @@ -.PHONY: proto test test-coverage vet +.PHONY: proto test test-coverage vet deps export GOPRIVATE=github.com/anytypeio +export PATH:=deps:$(PATH) proto: @echo 'Generating protobuf packages (Go)...' - @$(eval GOGO_START := GOGO_NO_UNDERSCORE=1 GOGO_EXPORT_ONEOF_INTERFACE=1) @$(eval P_ACL_RECORDS_PATH_PB := commonspace/object/acl/aclrecordproto) @$(eval P_TREE_CHANGES_PATH_PB := commonspace/object/tree/treechangeproto) @$(eval P_ACL_RECORDS := M$(P_ACL_RECORDS_PATH_PB)/protos/aclrecord.proto=github.com/anytypeio/any-sync/$(P_ACL_RECORDS_PATH_PB)) @$(eval P_TREE_CHANGES := M$(P_TREE_CHANGES_PATH_PB)/protos/treechange.proto=github.com/anytypeio/any-sync/$(P_TREE_CHANGES_PATH_PB)) - $(GOGO_START) protoc --gogofaster_out=:. $(P_ACL_RECORDS_PATH_PB)/protos/*.proto - $(GOGO_START) protoc --gogofaster_out=:. $(P_TREE_CHANGES_PATH_PB)/protos/*.proto + protoc --gogofaster_out=:. $(P_ACL_RECORDS_PATH_PB)/protos/*.proto + protoc --gogofaster_out=:. $(P_TREE_CHANGES_PATH_PB)/protos/*.proto $(eval PKGMAP := $$(P_TREE_CHANGES),$$(P_ACL_RECORDS)) - $(GOGO_START) protoc --gogofaster_out=$(PKGMAP):. --go-drpc_out=protolib=github.com/gogo/protobuf:. commonspace/spacesyncproto/protos/*.proto - $(GOGO_START) protoc --gogofaster_out=$(PKGMAP):. --go-drpc_out=protolib=github.com/gogo/protobuf:. commonfile/fileproto/protos/*.proto + protoc --gogofaster_out=$(PKGMAP):. --go-drpc_out=protolib=github.com/gogo/protobuf:. commonspace/spacesyncproto/protos/*.proto + protoc --gogofaster_out=$(PKGMAP):. --go-drpc_out=protolib=github.com/gogo/protobuf:. commonfile/fileproto/protos/*.proto vet: go vet ./... @@ -25,3 +25,8 @@ test: test-coverage: go test ./... -coverprofile coverage.out -covermode count go tool cover -func coverage.out + +deps: + go mod download + go build -o deps storj.io/drpc/cmd/protoc-gen-go-drpc + go build -o deps github.com/gogo/protobuf/protoc-gen-gogofaster diff --git a/deps/.gitignore b/deps/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/deps/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore From bb25dd0ea1ef34c9e8ada9604924261bb8689971 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Wed, 11 Jan 2023 14:50:20 +0300 Subject: [PATCH 14/60] files: export DAGService --- commonfile/fileservice/fileservice.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/commonfile/fileservice/fileservice.go b/commonfile/fileservice/fileservice.go index 0202fb02..0c23a5fc 100644 --- a/commonfile/fileservice/fileservice.go +++ b/commonfile/fileservice/fileservice.go @@ -31,6 +31,8 @@ type FileService interface { GetFile(ctx context.Context, c cid.Cid) (ufsio.ReadSeekCloser, error) // AddFile adds file to ipfs storage AddFile(ctx context.Context, r io.Reader) (ipld.Node, error) + // DAGService returns ipld.DAGService object + DAGService() ipld.DAGService app.Component } @@ -60,6 +62,10 @@ func (fs *fileService) Name() string { return CName } +func (fs *fileService) DAGService() ipld.DAGService { + return fs.merkledag +} + func (fs *fileService) AddFile(ctx context.Context, r io.Reader) (ipld.Node, error) { dbp := helpers.DagBuilderParams{ Dagserv: fs.merkledag, From c7254ed8bec12c3c719d284ef2953047af4ed41e Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 14:53:55 +0300 Subject: [PATCH 15/60] .github/workflows/coverage.yml add git config --- .github/workflows/coverage.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 85a6fff6..e004e503 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -12,6 +12,9 @@ jobs: with: go-version: '^1.19.0' + - name: git config + run: | + git config --global http.extraheader "PRIVATE-TOKEN: ${{ secrets.GITHUB_TOKEN }}" - name: Vet run: | make vet From 2f95c6636025ed0cdb2e15c54df7e6ff3ee8f77e Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 15:03:48 +0300 Subject: [PATCH 16/60] coverage.yml test --- .github/workflows/coverage.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index e004e503..df0fb241 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -15,6 +15,17 @@ jobs: - name: git config run: | git config --global http.extraheader "PRIVATE-TOKEN: ${{ secrets.GITHUB_TOKEN }}" + - name: Create issue using REST API + run: | + curl --request POST \ + --url https://api.github.com/repos/${{ github.repository }}/issues \ + --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ + --header 'content-type: application/json' \ + --data '{ + "title": "Automated issue for commit: ${{ github.sha }}", + "body": "This issue was automatically created by the GitHub Action workflow **${{ github.workflow }}**. \n\n The commit hash was: _${{ github.sha }}_." + }' \ + --fail - name: Vet run: | make vet From 0dd992b7a6e68ed5b868d963ed7f8ef93ef6fb40 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 15:06:17 +0300 Subject: [PATCH 17/60] fix --- .github/workflows/coverage.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index df0fb241..430678e3 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -15,17 +15,17 @@ jobs: - name: git config run: | git config --global http.extraheader "PRIVATE-TOKEN: ${{ secrets.GITHUB_TOKEN }}" - - name: Create issue using REST API + - name: Create issue using REST API run: | curl --request POST \ - --url https://api.github.com/repos/${{ github.repository }}/issues \ - --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ - --header 'content-type: application/json' \ - --data '{ - "title": "Automated issue for commit: ${{ github.sha }}", - "body": "This issue was automatically created by the GitHub Action workflow **${{ github.workflow }}**. \n\n The commit hash was: _${{ github.sha }}_." - }' \ - --fail + --url https://api.github.com/repos/${{ github.repository }}/issues \ + --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ + --header 'content-type: application/json' \ + --data '{ + "title": "Automated issue for commit: ${{ github.sha }}", + "body": "This issue was automatically created by the GitHub Action workflow **${{ github.workflow }}**. \n\n The commit hash was: _${{ github.sha }}_." + }' \ + --fail - name: Vet run: | make vet From d0ac577686730070ee00121c77690f71b40e7709 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 15:16:18 +0300 Subject: [PATCH 18/60] coverage.yml add GOPRIVATE --- .github/workflows/coverage.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 430678e3..ddfeae5b 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -15,19 +15,9 @@ jobs: - name: git config run: | git config --global http.extraheader "PRIVATE-TOKEN: ${{ secrets.GITHUB_TOKEN }}" - - name: Create issue using REST API - run: | - curl --request POST \ - --url https://api.github.com/repos/${{ github.repository }}/issues \ - --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ - --header 'content-type: application/json' \ - --data '{ - "title": "Automated issue for commit: ${{ github.sha }}", - "body": "This issue was automatically created by the GitHub Action workflow **${{ github.workflow }}**. \n\n The commit hash was: _${{ github.sha }}_." - }' \ - --fail - name: Vet run: | + go env -w GOPRIVATE=github.com/anytypeio/* make vet - name: Unit tests run: | From 071d85cd2222f155dadb7412034f57efd0e0cf17 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 15:18:40 +0300 Subject: [PATCH 19/60] fix .github/workflows/coverage.yml --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ddfeae5b..b302fea9 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -17,7 +17,7 @@ jobs: git config --global http.extraheader "PRIVATE-TOKEN: ${{ secrets.GITHUB_TOKEN }}" - name: Vet run: | - go env -w GOPRIVATE=github.com/anytypeio/* + export GOPRIVATE=github.com/anytypeio make vet - name: Unit tests run: | From 7e41284cc8a8e1a737e7a8348d9c3be3f6619676 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 15:20:13 +0300 Subject: [PATCH 20/60] add export GIT_TERMINAL_PROMPT=1 --- .github/workflows/coverage.yml | 1 - Makefile | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index b302fea9..e004e503 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -17,7 +17,6 @@ jobs: git config --global http.extraheader "PRIVATE-TOKEN: ${{ secrets.GITHUB_TOKEN }}" - name: Vet run: | - export GOPRIVATE=github.com/anytypeio make vet - name: Unit tests run: | diff --git a/Makefile b/Makefile index 140356d4..c4c028a8 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ .PHONY: proto test test-coverage vet deps export GOPRIVATE=github.com/anytypeio +export GIT_TERMINAL_PROMPT=1 export PATH:=deps:$(PATH) proto: From 527d104ba7f20ee8bab2aba4cd86fa2bea82c9e1 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 15:22:43 +0300 Subject: [PATCH 21/60] fix GOPRIVATE --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c4c028a8..f46b68b2 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: proto test test-coverage vet deps -export GOPRIVATE=github.com/anytypeio +export GOPRIVATE=github.com/anytypeio/* export GIT_TERMINAL_PROMPT=1 export PATH:=deps:$(PATH) From da74f45d462b38bb94207ec756a0ef2b35bc3402 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Wed, 11 Jan 2023 15:30:31 +0300 Subject: [PATCH 22/60] update chash --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3f74ae94..832c8a78 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/anytypeio/any-sync go 1.19 require ( - github.com/anytypeio/go-chash v0.0.0-20220629194632-4ad1154fe232 + github.com/anytypeio/go-chash v0.0.1 github.com/awalterschulze/gographviz v2.0.3+incompatible github.com/cespare/xxhash v1.1.0 github.com/cheggaaa/mb/v3 v3.0.0 diff --git a/go.sum b/go.sum index 603aa4e9..782a3638 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5K github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/anytypeio/go-chash v0.0.0-20220629194632-4ad1154fe232 h1:kMPPZYmJgbs4AJfodbg2OCXg5cp+9LPAJcLZJqmcghk= github.com/anytypeio/go-chash v0.0.0-20220629194632-4ad1154fe232/go.mod h1:+PeHBAWp7gUh/yw6uAauKc5ku0w4cFNg6DUddGxoGq0= +github.com/anytypeio/go-chash v0.0.1 h1:gmhq5c8qweYc+QiyJK2TCzSIp55HlaJlTD5aPUi8TFA= +github.com/anytypeio/go-chash v0.0.1/go.mod h1:MTQdZA8gD53OH9VFCt33w5WpIs5kSmBJiSYC2iEGSSU= github.com/awalterschulze/gographviz v2.0.3+incompatible h1:9sVEXJBJLwGX7EQVhLm2elIKCm7P2YHFC8v6096G09E= github.com/awalterschulze/gographviz v2.0.3+incompatible/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 8c369537900236deb335f8b1622fb89ab5ec8840 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 15:34:49 +0300 Subject: [PATCH 23/60] fix .github/workflows/coverage.yml --- .github/workflows/coverage.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index e004e503..98362527 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -12,11 +12,10 @@ jobs: with: go-version: '^1.19.0' - - name: git config - run: | - git config --global http.extraheader "PRIVATE-TOKEN: ${{ secrets.GITHUB_TOKEN }}" - name: Vet run: | + git config --global http.extraheader "PRIVATE-TOKEN: ${{ secrets.GITHUB_TOKEN }}" + export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make vet - name: Unit tests run: | From dcae5da0ec80e6a3926120600af2831dfd7a495b Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 15:38:53 +0300 Subject: [PATCH 24/60] fix .github/workflows/coverage.yml --- .github/workflows/coverage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 98362527..fbee34c5 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -15,6 +15,7 @@ jobs: - name: Vet run: | git config --global http.extraheader "PRIVATE-TOKEN: ${{ secrets.GITHUB_TOKEN }}" + git config --global url."ssh://git@github.com/".insteadOf "https://github.com/" export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make vet - name: Unit tests From d986290db8a07361c66634cdf1f47d2454c2003c Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Wed, 11 Jan 2023 15:39:32 +0300 Subject: [PATCH 25/60] fileservice.HasCid --- commonfile/fileservice/fileservice.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/commonfile/fileservice/fileservice.go b/commonfile/fileservice/fileservice.go index 0c23a5fc..fabc70c3 100644 --- a/commonfile/fileservice/fileservice.go +++ b/commonfile/fileservice/fileservice.go @@ -33,10 +33,12 @@ type FileService interface { AddFile(ctx context.Context, r io.Reader) (ipld.Node, error) // DAGService returns ipld.DAGService object DAGService() ipld.DAGService + // HasCid checks is CID exists + HasCid(ctx context.Context, c cid.Cid) (exists bool, err error) app.Component } - type fileService struct { + bs fileblockstore.BlockStoreLocal merkledag ipld.DAGService prefix cid.Prefix } @@ -52,8 +54,8 @@ func (fs *fileService) Init(a *app.App) (err error) { } prefix.MhType = hashFunCode prefix.MhLength = -1 - bs := newBlockService(a.MustComponent(fileblockstore.CName).(fileblockstore.BlockStore)) - fs.merkledag = merkledag.NewDAGService(bs) + fs.bs = a.MustComponent(fileblockstore.CName).(fileblockstore.BlockStoreLocal) + fs.merkledag = merkledag.NewDAGService(newBlockService(fs.bs)) fs.prefix = prefix return } @@ -92,3 +94,11 @@ func (fs *fileService) GetFile(ctx context.Context, c cid.Cid) (ufsio.ReadSeekCl } return ufsio.NewDagReader(ctx, n, fs.merkledag) } + +func (fs *fileService) HasCid(ctx context.Context, c cid.Cid) (exists bool, err error) { + res, err := fs.bs.ExistsCids(ctx, []cid.Cid{c}) + if err != nil { + return + } + return len(res) > 0, nil +} From 4186e309121f70c9fdf5a22179539f0fad0b35e5 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 15:43:55 +0300 Subject: [PATCH 26/60] fix .github/workflows/coverage.yml --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index fbee34c5..3409288b 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -15,7 +15,7 @@ jobs: - name: Vet run: | git config --global http.extraheader "PRIVATE-TOKEN: ${{ secrets.GITHUB_TOKEN }}" - git config --global url."ssh://git@github.com/".insteadOf "https://github.com/" + git config --global url."https://${{ secrets.GITHUB_TOKEN }}:x-oauth-basic@github.com/".insteadOf "https://github.com/" export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make vet - name: Unit tests From 27b6f754b5d6f0a01246bbd397c32718a5ca6e7a Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 15:53:34 +0300 Subject: [PATCH 27/60] fix .github/workflows/coverage.yml --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 3409288b..6b4590ca 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -15,7 +15,7 @@ jobs: - name: Vet run: | git config --global http.extraheader "PRIVATE-TOKEN: ${{ secrets.GITHUB_TOKEN }}" - git config --global url."https://${{ secrets.GITHUB_TOKEN }}:x-oauth-basic@github.com/".insteadOf "https://github.com/" + git config --global url."https://${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make vet - name: Unit tests From 35e58ae20f6a1e030437fde3c36c0481ff8331a9 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 16:25:52 +0300 Subject: [PATCH 28/60] fix .github/workflows/coverage.yml --- .github/workflows/coverage.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 6b4590ca..39ea7629 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -13,10 +13,12 @@ jobs: go-version: '^1.19.0' - name: Vet + env: + GITHUB_TOKEN: ${{ github.token }} run: | - git config --global http.extraheader "PRIVATE-TOKEN: ${{ secrets.GITHUB_TOKEN }}" - git config --global url."https://${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" - export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + #git config --global http.extraheader "PRIVATE-TOKEN: ${{ github.token }}" + #git config --global url."https://${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" + #export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make vet - name: Unit tests run: | From 993403bf768408149c87f09a9342d3f0928accac Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 16:29:21 +0300 Subject: [PATCH 29/60] fix .github/workflows/coverage.yml --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index f46b68b2..07ec9d69 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ .PHONY: proto test test-coverage vet deps export GOPRIVATE=github.com/anytypeio/* +export GOPROXY=direct export GIT_TERMINAL_PROMPT=1 export PATH:=deps:$(PATH) From 88cc804d011b02129f3bfd13ec0b644f5567a90a Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 17:08:43 +0300 Subject: [PATCH 30/60] Makefile: revert tests --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 07ec9d69..140356d4 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,5 @@ .PHONY: proto test test-coverage vet deps -export GOPRIVATE=github.com/anytypeio/* -export GOPROXY=direct -export GIT_TERMINAL_PROMPT=1 +export GOPRIVATE=github.com/anytypeio export PATH:=deps:$(PATH) proto: From 3bc23edfd3e4845b00a204c96d530e58916c1458 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 17:09:12 +0300 Subject: [PATCH 31/60] .github/workflows/coverage.yml add test go install --- .github/workflows/coverage.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 39ea7629..638b9ce8 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -6,19 +6,21 @@ on: jobs: test: runs-on: ubuntu-latest + env: + GOPRIVATE: github.com/anytypeio + GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} steps: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: go-version: '^1.19.0' - - name: Vet - env: - GITHUB_TOKEN: ${{ github.token }} + - name: test go install + run: | + git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/ + go install github.com/anytypeio/go-chash@v0.0.1 + - name: Vet run: | - #git config --global http.extraheader "PRIVATE-TOKEN: ${{ github.token }}" - #git config --global url."https://${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" - #export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make vet - name: Unit tests run: | From 876f94ef4e6117aa2c166cea414ec3655822eb73 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 17:11:52 +0300 Subject: [PATCH 32/60] testbytest --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 638b9ce8..7f856ada 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -18,7 +18,7 @@ jobs: - name: test go install run: | git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/ - go install github.com/anytypeio/go-chash@v0.0.1 + go install github.com/anytypeio/go-chash@latest - name: Vet run: | make vet From a9c564777a2b0a5abb55e4e7925c9305d7a808b8 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 17:17:19 +0300 Subject: [PATCH 33/60] testbytest --- .github/workflows/coverage.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 7f856ada..2bc6d145 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -6,9 +6,9 @@ on: jobs: test: runs-on: ubuntu-latest - env: - GOPRIVATE: github.com/anytypeio - GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} +# env: +# GOPRIVATE: github.com/anytypeio +# GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} steps: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 @@ -17,7 +17,7 @@ jobs: - name: test go install run: | - git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/ + #git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/ go install github.com/anytypeio/go-chash@latest - name: Vet run: | From 39f79e03ceb31d1fdda03d3afb32622c972c0f92 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 17:49:00 +0300 Subject: [PATCH 34/60] testbytest --- .github/workflows/coverage.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 2bc6d145..6d664879 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -6,9 +6,6 @@ on: jobs: test: runs-on: ubuntu-latest -# env: -# GOPRIVATE: github.com/anytypeio -# GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} steps: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 @@ -17,7 +14,6 @@ jobs: - name: test go install run: | - #git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/ go install github.com/anytypeio/go-chash@latest - name: Vet run: | From fe8f8c43f593b00c6dee045595f9cb3bfbddce28 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 18:04:16 +0300 Subject: [PATCH 35/60] testbytest --- .github/workflows/coverage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 6d664879..fb3a2e3a 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -15,6 +15,7 @@ jobs: - name: test go install run: | go install github.com/anytypeio/go-chash@latest + - name: Vet run: | make vet From 57eb691b53f298b84a6f74568bd4fe33a3db9f54 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 18:06:13 +0300 Subject: [PATCH 36/60] testbytest --- .github/workflows/coverage.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index fb3a2e3a..1527fc39 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -6,6 +6,8 @@ on: jobs: test: runs-on: ubuntu-latest + env: + GOPRIVATE: github.com/anytypeio steps: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 From d158b98b352527e11a09004f365f87dbf04b2422 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 18:33:17 +0300 Subject: [PATCH 37/60] testbytest --- .github/workflows/coverage.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 1527fc39..0727c065 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -14,6 +14,12 @@ jobs: with: go-version: '^1.19.0' + - name: Checkout private + uses: actions/checkout@v3 + with: + repository: github.com/anytypeio/go-chash + path: go-chash + - name: test go install run: | go install github.com/anytypeio/go-chash@latest From 96896d3060fd46f3df76074d1815b8e0b76ca960 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 18:34:28 +0300 Subject: [PATCH 38/60] testbytest --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 0727c065..0088618d 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -17,7 +17,7 @@ jobs: - name: Checkout private uses: actions/checkout@v3 with: - repository: github.com/anytypeio/go-chash + repository: anytypeio/go-chash path: go-chash - name: test go install From c12bb0f0815c92addc893b69395129241e4661eb Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 18:37:56 +0300 Subject: [PATCH 39/60] testbytest --- .github/workflows/coverage.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 0088618d..348d7da9 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -18,7 +18,6 @@ jobs: uses: actions/checkout@v3 with: repository: anytypeio/go-chash - path: go-chash - name: test go install run: | From 39bffdab9c706bef8befdfbfb04e1dc441899404 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 18:51:30 +0300 Subject: [PATCH 40/60] testbytest --- .github/workflows/coverage.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 348d7da9..b9d8d4ec 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -15,9 +15,8 @@ jobs: go-version: '^1.19.0' - name: Checkout private - uses: actions/checkout@v3 - with: - repository: anytypeio/go-chash + run: + git clone https://${{ GITHUB_TOKEN }}@github.com/anytypeio/go-chash.git - name: test go install run: | From da61fec0a0d17bda18b52d2df5fb94e9abc91fcf Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 18:52:12 +0300 Subject: [PATCH 41/60] testbytest --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index b9d8d4ec..edaeb510 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -16,7 +16,7 @@ jobs: - name: Checkout private run: - git clone https://${{ GITHUB_TOKEN }}@github.com/anytypeio/go-chash.git + git clone https://${{ secrets.GITHUB_TOKEN }}@github.com/anytypeio/go-chash.git - name: test go install run: | From abe46036392a85cf7554bce5b81242d3790e4d24 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 19:36:30 +0300 Subject: [PATCH 42/60] testbytest --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index edaeb510..aacc4e2f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -16,7 +16,7 @@ jobs: - name: Checkout private run: - git clone https://${{ secrets.GITHUB_TOKEN }}@github.com/anytypeio/go-chash.git + git clone https://${{ secrets.ANYTYPE_PAT }}@github.com/anytypeio/go-chash.git - name: test go install run: | From 7f165dd6f85628b574b3fbd9746a994146f9fed3 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 19:38:44 +0300 Subject: [PATCH 43/60] testbytest --- .github/workflows/coverage.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index aacc4e2f..850502a6 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -14,9 +14,8 @@ jobs: with: go-version: '^1.19.0' - - name: Checkout private - run: - git clone https://${{ secrets.ANYTYPE_PAT }}@github.com/anytypeio/go-chash.git + - name: git config + run: git config --global url.https://${{ secrets.ANYTYPE_PAT }}@github.com/.insteadOf https://github.com/ - name: test go install run: | From ca1ebaaf148bf383cedf9f1c25db6a6fa05ad0d6 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 19:39:50 +0300 Subject: [PATCH 44/60] testbytest --- .github/workflows/coverage.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 850502a6..4d8753e6 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -17,10 +17,6 @@ jobs: - name: git config run: git config --global url.https://${{ secrets.ANYTYPE_PAT }}@github.com/.insteadOf https://github.com/ - - name: test go install - run: | - go install github.com/anytypeio/go-chash@latest - - name: Vet run: | make vet From 0fb8ec2b7cd34a957e7852740cda84274533629c Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 19:43:52 +0300 Subject: [PATCH 45/60] fix .github/workflows/coverage.yml --- .github/workflows/coverage.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 4d8753e6..5937fc24 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -17,12 +17,13 @@ jobs: - name: git config run: git config --global url.https://${{ secrets.ANYTYPE_PAT }}@github.com/.insteadOf https://github.com/ - - name: Vet - run: | - make vet - - name: Unit tests - run: | - sudo make test-coverage + - name: deps + run: make deps + - name: vet + run: make vet + - name: unit tests + run: make test-coverage + - name: Quality Gate - Test coverage shall be above threshold env: TESTCOVERAGE_THRESHOLD: 0 From 28dd15e07eb26c62a620dc44ec65c643f8c6ea39 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 19:48:47 +0300 Subject: [PATCH 46/60] fix go version --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 5937fc24..8bf31192 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: '^1.19.0' + go-version: '^1.19.5' - name: git config run: git config --global url.https://${{ secrets.ANYTYPE_PAT }}@github.com/.insteadOf https://github.com/ From 1bb707f0a28505a1e49fe7b803c859ab893f3abc Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 19:59:54 +0300 Subject: [PATCH 47/60] refact --- .github/workflows/coverage.yml | 7 ++++--- Makefile | 16 +++++----------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 8bf31192..27b222a9 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -19,15 +19,16 @@ jobs: - name: deps run: make deps - - name: vet - run: make vet + - name: unit tests - run: make test-coverage + run: make test - name: Quality Gate - Test coverage shall be above threshold env: TESTCOVERAGE_THRESHOLD: 0 run: | + go test ./... -coverprofile coverage.out -covermode count + go tool cover -func coverage.out echo "Quality Gate: checking test coverage is above threshold ..." echo "Threshold : $TESTCOVERAGE_THRESHOLD %" totalCoverage=`go tool cover -func=coverage.out | grep total | grep -Eo '[0-9]+\.[0-9]+'` diff --git a/Makefile b/Makefile index 140356d4..d2229111 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: proto test test-coverage vet deps +.PHONY: proto test deps export GOPRIVATE=github.com/anytypeio export PATH:=deps:$(PATH) @@ -16,17 +16,11 @@ proto: protoc --gogofaster_out=$(PKGMAP):. --go-drpc_out=protolib=github.com/gogo/protobuf:. commonspace/spacesyncproto/protos/*.proto protoc --gogofaster_out=$(PKGMAP):. --go-drpc_out=protolib=github.com/gogo/protobuf:. commonfile/fileproto/protos/*.proto -vet: - go vet ./... - -test: - go test ./... --cover - -test-coverage: - go test ./... -coverprofile coverage.out -covermode count - go tool cover -func coverage.out - deps: go mod download go build -o deps storj.io/drpc/cmd/protoc-gen-go-drpc go build -o deps github.com/gogo/protobuf/protoc-gen-gogofaster + +test: + go test ./... --cover + From b875c504a09e7e1a43d35ba9046b08c773d6f1fc Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 20:09:54 +0300 Subject: [PATCH 48/60] .github/workflows/coverage.yml add actions/cach --- .github/workflows/coverage.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 27b222a9..d6030aa8 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -17,6 +17,17 @@ jobs: - name: git config run: git config --global url.https://${{ secrets.ANYTYPE_PAT }}@github.com/.insteadOf https://github.com/ + - uses: actions/cache@v3 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + ~\AppData\Local\go-build + key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ matrix.go-version }}- + - name: deps run: make deps From ec96ecafcca7087dca8d89b85d8ea74fdfcf95e9 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 20:20:28 +0300 Subject: [PATCH 49/60] .github/workflows/coverage.yml add go-cache-paths --- .github/workflows/coverage.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index d6030aa8..62e0397e 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -17,16 +17,20 @@ jobs: - name: git config run: git config --global url.https://${{ secrets.ANYTYPE_PAT }}@github.com/.insteadOf https://github.com/ + # cache {{ + - id: go-cache-paths + run: | + echo "::set-output name=GOCACHE::$(go env GOCACHE)" + echo "::set-output name=GOMODCACHE::$(go env GOMODCACHE)" - uses: actions/cache@v3 with: path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - ~\AppData\Local\go-build + ${{ steps.go-cache-paths.outputs.GOCACHE }} + ${{ steps.go-cache-paths.outputs.GOMODCACHE }} key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go-${{ matrix.go-version }}- + # }} - name: deps run: make deps From c792ff1fcb15f89a31f1f72c0d8c9db2d6385182 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 20:25:15 +0300 Subject: [PATCH 50/60] fix set-output --- .github/workflows/coverage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 62e0397e..1f55870b 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -20,8 +20,8 @@ jobs: # cache {{ - id: go-cache-paths run: | - echo "::set-output name=GOCACHE::$(go env GOCACHE)" - echo "::set-output name=GOMODCACHE::$(go env GOMODCACHE)" + echo "GOCACHE=$(go env GOCACHE)" >> $GITHUB_OUTPUT + echo "GOMODCACHE=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT - uses: actions/cache@v3 with: path: | From d21dd91a77a196932b7cb5f1451d6239ea873bf3 Mon Sep 17 00:00:00 2001 From: Grigory Efimov Date: Wed, 11 Jan 2023 20:32:06 +0300 Subject: [PATCH 51/60] .github/workflows/coverage.yml update actions --- .github/workflows/coverage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 1f55870b..1780a7f4 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -9,8 +9,8 @@ jobs: env: GOPRIVATE: github.com/anytypeio steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 with: go-version: '^1.19.5' From 1eed7e50501e68123013ec0d5de0424b5a69509a Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 11 Jan 2023 19:04:47 +0100 Subject: [PATCH 52/60] Change object tree debug dump --- commonspace/object/tree/objecttree/objecttree.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/commonspace/object/tree/objecttree/objecttree.go b/commonspace/object/tree/objecttree/objecttree.go index 3dc11cf8..1809ec58 100644 --- a/commonspace/object/tree/objecttree/objecttree.go +++ b/commonspace/object/tree/objecttree/objecttree.go @@ -52,7 +52,7 @@ type ObjectTree interface { Heads() []string Root() *Change HasChanges(...string) bool - DebugDump() (string, error) + DebugDump(parser DescriptionParser) (string, error) IterateRoot(convert ChangeConvertFunc, iterate ChangeIterateFunc) error IterateFrom(id string, convert ChangeConvertFunc, iterate ChangeIterateFunc) error @@ -626,6 +626,6 @@ func (ot *objectTree) validateTree(newChanges []*Change) error { return ot.validator.ValidateNewChanges(ot.tree, ot.aclList, newChanges) } -func (ot *objectTree) DebugDump() (string, error) { - return ot.tree.Graph(NoOpDescriptionParser) +func (ot *objectTree) DebugDump(parser DescriptionParser) (string, error) { + return ot.tree.Graph(parser) } From a7a54bcc8bb2853c829e27813e600a2845913611 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 11 Jan 2023 19:18:04 +0100 Subject: [PATCH 53/60] Add handling for root ids --- commonspace/object/tree/objecttree/descriptionparser.go | 4 ++-- commonspace/object/tree/objecttree/treegraph_nix.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/commonspace/object/tree/objecttree/descriptionparser.go b/commonspace/object/tree/objecttree/descriptionparser.go index a3e79a04..0edc35f9 100644 --- a/commonspace/object/tree/objecttree/descriptionparser.go +++ b/commonspace/object/tree/objecttree/descriptionparser.go @@ -1,13 +1,13 @@ package objecttree type DescriptionParser interface { - ParseChange(*Change) ([]string, error) + ParseChange(ch *Change, isRoot bool) ([]string, error) } var NoOpDescriptionParser = noopDescriptionParser{} type noopDescriptionParser struct{} -func (n noopDescriptionParser) ParseChange(change *Change) ([]string, error) { +func (n noopDescriptionParser) ParseChange(ch *Change, isRoot bool) ([]string, error) { return []string{"DOC"}, nil } diff --git a/commonspace/object/tree/objecttree/treegraph_nix.go b/commonspace/object/tree/objecttree/treegraph_nix.go index 355ca96d..d8e104cf 100644 --- a/commonspace/object/tree/objecttree/treegraph_nix.go +++ b/commonspace/object/tree/objecttree/treegraph_nix.go @@ -50,7 +50,7 @@ func (t *Tree) Graph(parser DescriptionParser) (data string, err error) { if ord == "" { ord = "miss" } - chSymbs, err := parser.ParseChange(c) + chSymbs, err := parser.ParseChange(c, c.Id == t.RootId()) if err != nil { return err } From cfbae15999598ab55519306c8b5629cf8291e59c Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Thu, 12 Jan 2023 13:53:32 +0300 Subject: [PATCH 54/60] net pool metrics --- app/ocache/metrics.go | 3 +++ app/ocache/ocache.go | 4 +++- net/pool/pool.go | 7 +++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/app/ocache/metrics.go b/app/ocache/metrics.go index 77bbfee3..b5fe9aa4 100644 --- a/app/ocache/metrics.go +++ b/app/ocache/metrics.go @@ -6,6 +6,9 @@ func WithPrometheus(reg *prometheus.Registry, namespace, subsystem string) Optio if subsystem == "" { subsystem = "cache" } + if reg == nil { + return nil + } return func(cache *oCache) { cache.metrics = &metrics{ hit: prometheus.NewCounter(prometheus.CounterOpts{ diff --git a/app/ocache/ocache.go b/app/ocache/ocache.go index 8abf14ce..31852286 100644 --- a/app/ocache/ocache.go +++ b/app/ocache/ocache.go @@ -61,7 +61,9 @@ func New(loadFunc LoadFunc, opts ...Option) OCache { log: log.Sugar(), } for _, o := range opts { - o(c) + if o != nil { + o(c) + } } if c.ttl != 0 && c.gc != 0 { go c.ticker() diff --git a/net/pool/pool.go b/net/pool/pool.go index 5ca10c4e..0b202106 100644 --- a/net/pool/pool.go +++ b/net/pool/pool.go @@ -6,8 +6,10 @@ import ( "github.com/anytypeio/any-sync/app" "github.com/anytypeio/any-sync/app/logger" "github.com/anytypeio/any-sync/app/ocache" + "github.com/anytypeio/any-sync/metric" "github.com/anytypeio/any-sync/net/dialer" "github.com/anytypeio/any-sync/net/peer" + "github.com/prometheus/client_golang/prometheus" "math/rand" "time" ) @@ -47,6 +49,10 @@ type pool struct { func (p *pool) Init(a *app.App) (err error) { p.dialer = a.MustComponent(dialer.CName).(dialer.Dialer) + var reg *prometheus.Registry + if m := a.Component(metric.CName); m != nil { + reg = m.(metric.Metric).Registry() + } p.cache = ocache.New( func(ctx context.Context, id string) (value ocache.Object, err error) { return p.dialer.Dial(ctx, id) @@ -54,6 +60,7 @@ func (p *pool) Init(a *app.App) (err error) { ocache.WithLogger(log.Sugar()), ocache.WithGCPeriod(time.Minute), ocache.WithTTL(time.Minute*5), + ocache.WithPrometheus(reg, "netpool", "cache"), ) return nil } From 4cb79eeab95526a2897198b5e276864e4dcebc06 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 13 Jan 2023 10:14:50 +0300 Subject: [PATCH 55/60] nodeconf: expose CHash --- nodeconf/configuration.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nodeconf/configuration.go b/nodeconf/configuration.go index 75924a57..af8e5f33 100644 --- a/nodeconf/configuration.go +++ b/nodeconf/configuration.go @@ -16,6 +16,8 @@ type Configuration interface { ConsensusPeers() []string // Addresses returns map[peerId][]addr with connection addresses for all known nodes Addresses() map[string][]string + // CHash returns nodes consistent table + CHash() chash.CHash } type configuration struct { @@ -66,3 +68,7 @@ func (c *configuration) Addresses() map[string][]string { } return res } + +func (c *configuration) CHash() chash.CHash { + return c.chash +} From 059f0ca636389d471476033606cc6d607651f177 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 13 Jan 2023 10:40:34 +0300 Subject: [PATCH 56/60] update chash --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 832c8a78..8bd2125b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/anytypeio/any-sync go 1.19 require ( - github.com/anytypeio/go-chash v0.0.1 + github.com/anytypeio/go-chash v0.0.2 github.com/awalterschulze/gographviz v2.0.3+incompatible github.com/cespare/xxhash v1.1.0 github.com/cheggaaa/mb/v3 v3.0.0 diff --git a/go.sum b/go.sum index 782a3638..fae66ad0 100644 --- a/go.sum +++ b/go.sum @@ -42,10 +42,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/anytypeio/go-chash v0.0.0-20220629194632-4ad1154fe232 h1:kMPPZYmJgbs4AJfodbg2OCXg5cp+9LPAJcLZJqmcghk= -github.com/anytypeio/go-chash v0.0.0-20220629194632-4ad1154fe232/go.mod h1:+PeHBAWp7gUh/yw6uAauKc5ku0w4cFNg6DUddGxoGq0= -github.com/anytypeio/go-chash v0.0.1 h1:gmhq5c8qweYc+QiyJK2TCzSIp55HlaJlTD5aPUi8TFA= -github.com/anytypeio/go-chash v0.0.1/go.mod h1:MTQdZA8gD53OH9VFCt33w5WpIs5kSmBJiSYC2iEGSSU= +github.com/anytypeio/go-chash v0.0.2 h1:BSpyMC3HXNkf2eosQrHM4svov0DrvxL9tb4gnHbdmbA= +github.com/anytypeio/go-chash v0.0.2/go.mod h1:G+R6q7jYgNa52NqcRhnNm28pogfWW+cuHtgBktrc2QA= github.com/awalterschulze/gographviz v2.0.3+incompatible h1:9sVEXJBJLwGX7EQVhLm2elIKCm7P2YHFC8v6096G09E= github.com/awalterschulze/gographviz v2.0.3+incompatible/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= From 8cf6ad560e8ddc96594535ee97cc3a476529a6db Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 13 Jan 2023 12:11:20 +0300 Subject: [PATCH 57/60] test account improve --- testutil/accounttest/accountservice.go | 30 ++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/testutil/accounttest/accountservice.go b/testutil/accounttest/accountservice.go index d8c3268b..e745f1e7 100644 --- a/testutil/accounttest/accountservice.go +++ b/testutil/accounttest/accountservice.go @@ -4,8 +4,11 @@ import ( accountService "github.com/anytypeio/any-sync/accountservice" "github.com/anytypeio/any-sync/app" "github.com/anytypeio/any-sync/commonspace/object/accountdata" + "github.com/anytypeio/any-sync/nodeconf" + "github.com/anytypeio/any-sync/util/keys" "github.com/anytypeio/any-sync/util/keys/asymmetric/encryptionkey" "github.com/anytypeio/any-sync/util/keys/asymmetric/signingkey" + "github.com/anytypeio/any-sync/util/peer" ) // AccountTestService provides service for test purposes, generates new random account every Init @@ -14,6 +17,9 @@ type AccountTestService struct { } func (s *AccountTestService) Init(a *app.App) (err error) { + if s.acc != nil { + return + } encKey, _, err := encryptionkey.GenerateRandomRSAKeyPair(2048) if err != nil { return @@ -27,7 +33,13 @@ func (s *AccountTestService) Init(a *app.App) (err error) { if err != nil { return } + + peerId, err := peer.IdFromSigningPubKey(signKey.GetPublic()) + if err != nil { + return err + } s.acc = &accountdata.AccountData{ + PeerId: peerId.String(), Identity: ident, SignKey: signKey, EncKey: encKey, @@ -42,3 +54,21 @@ func (s *AccountTestService) Name() (name string) { func (s *AccountTestService) Account() *accountdata.AccountData { return s.acc } + +func (s *AccountTestService) NodeConf(addrs []string) nodeconf.NodeConfig { + encSk, err := keys.EncodeKeyToString(s.acc.SignKey) + if err != nil { + panic(err) + } + encEk, err := keys.EncodeKeyToString(s.acc.EncKey) + if err != nil { + panic(err) + } + return nodeconf.NodeConfig{ + PeerId: s.acc.PeerId, + Addresses: addrs, + SigningKey: encSk, + EncryptionKey: encEk, + Types: []nodeconf.NodeType{nodeconf.NodeTypeTree}, + } +} From 4c4abc926633196be846048554ead2fee41ccd43 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 13 Jan 2023 11:42:44 +0100 Subject: [PATCH 58/60] Update any-sync to wait sync for docs --- .../mock_objecttree/mock_objecttree.go | 8 ++-- .../synctree/mock_synctree/mock_synctree.go | 8 ++-- commonspace/object/tree/synctree/synctree.go | 42 ++++++++++++++----- commonspace/space.go | 33 +++++++++------ 4 files changed, 60 insertions(+), 31 deletions(-) diff --git a/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go b/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go index 0af84819..6e743ba5 100644 --- a/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go +++ b/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go @@ -97,18 +97,18 @@ func (mr *MockObjectTreeMockRecorder) Close() *gomock.Call { } // DebugDump mocks base method. -func (m *MockObjectTree) DebugDump() (string, error) { +func (m *MockObjectTree) DebugDump(arg0 objecttree.DescriptionParser) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DebugDump") + ret := m.ctrl.Call(m, "DebugDump", arg0) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // DebugDump indicates an expected call of DebugDump. -func (mr *MockObjectTreeMockRecorder) DebugDump() *gomock.Call { +func (mr *MockObjectTreeMockRecorder) DebugDump(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DebugDump", reflect.TypeOf((*MockObjectTree)(nil).DebugDump)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DebugDump", reflect.TypeOf((*MockObjectTree)(nil).DebugDump), arg0) } // Delete mocks base method. diff --git a/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go b/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go index 61c3cfb7..1ca1e659 100644 --- a/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go +++ b/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go @@ -221,18 +221,18 @@ func (mr *MockSyncTreeMockRecorder) Close() *gomock.Call { } // DebugDump mocks base method. -func (m *MockSyncTree) DebugDump() (string, error) { +func (m *MockSyncTree) DebugDump(arg0 objecttree.DescriptionParser) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DebugDump") + ret := m.ctrl.Call(m, "DebugDump", arg0) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // DebugDump indicates an expected call of DebugDump. -func (mr *MockSyncTreeMockRecorder) DebugDump() *gomock.Call { +func (mr *MockSyncTreeMockRecorder) DebugDump(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DebugDump", reflect.TypeOf((*MockSyncTree)(nil).DebugDump)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DebugDump", reflect.TypeOf((*MockSyncTree)(nil).DebugDump), arg0) } // Delete mocks base method. diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index 8279cb93..4aae1bf7 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -55,16 +55,17 @@ var buildObjectTree = objecttree.BuildObjectTree var createSyncClient = newWrappedSyncClient type BuildDeps struct { - SpaceId string - ObjectSync objectsync.ObjectSync - Configuration nodeconf.Configuration - HeadNotifiable HeadNotifiable - Listener updatelistener.UpdateListener - AclList list.AclList - SpaceStorage spacestorage.SpaceStorage - TreeStorage treestorage.TreeStorage - TreeUsage *atomic.Int32 - SyncStatus syncstatus.StatusUpdater + SpaceId string + ObjectSync objectsync.ObjectSync + Configuration nodeconf.Configuration + HeadNotifiable HeadNotifiable + Listener updatelistener.UpdateListener + AclList list.AclList + SpaceStorage spacestorage.SpaceStorage + TreeStorage treestorage.TreeStorage + TreeUsage *atomic.Int32 + SyncStatus syncstatus.StatusUpdater + WaitTreeRemoteSync bool } func newWrappedSyncClient( @@ -104,6 +105,25 @@ func BuildSyncTreeOrGetRemote(ctx context.Context, id string, deps BuildDeps) (t return } + waitTree := func(wait bool) (msg *treechangeproto.TreeSyncMessage, err error) { + if !wait { + return getTreeRemote() + } + for { + msg, err = getTreeRemote() + if err == nil { + return + } + select { + case <-ctx.Done(): + err = fmt.Errorf("waiting for object %s interrupted, context closed", id) + return + default: + break + } + } + } + deps.TreeStorage, err = deps.SpaceStorage.TreeStorage(id) if err == nil { return buildSyncTree(ctx, false, deps) @@ -122,7 +142,7 @@ func BuildSyncTreeOrGetRemote(ctx context.Context, id string, deps BuildDeps) (t return } - resp, err := getTreeRemote() + resp, err := waitTree(deps.WaitTreeRemoteSync) if err != nil { return } diff --git a/commonspace/space.go b/commonspace/space.go index fa33a9ea..a52f4d82 100644 --- a/commonspace/space.go +++ b/commonspace/space.go @@ -84,7 +84,7 @@ type Space interface { DeriveTree(ctx context.Context, payload objecttree.ObjectTreeCreatePayload) (res treestorage.TreeStorageCreatePayload, err error) CreateTree(ctx context.Context, payload objecttree.ObjectTreeCreatePayload) (res treestorage.TreeStorageCreatePayload, err error) PutTree(ctx context.Context, payload treestorage.TreeStorageCreatePayload, listener updatelistener.UpdateListener) (t objecttree.ObjectTree, err error) - BuildTree(ctx context.Context, id string, listener updatelistener.UpdateListener) (objecttree.ObjectTree, error) + BuildTree(ctx context.Context, id string, opts BuildTreeOpts) (t objecttree.ObjectTree, err error) DeleteTree(ctx context.Context, id string) (err error) SyncStatus() syncstatus.StatusUpdater @@ -176,7 +176,10 @@ func (s *space) Init(ctx context.Context) (err error) { deletionState := deletionstate.NewDeletionState(s.storage) deps := settings.Deps{ BuildFunc: func(ctx context.Context, id string, listener updatelistener.UpdateListener) (t synctree.SyncTree, err error) { - res, err := s.BuildTree(ctx, id, listener) + res, err := s.BuildTree(ctx, id, BuildTreeOpts{ + Listener: listener, + WaitTreeRemoteSync: false, + }) if err != nil { return } @@ -289,22 +292,28 @@ func (s *space) PutTree(ctx context.Context, payload treestorage.TreeStorageCrea return } -func (s *space) BuildTree(ctx context.Context, id string, listener updatelistener.UpdateListener) (t objecttree.ObjectTree, err error) { +type BuildTreeOpts struct { + Listener updatelistener.UpdateListener + WaitTreeRemoteSync bool +} + +func (s *space) BuildTree(ctx context.Context, id string, opts BuildTreeOpts) (t objecttree.ObjectTree, err error) { if s.isClosed.Load() { err = ErrSpaceClosed return } deps := synctree.BuildDeps{ - SpaceId: s.id, - ObjectSync: s.objectSync, - Configuration: s.configuration, - HeadNotifiable: s.headSync, - Listener: listener, - AclList: s.aclList, - SpaceStorage: s.storage, - TreeUsage: &s.treesUsed, - SyncStatus: s.syncStatus, + SpaceId: s.id, + ObjectSync: s.objectSync, + Configuration: s.configuration, + HeadNotifiable: s.headSync, + Listener: opts.Listener, + AclList: s.aclList, + SpaceStorage: s.storage, + TreeUsage: &s.treesUsed, + SyncStatus: s.syncStatus, + WaitTreeRemoteSync: opts.WaitTreeRemoteSync, } return synctree.BuildSyncTreeOrGetRemote(ctx, id, deps) } From 25a52f41b8dc5add23734631b326548dfb12f97c Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 13 Jan 2023 12:55:28 +0100 Subject: [PATCH 59/60] Update synctree to connect with responsible node --- commonspace/object/tree/synctree/synctree.go | 7 ++++++- commonspace/objectsync/streamchecker.go | 15 ++++++++++++++- nodeconf/mock_nodeconf/mock_nodeconf.go | 15 +++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index 4aae1bf7..c0f4c462 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -79,9 +79,14 @@ func newWrappedSyncClient( func BuildSyncTreeOrGetRemote(ctx context.Context, id string, deps BuildDeps) (t SyncTree, err error) { getTreeRemote := func() (msg *treechangeproto.TreeSyncMessage, err error) { + streamChecker := deps.ObjectSync.StreamChecker() peerId, err := peer.CtxPeerId(ctx) if err != nil { - return + streamChecker.CheckResponsiblePeers() + peerId, err = streamChecker.FirstResponsiblePeer() + if err != nil { + return + } } newTreeRequest := GetRequestFactory().CreateNewTreeRequest() diff --git a/commonspace/objectsync/streamchecker.go b/commonspace/objectsync/streamchecker.go index 4f99794d..5bf53e5e 100644 --- a/commonspace/objectsync/streamchecker.go +++ b/commonspace/objectsync/streamchecker.go @@ -16,6 +16,7 @@ import ( type StreamChecker interface { CheckResponsiblePeers() CheckPeerConnection(peerId string) (err error) + FirstResponsiblePeer() (peerId string, err error) } type streamChecker struct { @@ -28,7 +29,7 @@ type streamChecker struct { lastCheck *atomic.Time } -const streamCheckerInterval = time.Second * 10 +const streamCheckerInterval = time.Second * 5 func NewStreamChecker( spaceId string, @@ -131,3 +132,15 @@ func (s *streamChecker) createStream(p peer.Peer) (err error) { } return } + +func (s *streamChecker) FirstResponsiblePeer() (peerId string, err error) { + nodeIds := s.connector.Configuration().NodeIds(s.spaceId) + for _, nodeId := range nodeIds { + if s.streamPool.HasActiveStream(nodeId) { + peerId = nodeId + return + } + } + err = fmt.Errorf("no responsible peers are connected") + return +} diff --git a/nodeconf/mock_nodeconf/mock_nodeconf.go b/nodeconf/mock_nodeconf/mock_nodeconf.go index 38d5ef64..9e0bd4f4 100644 --- a/nodeconf/mock_nodeconf/mock_nodeconf.go +++ b/nodeconf/mock_nodeconf/mock_nodeconf.go @@ -9,6 +9,7 @@ import ( app "github.com/anytypeio/any-sync/app" nodeconf "github.com/anytypeio/any-sync/nodeconf" + chash "github.com/anytypeio/go-chash" gomock "github.com/golang/mock/gomock" ) @@ -128,6 +129,20 @@ func (mr *MockConfigurationMockRecorder) Addresses() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Addresses", reflect.TypeOf((*MockConfiguration)(nil).Addresses)) } +// CHash mocks base method. +func (m *MockConfiguration) CHash() chash.CHash { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CHash") + ret0, _ := ret[0].(chash.CHash) + return ret0 +} + +// CHash indicates an expected call of CHash. +func (mr *MockConfigurationMockRecorder) CHash() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CHash", reflect.TypeOf((*MockConfiguration)(nil).CHash)) +} + // ConsensusPeers mocks base method. func (m *MockConfiguration) ConsensusPeers() []string { m.ctrl.T.Helper() From b6be4e89757b6129eaad9b4b0ac20f39a8b29b5b Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 13 Jan 2023 12:59:13 +0100 Subject: [PATCH 60/60] Change PutTree --- commonspace/space.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/commonspace/space.go b/commonspace/space.go index a52f4d82..7f9936fc 100644 --- a/commonspace/space.go +++ b/commonspace/space.go @@ -284,12 +284,7 @@ func (s *space) PutTree(ctx context.Context, payload treestorage.TreeStorageCrea TreeUsage: &s.treesUsed, SyncStatus: s.syncStatus, } - t, err = synctree.PutSyncTree(ctx, payload, deps) - // this can happen only for derived trees, when we've synced same tree already - if err == treestorage.ErrTreeExists { - return synctree.BuildSyncTreeOrGetRemote(ctx, payload.RootRawChange.Id, deps) - } - return + return synctree.PutSyncTree(ctx, payload, deps) } type BuildTreeOpts struct {