diff --git a/net/pool/pool.go b/net/pool/pool.go
index 287ab512..b2e4ae22 100644
--- a/net/pool/pool.go
+++ b/net/pool/pool.go
@@ -67,6 +67,7 @@ func (p *pool) GetOneOf(ctx context.Context, peerIds []string) (peer.Peer, error
 			default:
 				return pr, nil
 			}
+			_, _ = p.cache.Remove(ctx, peerId)
 		}
 	}
 	// shuffle ids for better consistency