diff --git a/net/rpc/server/baseserver.go b/net/rpc/server/baseserver.go
index 7a3a598f..49136aaf 100644
--- a/net/rpc/server/baseserver.go
+++ b/net/rpc/server/baseserver.go
@@ -77,10 +77,6 @@ func (s *BaseDrpcServer) serve(ctx context.Context, lis net.Listener) {
 				}
 				continue
 			}
-			if herr, ok := err.(secureservice.HandshakeError); ok {
-				l.Warn("listener handshake error", zap.Error(herr), zap.String("remoteAddr", herr.RemoteAddr()))
-				continue
-			}
 			l.Error("listener accept error", zap.Error(err))
 			return
 		}
@@ -98,6 +94,7 @@ func (s *BaseDrpcServer) serveConn(conn net.Conn) {
 		ctx, conn, err = s.handshake(conn)
 		if err != nil {
 			l.Info("handshake error", zap.Error(err))
+			return
 		}
 	}