Properly handle async calls to restart the server

The watchdog thread calls the server restart function asynchronously. Prior to
this change, it attempted to do several non-safe operations from the watchdog
thread, rather than the main. Specifically, because of a separate upstream change,
it causes player entities to be ticked asynchronously, among other things.

This is dangerous.

This patch moves the old handling into a synchronous variant, for calls from the
restart command, and adds separate handling for async calls, such as those from
the watchdog thread.

When calling from the watchdog thread, we cannot assume the main thread is in a
tickable state; it may be completely deadlocked. In order to handle this, we mark
the server as stopping, in order to account for situations where the server should
complete a tick reasonbly soon, i.e. 99% of cases.

Should the server not enter a state where it is stopping within 10 seconds, We
will assume that the server has in fact deadlocked and will proceed to force
kill the server.

This modification does not force restart the server should we actually enter a
deadlocked state where the server is stopping, whereas this will in most cases
exit within a reasonable amount of time, to put a fixed limit on a process that
will have plugins and worlds saving to the disk has a high potential to result
in corruption/dataloss.

paper-server/patches/sources/net/minecraft/server/players/PlayerList.java.patch

@@ -729,14 +729,10 @@
              this.sendAllPlayerInfoIn = 0;
          }
  
-@@ -537,9 +832,28 @@
-             ServerPlayer entityplayer = (ServerPlayer) iterator.next();
+@@ -541,6 +836,25 @@
+ 
+     }
  
-             entityplayer.connection.send(packet);
-+        }
-+
-+    }
-+
 +    // CraftBukkit start - add a world/entity limited version
 +    public void broadcastAll(Packet packet, net.minecraft.world.entity.player.Player entityhuman) {
 +        for (int i = 0; i < this.players.size(); ++i) {
@@ -751,13 +747,14 @@
 +    public void broadcastAll(Packet packet, Level world) {
 +        for (int i = 0; i < world.players().size(); ++i) {
 +            ((ServerPlayer) world.players().get(i)).connection.send(packet);
-         }
- 
-     }
++        }
++
++    }
 +    // CraftBukkit end
- 
++
      public void broadcastAll(Packet<?> packet, ResourceKey<Level> dimension) {
          Iterator iterator = this.players.iterator();
+ 
 @@ -554,7 +868,7 @@
  
      }
@@ -880,14 +877,21 @@
      }
  
      public int getPlayerCount() {
-@@ -786,12 +1111,29 @@
+@@ -786,12 +1111,36 @@
      }
  
      public void removeAll() {
 -        for (int i = 0; i < this.players.size(); ++i) {
 -            ((ServerPlayer) this.players.get(i)).connection.disconnect((Component) Component.translatable("multiplayer.disconnect.server_shutdown"));
++        // Paper start - Extract method to allow for restarting flag
++        this.removeAll(false);
++    }
++
++    public void removeAll(boolean isRestarting) {
++        // Paper end
 +        // CraftBukkit start - disconnect safely
 +        for (ServerPlayer player : this.players) {
++            if (isRestarting) player.connection.disconnect(net.kyori.adventure.text.serializer.legacy.LegacyComponentSerializer.legacySection().deserialize(org.spigotmc.SpigotConfig.restartMessage)); else // Paper
 +            player.connection.disconnect(java.util.Objects.requireNonNullElseGet(this.server.server.shutdownMessage(), net.kyori.adventure.text.Component::empty)); // CraftBukkit - add custom shutdown message // Paper - Adventure
          }
 +        // CraftBukkit end
@@ -912,7 +916,7 @@
      public void broadcastSystemMessage(Component message, boolean overlay) {
          this.broadcastSystemMessage(message, (entityplayer) -> {
              return message;
-@@ -819,24 +1161,43 @@
+@@ -819,24 +1168,43 @@
      }
  
      public void broadcastChatMessage(PlayerChatMessage message, ServerPlayer sender, ChatType.Bound params) {
@@ -959,7 +963,7 @@
          }
  
          if (flag1 && sender != null) {
-@@ -845,20 +1206,27 @@
+@@ -845,20 +1213,27 @@
  
      }
  
@@ -992,7 +996,7 @@
                  Path path = file2.toPath();
  
                  if (FileUtil.isPathNormalized(path) && FileUtil.isPathPortable(path) && path.startsWith(file.getPath()) && file2.isFile()) {
-@@ -867,7 +1235,7 @@
+@@ -867,7 +1242,7 @@
              }
  
              serverstatisticmanager = new ServerStatsCounter(this.server, file1);
@@ -1001,7 +1005,7 @@
          }
  
          return serverstatisticmanager;
-@@ -875,13 +1243,13 @@
+@@ -875,13 +1250,13 @@
  
      public PlayerAdvancements getPlayerAdvancements(ServerPlayer player) {
          UUID uuid = player.getUUID();
@@ -1017,7 +1021,7 @@
          }
  
          advancementdataplayer.setPlayer(player);
-@@ -932,15 +1300,28 @@
+@@ -932,15 +1307,28 @@
      }
  
      public void reloadResources() {