Add Early Warning Feature to WatchDog

Detect when the server has been hung for a long duration, and start printing
thread dumps at an interval until the point of crash.

This will help diagnose what was going on in that time before the crash.

paper-server/patches/sources/net/minecraft/server/MinecraftServer.java.patch

@@ -822,7 +822,7 @@
      protected void runServer() {
          try {
              if (!this.initServer()) {
-@@ -727,9 +1116,15 @@
+@@ -727,9 +1116,16 @@
              }
  
              this.nextTickTimeNanos = Util.getNanos();
@@ -831,6 +831,7 @@
              this.status = this.buildServerStatus();
  
 +            // Spigot start
++            org.spigotmc.WatchdogThread.hasStarted = true; // Paper
 +            Arrays.fill( this.recentTps, 20 );
 +            // Paper start - further improve server tick loop
 +            long tickSection = Util.getNanos();
@@ -839,7 +840,7 @@
              while (this.running) {
                  long i;
  
-@@ -744,11 +1139,30 @@
+@@ -744,11 +1140,30 @@
                      if (j > MinecraftServer.OVERLOADED_THRESHOLD_NANOS + 20L * i && this.nextTickTimeNanos - this.lastOverloadWarningNanos >= MinecraftServer.OVERLOADED_WARNING_INTERVAL_NANOS + 100L * i) {
                          long k = j / i;
  
@@ -870,7 +871,7 @@
  
                  boolean flag = i == 0L;
  
-@@ -757,6 +1171,8 @@
+@@ -757,6 +1172,8 @@
                      this.debugCommandProfiler = new MinecraftServer.TimeProfiler(Util.getNanos(), this.tickCount);
                  }
  
@@ -879,7 +880,7 @@
                  this.nextTickTimeNanos += i;
  
                  try {
-@@ -830,6 +1246,13 @@
+@@ -830,6 +1247,13 @@
                      this.services.profileCache().clearExecutor();
                  }
  
@@ -893,7 +894,7 @@
                  this.onServerExit();
              }
  
-@@ -889,9 +1312,16 @@
+@@ -889,9 +1313,16 @@
      }
  
      private boolean haveTime() {
@@ -911,7 +912,7 @@
      public static boolean throwIfFatalException() {
          RuntimeException runtimeexception = (RuntimeException) MinecraftServer.fatalException.get();
  
-@@ -903,7 +1333,7 @@
+@@ -903,7 +1334,7 @@
      }
  
      public static void setFatalException(RuntimeException exception) {
@@ -920,7 +921,7 @@
      }
  
      @Override
-@@ -977,7 +1407,7 @@
+@@ -977,7 +1408,7 @@
          }
      }
  
@@ -929,7 +930,7 @@
          Profiler.get().incrementCounter("runTask");
          super.doRunTask(ticktask);
      }
-@@ -1025,6 +1455,7 @@
+@@ -1025,6 +1456,7 @@
      }
  
      public void tickServer(BooleanSupplier shouldKeepTicking) {
@@ -937,7 +938,7 @@
          long i = Util.getNanos();
          int j = this.pauseWhileEmptySeconds() * 20;
  
-@@ -1041,6 +1472,7 @@
+@@ -1041,6 +1473,7 @@
                      this.autoSave();
                  }
  
@@ -945,7 +946,7 @@
                  this.tickConnection();
                  return;
              }
-@@ -1055,12 +1487,13 @@
+@@ -1055,12 +1488,13 @@
          }
  
          --this.ticksUntilAutosave;
@@ -960,7 +961,7 @@
          gameprofilerfiller.push("tallying");
          long k = Util.getNanos() - i;
          int l = this.tickCount % 100;
-@@ -1074,7 +1507,7 @@
+@@ -1074,7 +1508,7 @@
      }
  
      private void autoSave() {
@@ -969,7 +970,7 @@
          MinecraftServer.LOGGER.debug("Autosave started");
          ProfilerFiller gameprofilerfiller = Profiler.get();
  
-@@ -1123,7 +1556,7 @@
+@@ -1123,7 +1557,7 @@
      private ServerStatus buildServerStatus() {
          ServerStatus.Players serverping_serverpingplayersample = this.buildPlayerStatus();
  
@@ -978,7 +979,7 @@
      }
  
      private ServerStatus.Players buildPlayerStatus() {
-@@ -1133,7 +1566,7 @@
+@@ -1133,7 +1567,7 @@
          if (this.hidesOnlinePlayers()) {
              return new ServerStatus.Players(i, list.size(), List.of());
          } else {
@@ -987,7 +988,7 @@
              ObjectArrayList<GameProfile> objectarraylist = new ObjectArrayList(j);
              int k = Mth.nextInt(this.random, 0, list.size() - j);
  
-@@ -1154,24 +1587,43 @@
+@@ -1154,24 +1588,43 @@
          this.getPlayerList().getPlayers().forEach((entityplayer) -> {
              entityplayer.connection.suspendFlushing();
          });
@@ -1031,7 +1032,7 @@
  
              gameprofilerfiller.push("tick");
  
-@@ -1186,6 +1638,7 @@
+@@ -1186,6 +1639,7 @@
  
              gameprofilerfiller.pop();
              gameprofilerfiller.pop();
@@ -1039,7 +1040,7 @@
          }
  
          gameprofilerfiller.popPush("connection");
-@@ -1265,8 +1718,24 @@
+@@ -1265,8 +1719,24 @@
      @Nullable
      public ServerLevel getLevel(ResourceKey<Level> key) {
          return (ServerLevel) this.levels.get(key);
@@ -1064,7 +1065,7 @@
      public Set<ResourceKey<Level>> levelKeys() {
          return this.levels.keySet();
      }
-@@ -1296,7 +1765,7 @@
+@@ -1296,7 +1766,7 @@
  
      @DontObfuscate
      public String getServerModName() {
@@ -1073,7 +1074,7 @@
      }
  
      public SystemReport fillSystemReport(SystemReport details) {
-@@ -1347,7 +1816,7 @@
+@@ -1347,7 +1817,7 @@
  
      @Override
      public void sendSystemMessage(Component message) {
@@ -1082,7 +1083,7 @@
      }
  
      public KeyPair getKeyPair() {
-@@ -1481,10 +1950,20 @@
+@@ -1481,10 +1951,20 @@
  
      @Override
      public String getMotd() {
@@ -1104,7 +1105,7 @@
          this.motd = motd;
      }
  
-@@ -1507,7 +1986,7 @@
+@@ -1507,7 +1987,7 @@
      }
  
      public ServerConnectionListener getConnection() {
@@ -1113,7 +1114,7 @@
      }
  
      public boolean isReady() {
-@@ -1634,11 +2113,11 @@
+@@ -1634,11 +2114,11 @@
  
      public CompletableFuture<Void> reloadResources(Collection<String> dataPacks) {
          CompletableFuture<Void> completablefuture = CompletableFuture.supplyAsync(() -> {
@@ -1127,7 +1128,7 @@
          }, this).thenCompose((immutablelist) -> {
              MultiPackResourceManager resourcemanager = new MultiPackResourceManager(PackType.SERVER_DATA, immutablelist);
              List<Registry.PendingTags<?>> list = TagLoader.loadTagsForExistingRegistries(resourcemanager, this.registries.compositeAccess());
-@@ -1654,6 +2133,7 @@
+@@ -1654,6 +2134,7 @@
          }).thenAcceptAsync((minecraftserver_reloadableresources) -> {
              this.resources.close();
              this.resources = minecraftserver_reloadableresources;
@@ -1135,7 +1136,7 @@
              this.packRepository.setSelected(dataPacks);
              WorldDataConfiguration worlddataconfiguration = new WorldDataConfiguration(MinecraftServer.getSelectedPacks(this.packRepository, true), this.worldData.enabledFeatures());
  
-@@ -1952,7 +2432,7 @@
+@@ -1952,7 +2433,7 @@
              final List<String> list = Lists.newArrayList();
              final GameRules gamerules = this.getGameRules();
  
@@ -1144,7 +1145,7 @@
                  @Override
                  public <T extends GameRules.Value<T>> void visit(GameRules.Key<T> key, GameRules.Type<T> type) {
                      list.add(String.format(Locale.ROOT, "%s=%s\n", key.getId(), gamerules.getRule(key)));
-@@ -2058,7 +2538,7 @@
+@@ -2058,7 +2539,7 @@
              try {
                  label51:
                  {
@@ -1153,7 +1154,7 @@
  
                      try {
                          arraylist = Lists.newArrayList(NativeModuleLister.listModules());
-@@ -2108,6 +2588,21 @@
+@@ -2108,6 +2589,21 @@
  
      }
  
@@ -1175,7 +1176,7 @@
      private ProfilerFiller createProfiler() {
          if (this.willStartRecordingMetrics) {
              this.metricsRecorder = ActiveMetricsRecorder.createStarted(new ServerMetricsSamplersProvider(Util.timeSource, this.isDedicatedServer()), Util.timeSource, Util.ioPool(), new MetricsPersister("server"), this.onMetricsRecordingStopped, (path) -> {
-@@ -2225,18 +2720,24 @@
+@@ -2225,18 +2721,24 @@
      }
  
      public void logChatMessage(Component message, ChatType.Bound params, @Nullable String prefix) {

paper-server/patches/sources/net/minecraft/server/dedicated/DedicatedServer.java.patch

@@ -142,7 +142,7 @@
          thread.setDaemon(true);
          thread.setUncaughtExceptionHandler(new DefaultUncaughtExceptionHandler(DedicatedServer.LOGGER));
          thread.start();
-@@ -126,13 +203,26 @@
+@@ -126,13 +203,27 @@
              this.setPreventProxyConnections(dedicatedserverproperties.preventProxyConnections);
              this.setLocalIp(dedicatedserverproperties.serverIp);
          }
@@ -156,6 +156,7 @@
 +        this.paperConfigurations.initializeGlobalConfiguration(this.registryAccess());
 +        this.paperConfigurations.initializeWorldDefaultsConfiguration(this.registryAccess());
 +        // Paper end - initialize global and world-defaults configuration
++        org.spigotmc.WatchdogThread.doStart(org.spigotmc.SpigotConfig.timeoutTime, org.spigotmc.SpigotConfig.restartOnCrash); // Paper - start watchdog thread
 +        io.papermc.paper.command.PaperCommands.registerCommands(this); // Paper - setup /paper command
 +        com.destroystokyo.paper.Metrics.PaperMetrics.startMetrics(); // Paper - start metrics
 +        com.destroystokyo.paper.VersionHistoryManager.INSTANCE.getClass(); // Paper - load version history now
@@ -170,7 +171,7 @@
          DedicatedServer.LOGGER.info("Default game type: {}", dedicatedserverproperties.gamemode);
          InetAddress inetaddress = null;
  
-@@ -156,21 +246,34 @@
+@@ -156,21 +247,34 @@
              return false;
          }
  
@@ -208,7 +209,7 @@
              this.debugSampleSubscriptionTracker = new DebugSampleSubscriptionTracker(this.getPlayerList());
              this.tickTimeLogger = new RemoteSampleLogger(TpsDebugDimensions.values().length, this.debugSampleSubscriptionTracker, RemoteDebugSampleType.TICK_TIME);
              long i = Util.getNanos();
-@@ -178,13 +281,13 @@
+@@ -178,13 +282,13 @@
              SkullBlockEntity.setup(this.services, this);
              GameProfileCache.setUsesAuthentication(this.usesAuthentication());
              DedicatedServer.LOGGER.info("Preparing level \"{}\"", this.getLevelIdName());
@@ -224,7 +225,7 @@
              }
  
              if (dedicatedserverproperties.enableQuery) {
-@@ -197,7 +300,7 @@
+@@ -197,7 +301,7 @@
                  this.rconThread = RconThread.create(this);
              }
  
@@ -233,7 +234,7 @@
                  Thread thread1 = new Thread(new ServerWatchdog(this));
  
                  thread1.setUncaughtExceptionHandler(new DefaultUncaughtExceptionHandlerWithName(DedicatedServer.LOGGER));
-@@ -213,7 +316,13 @@
+@@ -213,7 +317,13 @@
  
              return true;
          }
@@ -247,7 +248,7 @@
  
      @Override
      public boolean isSpawningMonsters() {
-@@ -293,6 +402,7 @@
+@@ -293,6 +403,7 @@
              this.queryThreadGs4.stop();
          }
  
@@ -255,7 +256,7 @@
      }
  
      @Override
-@@ -302,8 +412,8 @@
+@@ -302,8 +413,8 @@
      }
  
      @Override
@@ -266,7 +267,7 @@
      }
  
      public void handleConsoleInput(String command, CommandSourceStack commandSource) {
-@@ -314,7 +424,15 @@
+@@ -314,7 +425,15 @@
          while (!this.consoleInput.isEmpty()) {
              ConsoleInput servercommand = (ConsoleInput) this.consoleInput.remove(0);
  
@@ -283,7 +284,7 @@
          }
  
      }
-@@ -383,7 +501,7 @@
+@@ -383,7 +502,7 @@
  
      @Override
      public boolean isUnderSpawnProtection(ServerLevel world, BlockPos pos, Player player) {
@@ -292,7 +293,7 @@
              return false;
          } else if (this.getPlayerList().getOps().isEmpty()) {
              return false;
-@@ -453,7 +571,11 @@
+@@ -453,7 +572,11 @@
      public boolean enforceSecureProfile() {
          DedicatedServerProperties dedicatedserverproperties = this.getProperties();
  
@@ -305,7 +306,7 @@
      }
  
      @Override
-@@ -541,16 +663,52 @@
+@@ -541,16 +664,52 @@
  
      @Override
      public String getPluginNames() {
@@ -362,7 +363,7 @@
      }
  
      public void storeUsingWhiteList(boolean useWhitelist) {
-@@ -660,4 +818,15 @@
+@@ -660,4 +819,15 @@
              }
          }
      }

paper-server/src/main/java/org/bukkit/craftbukkit/CraftServer.java

@@ -965,6 +965,7 @@ public final class CraftServer implements Server {
 
     @Override
     public void reload() {
+        org.spigotmc.WatchdogThread.hasStarted = false; // Paper - Disable watchdog early timeout on reload
         this.reloadCount++;
         this.configuration = YamlConfiguration.loadConfiguration(this.getConfigFile());
         this.commandsConfiguration = YamlConfiguration.loadConfiguration(this.getCommandsConfigFile());
@@ -1057,6 +1058,7 @@ public final class CraftServer implements Server {
         this.enablePlugins(PluginLoadOrder.POSTWORLD);
         if (io.papermc.paper.plugin.PluginInitializerManager.instance().pluginRemapper != null) io.papermc.paper.plugin.PluginInitializerManager.instance().pluginRemapper.pluginsEnabled(); // Paper - Remap plugins
         this.getPluginManager().callEvent(new ServerLoadEvent(ServerLoadEvent.LoadType.RELOAD));
+        org.spigotmc.WatchdogThread.hasStarted = true; // Paper - Disable watchdog early timeout on reload
     }
 
     @Override

paper-server/src/main/java/org/spigotmc/SpigotConfig.java

@@ -225,7 +225,7 @@ public class SpigotConfig
         SpigotConfig.restartScript = SpigotConfig.getString( "settings.restart-script", SpigotConfig.restartScript );
         SpigotConfig.restartMessage = SpigotConfig.transform( SpigotConfig.getString( "messages.restart", "Server is restarting" ) );
         SpigotConfig.commands.put( "restart", new RestartCommand( "restart" ) );
-        WatchdogThread.doStart( SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash );
+        // WatchdogThread.doStart( SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash ); // Paper - moved to after paper config initialization
     }
 
     public static boolean bungee;

paper-server/src/main/java/org/spigotmc/WatchdogThread.java

@@ -14,6 +14,10 @@ public class WatchdogThread extends Thread
     private static WatchdogThread instance;
     private long timeoutTime;
     private boolean restart;
+    private final long earlyWarningEvery; // Paper - Timeout time for just printing a dump but not restarting
+    private final long earlyWarningDelay; // Paper
+    public static volatile boolean hasStarted; // Paper
+    private long lastEarlyWarning; // Paper - Keep track of short dump times to avoid spamming console with short dumps
     private volatile long lastTick;
     private volatile boolean stopping;
 
@@ -22,6 +26,8 @@ public class WatchdogThread extends Thread
         super( "Paper Watchdog Thread" );
         this.timeoutTime = timeoutTime;
         this.restart = restart;
+        earlyWarningEvery = Math.min(io.papermc.paper.configuration.GlobalConfiguration.get().watchdog.earlyWarningEvery, timeoutTime); // Paper
+        earlyWarningDelay = Math.min(io.papermc.paper.configuration.GlobalConfiguration.get().watchdog.earlyWarningDelay, timeoutTime); // Paper
     }
 
     private static long monotonicMillis()
@@ -61,9 +67,18 @@ public class WatchdogThread extends Thread
         while ( !this.stopping )
         {
             //
-            if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.monotonicMillis() > this.lastTick + this.timeoutTime && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
+            // Paper start
+            Logger log = Bukkit.getServer().getLogger();
+            long currentTime = WatchdogThread.monotonicMillis();
+            if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
             {
-                Logger log = Bukkit.getServer().getLogger();
+                boolean isLongTimeout = currentTime > lastTick + timeoutTime;
+                // Don't spam early warning dumps
+                if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
+                if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
+                lastEarlyWarning = currentTime;
+                if (isLongTimeout) {
+                // Paper end
                 log.log( Level.SEVERE, "------------------------------" );
                 log.log( Level.SEVERE, "The server has stopped responding! This is (probably) not a Paper bug." ); // Paper
                 log.log( Level.SEVERE, "If you see a plugin in the Server thread dump below, then please report it to that author" );
@@ -92,29 +107,45 @@ public class WatchdogThread extends Thread
                     }
                 }
                 // Paper end
+                } else
+                {
+                    log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH  - " + Bukkit.getServer().getVersion() + " ---");
+                    log.log(Level.SEVERE, "The server has not responded for " + (currentTime - lastTick) / 1000 + " seconds! Creating thread dump");
+                }
+                // Paper end - Different message for short timeout
                 log.log( Level.SEVERE, "------------------------------" );
                 log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper
                 WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log );
                 log.log( Level.SEVERE, "------------------------------" );
                 //
+                // Paper start - Only print full dump on long timeouts
+                if ( isLongTimeout )
+                {
                 log.log( Level.SEVERE, "Entire Thread Dump:" );
                 ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true );
                 for ( ThreadInfo thread : threads )
                 {
                     WatchdogThread.dumpThread( thread, log );
                 }
+                } else {
+                    log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH ---");
+                }
+
                 log.log( Level.SEVERE, "------------------------------" );
 
+                if ( isLongTimeout )
+                {
                 if ( this.restart && !MinecraftServer.getServer().hasStopped() )
                 {
                     RestartCommand.restart();
                 }
                 break;
+                } // Paper end
             }
 
             try
             {
-                sleep( 10000 );
+                sleep( 1000 ); // Paper - Reduce check time to every second instead of every ten seconds, more consistent and allows for short timeout
             } catch ( InterruptedException ex )
             {
                 this.interrupt();