From 65b49384e0ccf5843fc69b1735c399d06ad204d7 Mon Sep 17 00:00:00 2001
From: Louis Lam <louislam@users.noreply.github.com>
Date: Sat, 26 Oct 2024 15:37:50 +0800
Subject: [PATCH] Clean up data

---
 server/database.js            | 37 +++++++++++++++++++++++++--
 server/jobs/clear-old-data.js | 47 ++++++++++++++---------------------
 server/uptime-calculator.js   |  2 +-
 3 files changed, 55 insertions(+), 31 deletions(-)

diff --git a/server/database.js b/server/database.js
index ce4e04e96..16cdecc5c 100644
--- a/server/database.js
+++ b/server/database.js
@@ -829,8 +829,7 @@ class Database {
             i++;
         }
 
-        // TODO: Remove all non-important heartbeats from heartbeat table
-        log.info("db", "[DON'T STOP] Deleting all data from heartbeat table");
+        await Database.clearHeartbeatData(true);
 
         await Settings.set("migrateAggregateTableState", "migrated");
 
@@ -841,6 +840,40 @@ class Database {
         }
     }
 
+    /**
+     * Remove all non-important heartbeats from heartbeat table, keep last 24-hour or {KEEP_LAST_ROWS} rows for each monitor
+     * @param {boolean} detailedLog
+     * @returns {Promise<void>}
+     */
+    static async clearHeartbeatData(detailedLog = false) {
+        let monitors = await R.getAll("SELECT id FROM monitor");
+        const sqlHourOffset = Database.sqlHourOffset();
+
+        for (let monitor of monitors) {
+            if (detailedLog) {
+                log.info("db", "Deleting non-important heartbeats for monitor " + monitor.id);
+            }
+            await R.exec(`
+                DELETE FROM heartbeat
+                WHERE monitor_id = ?
+                AND important = 0
+                AND time < ${sqlHourOffset}
+                AND id NOT IN (
+                    SELECT id
+                    FROM heartbeat
+                    WHERE monitor_id = ?
+                    ORDER BY time DESC
+                    LIMIT ?
+                )
+            `, [
+                monitor.id,
+                -24,
+                monitor.id,
+                100,
+            ]);
+        }
+    }
+
 }
 
 module.exports = Database;
diff --git a/server/jobs/clear-old-data.js b/server/jobs/clear-old-data.js
index fcdeac187..df5a943a8 100644
--- a/server/jobs/clear-old-data.js
+++ b/server/jobs/clear-old-data.js
@@ -1,35 +1,19 @@
 const { R } = require("redbean-node");
 const { log } = require("../../src/util");
-const { setSetting, setting } = require("../util-server");
+const { setSetting } = require("../util-server");
 const Database = require("../database");
 const { Settings } = require("../settings");
+const dayjs = require("dayjs");
 
-const DEFAULT_KEEP_PERIOD = 180;
+const DEFAULT_KEEP_PERIOD = 365;
 
 /**
- * Clears old data from the heartbeat table of the database.
+ * Clears old data from the heartbeat table and the stat_daily of the database.
  * @returns {Promise<void>} A promise that resolves when the data has been cleared.
  */
-
 const clearOldData = async () => {
-
-    // TODO: Temporary disable for testing
-    return;
-
-    /*
-    * TODO:
-    * Since we have aggregated table now, we don't need so much data in heartbeat table.
-    * But we still need to keep the important rows, because they contain the message.
-    *
-    * In the heartbeat table:
-    *    - important rows: keep according to the setting (keepDataPeriodDays) (default 180 days)
-    *    - not important rows: keep 2 days
-    *
-    * stat_* tables:
-    *   - keep according to the setting (keepDataPeriodDays) (default 180 days)
-    */
-
-    let period = await setting("keepDataPeriodDays");
+    await Database.clearHeartbeatData();
+    let period = await Settings.get("keepDataPeriodDays");
 
     // Set Default Period
     if (period == null) {
@@ -50,16 +34,21 @@ const clearOldData = async () => {
     if (parsedPeriod < 1) {
         log.info("clearOldData", `Data deletion has been disabled as period is less than 1. Period is ${parsedPeriod} days.`);
     } else {
-
         log.debug("clearOldData", `Clearing Data older than ${parsedPeriod} days...`);
-
         const sqlHourOffset = Database.sqlHourOffset();
 
         try {
-            await R.exec(
-                "DELETE FROM heartbeat WHERE time < " + sqlHourOffset,
-                [ parsedPeriod * -24 ]
-            );
+            // Heartbeat
+            await R.exec("DELETE FROM heartbeat WHERE time < " + sqlHourOffset, [
+                parsedPeriod * -24,
+            ]);
+
+            let timestamp = dayjs().subtract(parsedPeriod, "day").utc().startOf("day").unix();
+
+            // stat_daily
+            await R.exec("DELETE FROM stat_daily WHERE timestamp < ? ", [
+                timestamp,
+            ]);
 
             if (Database.dbConfig.type === "sqlite") {
                 await R.exec("PRAGMA optimize;");
@@ -68,6 +57,8 @@ const clearOldData = async () => {
             log.error("clearOldData", `Failed to clear old data: ${e.message}`);
         }
     }
+
+    log.debug("clearOldData", "Data cleared.");
 };
 
 module.exports = {
diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js
index 52a34fc29..71d1d458c 100644
--- a/server/uptime-calculator.js
+++ b/server/uptime-calculator.js
@@ -209,7 +209,7 @@ class UptimeCalculator {
         let flatStatus = this.flatStatus(status);
 
         if (flatStatus === DOWN && ping > 0) {
-            log.warn("uptime-calc", "The ping is not effective when the status is DOWN");
+            log.debug("uptime-calc", "The ping is not effective when the status is DOWN");
         }
 
         let divisionKey = this.getMinutelyKey(date);