mirror of
https://github.com/louislam/uptime-kuma.git
synced 2024-10-29 18:50:39 +00:00
Compare commits
5 commits
9c1cdc430e
...
88b1fd3868
Author | SHA1 | Date | |
---|---|---|---|
|
88b1fd3868 | ||
|
459fb138f2 | ||
|
8f950a5145 | ||
|
4d779cfc69 | ||
|
2470451f6d |
9 changed files with 2197 additions and 2400 deletions
|
@ -1,3 +1,13 @@
|
||||||
|
# Download Apprise deb package
|
||||||
|
FROM node:20-bookworm-slim AS download-apprise
|
||||||
|
WORKDIR /app
|
||||||
|
COPY ./extra/download-apprise.mjs ./download-apprise.mjs
|
||||||
|
RUN apt update && \
|
||||||
|
apt --yes --no-install-recommends install curl && \
|
||||||
|
npm install cheerio semver && \
|
||||||
|
node ./download-apprise.mjs
|
||||||
|
|
||||||
|
# Base Image (Slim)
|
||||||
# If the image changed, the second stage image should be changed too
|
# If the image changed, the second stage image should be changed too
|
||||||
FROM node:20-bookworm-slim AS base2-slim
|
FROM node:20-bookworm-slim AS base2-slim
|
||||||
ARG TARGETPLATFORM
|
ARG TARGETPLATFORM
|
||||||
|
@ -27,8 +37,9 @@ RUN apt update && \
|
||||||
# apprise = for notifications (Install from the deb package, as the stable one is too old) (workaround for #4867)
|
# apprise = for notifications (Install from the deb package, as the stable one is too old) (workaround for #4867)
|
||||||
# Switching to testing repo is no longer working, as the testing repo is not bookworm anymore.
|
# Switching to testing repo is no longer working, as the testing repo is not bookworm anymore.
|
||||||
# python3-paho-mqtt (#4859)
|
# python3-paho-mqtt (#4859)
|
||||||
RUN curl http://ftp.debian.org/debian/pool/main/a/apprise/apprise_1.8.0-2_all.deb --output apprise.deb && \
|
# TODO: no idea how to delete the deb file after installation as it becomes a layer already
|
||||||
apt update && \
|
COPY --from=download-apprise /app/apprise.deb ./apprise.deb
|
||||||
|
RUN apt update && \
|
||||||
apt --yes --no-install-recommends install ./apprise.deb python3-paho-mqtt && \
|
apt --yes --no-install-recommends install ./apprise.deb python3-paho-mqtt && \
|
||||||
rm -rf /var/lib/apt/lists/* && \
|
rm -rf /var/lib/apt/lists/* && \
|
||||||
rm -f apprise.deb && \
|
rm -f apprise.deb && \
|
||||||
|
|
57
extra/download-apprise.mjs
Normal file
57
extra/download-apprise.mjs
Normal file
|
@ -0,0 +1,57 @@
|
||||||
|
// Go to http://ftp.debian.org/debian/pool/main/a/apprise/ using fetch api, where it is a apache directory listing page
|
||||||
|
// Use cheerio to parse the html and get the latest version of Apprise
|
||||||
|
// call curl to download the latest version of Apprise
|
||||||
|
// Target file: the latest version of Apprise, which the format is apprise_{VERSION}_all.deb
|
||||||
|
|
||||||
|
import * as cheerio from "cheerio";
|
||||||
|
import semver from "semver";
|
||||||
|
import * as childProcess from "child_process";
|
||||||
|
|
||||||
|
const baseURL = "http://ftp.debian.org/debian/pool/main/a/apprise/";
|
||||||
|
const response = await fetch(baseURL);
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error("Failed to fetch page of Apprise Debian repository.");
|
||||||
|
}
|
||||||
|
|
||||||
|
const html = await response.text();
|
||||||
|
|
||||||
|
const $ = cheerio.load(html);
|
||||||
|
|
||||||
|
// Get all the links in the page
|
||||||
|
const linkElements = $("a");
|
||||||
|
|
||||||
|
// Filter the links which match apprise_{VERSION}_all.deb
|
||||||
|
const links = [];
|
||||||
|
const pattern = /apprise_(.*?)_all.deb/;
|
||||||
|
|
||||||
|
for (let i = 0; i < linkElements.length; i++) {
|
||||||
|
const link = linkElements[i];
|
||||||
|
if (link.attribs.href.match(pattern) && !link.attribs.href.includes("~")) {
|
||||||
|
links.push({
|
||||||
|
filename: link.attribs.href,
|
||||||
|
version: link.attribs.href.match(pattern)[1],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(links);
|
||||||
|
|
||||||
|
// semver compare and download
|
||||||
|
let latestLink = {
|
||||||
|
filename: "",
|
||||||
|
version: "0.0.0",
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const link of links) {
|
||||||
|
if (semver.gt(link.version, latestLink.version)) {
|
||||||
|
latestLink = link;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const downloadURL = baseURL + latestLink.filename;
|
||||||
|
console.log(`Downloading ${downloadURL}...`);
|
||||||
|
let result = childProcess.spawnSync("curl", [ downloadURL, "--output", "apprise.deb" ]);
|
||||||
|
console.log(result.stdout?.toString());
|
||||||
|
console.error(result.stderr?.toString());
|
||||||
|
process.exit(result.status !== null ? result.status : 1);
|
24
extra/reset-migrate-aggregate-table-state.js
Normal file
24
extra/reset-migrate-aggregate-table-state.js
Normal file
|
@ -0,0 +1,24 @@
|
||||||
|
const { R } = require("redbean-node");
|
||||||
|
const Database = require("../server/database");
|
||||||
|
const args = require("args-parser")(process.argv);
|
||||||
|
const { Settings } = require("../server/settings");
|
||||||
|
|
||||||
|
const main = async () => {
|
||||||
|
console.log("Connecting the database");
|
||||||
|
Database.initDataDir(args);
|
||||||
|
await Database.connect(false, false, true);
|
||||||
|
|
||||||
|
console.log("Deleting all data from aggregate tables");
|
||||||
|
await R.exec("DELETE FROM stat_minutely");
|
||||||
|
await R.exec("DELETE FROM stat_hourly");
|
||||||
|
await R.exec("DELETE FROM stat_daily");
|
||||||
|
|
||||||
|
console.log("Resetting the aggregate table state");
|
||||||
|
await Settings.set("migrateAggregateTableState", "");
|
||||||
|
|
||||||
|
await Database.close();
|
||||||
|
console.log("Done");
|
||||||
|
};
|
||||||
|
|
||||||
|
main();
|
||||||
|
|
4172
package-lock.json
generated
4172
package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
@ -38,8 +38,8 @@
|
||||||
"build-docker-base": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base2 --target base2 . --push",
|
"build-docker-base": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base2 --target base2 . --push",
|
||||||
"build-docker-base-slim": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base2-slim --target base2-slim . --push",
|
"build-docker-base-slim": "docker buildx build -f docker/debian-base.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:base2-slim --target base2-slim . --push",
|
||||||
"build-docker-builder-go": "docker buildx build -f docker/builder-go.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:builder-go . --push",
|
"build-docker-builder-go": "docker buildx build -f docker/builder-go.dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:builder-go . --push",
|
||||||
"build-docker-slim": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2-slim -t louislam/uptime-kuma:$VERSION-slim --target release --build-arg BASE_IMAGE=louislam/uptime-kuma:base2-slim . --push",
|
"build-docker-slim": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:next-slim -t louislam/uptime-kuma:2-slim -t louislam/uptime-kuma:$VERSION-slim --target release --build-arg BASE_IMAGE=louislam/uptime-kuma:base2-slim . --push",
|
||||||
"build-docker-full": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2 -t louislam/uptime-kuma:$VERSION --target release . --push",
|
"build-docker-full": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:next -t louislam/uptime-kuma:2 -t louislam/uptime-kuma:$VERSION --target release . --push",
|
||||||
"build-docker-nightly": "node ./extra/test-docker.js && npm run build && docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly2 --target nightly . --push",
|
"build-docker-nightly": "node ./extra/test-docker.js && npm run build && docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:nightly2 --target nightly . --push",
|
||||||
"build-docker-slim-rootless": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2-slim-rootless -t louislam/uptime-kuma:$VERSION-slim-rootless --target rootless --build-arg BASE_IMAGE=louislam/uptime-kuma:base2-slim . --push",
|
"build-docker-slim-rootless": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2-slim-rootless -t louislam/uptime-kuma:$VERSION-slim-rootless --target rootless --build-arg BASE_IMAGE=louislam/uptime-kuma:base2-slim . --push",
|
||||||
"build-docker-full-rootless": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2-rootless -t louislam/uptime-kuma:$VERSION-rootless --target rootless . --push",
|
"build-docker-full-rootless": "node ./extra/env2arg.js docker buildx build -f docker/dockerfile --platform linux/amd64,linux/arm64,linux/arm/v7 -t louislam/uptime-kuma:2-rootless -t louislam/uptime-kuma:$VERSION-rootless --target rootless . --push",
|
||||||
|
@ -68,7 +68,8 @@
|
||||||
"sort-contributors": "node extra/sort-contributors.js",
|
"sort-contributors": "node extra/sort-contributors.js",
|
||||||
"quick-run-nightly": "docker run --rm --env NODE_ENV=development -p 3001:3001 louislam/uptime-kuma:nightly2",
|
"quick-run-nightly": "docker run --rm --env NODE_ENV=development -p 3001:3001 louislam/uptime-kuma:nightly2",
|
||||||
"start-dev-container": "cd docker && docker-compose -f docker-compose-dev.yml up --force-recreate",
|
"start-dev-container": "cd docker && docker-compose -f docker-compose-dev.yml up --force-recreate",
|
||||||
"rebase-pr-to-1.23.X": "node extra/rebase-pr.js 1.23.X"
|
"rebase-pr-to-1.23.X": "node extra/rebase-pr.js 1.23.X",
|
||||||
|
"reset-migrate-aggregate-table-state": "node extra/reset-migrate-aggregate-table-state.js"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@grpc/grpc-js": "~1.8.22",
|
"@grpc/grpc-js": "~1.8.22",
|
||||||
|
|
|
@ -6,6 +6,9 @@ const knex = require("knex");
|
||||||
const path = require("path");
|
const path = require("path");
|
||||||
const { EmbeddedMariaDB } = require("./embedded-mariadb");
|
const { EmbeddedMariaDB } = require("./embedded-mariadb");
|
||||||
const mysql = require("mysql2/promise");
|
const mysql = require("mysql2/promise");
|
||||||
|
const { Settings } = require("./settings");
|
||||||
|
const { UptimeCalculator } = require("./uptime-calculator");
|
||||||
|
const dayjs = require("dayjs");
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Database & App Data Folder
|
* Database & App Data Folder
|
||||||
|
@ -391,9 +394,23 @@ class Database {
|
||||||
// https://knexjs.org/guide/migrations.html
|
// https://knexjs.org/guide/migrations.html
|
||||||
// https://gist.github.com/NigelEarle/70db130cc040cc2868555b29a0278261
|
// https://gist.github.com/NigelEarle/70db130cc040cc2868555b29a0278261
|
||||||
try {
|
try {
|
||||||
|
// Disable foreign key check for SQLite
|
||||||
|
// Known issue of knex: https://github.com/drizzle-team/drizzle-orm/issues/1813
|
||||||
|
if (Database.dbConfig.type === "sqlite") {
|
||||||
|
await R.exec("PRAGMA foreign_keys = OFF");
|
||||||
|
}
|
||||||
|
|
||||||
await R.knex.migrate.latest({
|
await R.knex.migrate.latest({
|
||||||
directory: Database.knexMigrationsPath,
|
directory: Database.knexMigrationsPath,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Enable foreign key check for SQLite
|
||||||
|
if (Database.dbConfig.type === "sqlite") {
|
||||||
|
await R.exec("PRAGMA foreign_keys = ON");
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.migrateAggregateTable();
|
||||||
|
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
// Allow missing patch files for downgrade or testing pr.
|
// Allow missing patch files for downgrade or testing pr.
|
||||||
if (e.message.includes("the following files are missing:")) {
|
if (e.message.includes("the following files are missing:")) {
|
||||||
|
@ -711,6 +728,152 @@ class Database {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migrate the old data in the heartbeat table to the new format (stat_daily, stat_hourly, stat_minutely)
|
||||||
|
* It should be run once while upgrading V1 to V2
|
||||||
|
*
|
||||||
|
* Normally, it should be in transaction, but UptimeCalculator wasn't designed to be in transaction before that.
|
||||||
|
* I don't want to heavily modify the UptimeCalculator, so it is not in transaction.
|
||||||
|
* Run `npm run reset-migrate-aggregate-table-state` to reset, in case the migration is interrupted.
|
||||||
|
* @returns {Promise<void>}
|
||||||
|
*/
|
||||||
|
static async migrateAggregateTable() {
|
||||||
|
log.debug("db", "Enter Migrate Aggregate Table function");
|
||||||
|
|
||||||
|
// Add a setting for 2.0.0-dev users to skip this migration
|
||||||
|
if (process.env.SET_MIGRATE_AGGREGATE_TABLE_TO_TRUE === "1") {
|
||||||
|
log.warn("db", "SET_MIGRATE_AGGREGATE_TABLE_TO_TRUE is set to 1, skipping aggregate table migration forever (for 2.0.0-dev users)");
|
||||||
|
await Settings.set("migrateAggregateTableState", "migrated");
|
||||||
|
}
|
||||||
|
|
||||||
|
let migrateState = await Settings.get("migrateAggregateTableState");
|
||||||
|
|
||||||
|
// Skip if already migrated
|
||||||
|
// If it is migrating, it possibly means the migration was interrupted, or the migration is in progress
|
||||||
|
if (migrateState === "migrated") {
|
||||||
|
log.debug("db", "Migrated aggregate table already, skip");
|
||||||
|
return;
|
||||||
|
} else if (migrateState === "migrating") {
|
||||||
|
log.warn("db", "Aggregate table migration is already in progress, or it was interrupted");
|
||||||
|
throw new Error("Aggregate table migration is already in progress");
|
||||||
|
}
|
||||||
|
|
||||||
|
await Settings.set("migrateAggregateTableState", "migrating");
|
||||||
|
|
||||||
|
log.info("db", "Migrating Aggregate Table");
|
||||||
|
|
||||||
|
log.info("db", "Getting list of unique monitors");
|
||||||
|
|
||||||
|
// Get a list of unique monitors from the heartbeat table, using raw sql
|
||||||
|
let monitors = await R.getAll(`
|
||||||
|
SELECT DISTINCT monitor_id
|
||||||
|
FROM heartbeat
|
||||||
|
ORDER BY monitor_id ASC
|
||||||
|
`);
|
||||||
|
|
||||||
|
// Stop if stat_* tables are not empty
|
||||||
|
for (let table of [ "stat_minutely", "stat_hourly", "stat_daily" ]) {
|
||||||
|
let countResult = await R.getRow(`SELECT COUNT(*) AS count FROM ${table}`);
|
||||||
|
let count = countResult.count;
|
||||||
|
if (count > 0) {
|
||||||
|
log.warn("db", `Aggregate table ${table} is not empty, migration will not be started (Maybe you were using 2.0.0-dev?)`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let progressPercent = 0;
|
||||||
|
let part = 100 / monitors.length;
|
||||||
|
let i = 1;
|
||||||
|
for (let monitor of monitors) {
|
||||||
|
// Get a list of unique dates from the heartbeat table, using raw sql
|
||||||
|
let dates = await R.getAll(`
|
||||||
|
SELECT DISTINCT DATE(time) AS date
|
||||||
|
FROM heartbeat
|
||||||
|
WHERE monitor_id = ?
|
||||||
|
ORDER BY date ASC
|
||||||
|
`, [
|
||||||
|
monitor.monitor_id
|
||||||
|
]);
|
||||||
|
|
||||||
|
for (let date of dates) {
|
||||||
|
// New Uptime Calculator
|
||||||
|
let calculator = new UptimeCalculator();
|
||||||
|
calculator.monitorID = monitor.monitor_id;
|
||||||
|
calculator.setMigrationMode(true);
|
||||||
|
|
||||||
|
// Get all the heartbeats for this monitor and date
|
||||||
|
let heartbeats = await R.getAll(`
|
||||||
|
SELECT status, ping, time
|
||||||
|
FROM heartbeat
|
||||||
|
WHERE monitor_id = ?
|
||||||
|
AND DATE(time) = ?
|
||||||
|
ORDER BY time ASC
|
||||||
|
`, [ monitor.monitor_id, date.date ]);
|
||||||
|
|
||||||
|
if (heartbeats.length > 0) {
|
||||||
|
log.info("db", `[DON'T STOP] Migrating monitor data ${monitor.monitor_id} - ${date.date} [${progressPercent.toFixed(2)}%][${i}/${monitors.length}]`);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let heartbeat of heartbeats) {
|
||||||
|
await calculator.update(heartbeat.status, parseFloat(heartbeat.ping), dayjs(heartbeat.time));
|
||||||
|
}
|
||||||
|
|
||||||
|
progressPercent += (Math.round(part / dates.length * 100) / 100);
|
||||||
|
|
||||||
|
// Lazy to fix the floating point issue, it is acceptable since it is just a progress bar
|
||||||
|
if (progressPercent > 100) {
|
||||||
|
progressPercent = 100;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
|
||||||
|
await Database.clearHeartbeatData(true);
|
||||||
|
|
||||||
|
await Settings.set("migrateAggregateTableState", "migrated");
|
||||||
|
|
||||||
|
if (monitors.length > 0) {
|
||||||
|
log.info("db", "Aggregate Table Migration Completed");
|
||||||
|
} else {
|
||||||
|
log.info("db", "No data to migrate");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remove all non-important heartbeats from heartbeat table, keep last 24-hour or {KEEP_LAST_ROWS} rows for each monitor
|
||||||
|
* @param {boolean} detailedLog Log detailed information
|
||||||
|
* @returns {Promise<void>}
|
||||||
|
*/
|
||||||
|
static async clearHeartbeatData(detailedLog = false) {
|
||||||
|
let monitors = await R.getAll("SELECT id FROM monitor");
|
||||||
|
const sqlHourOffset = Database.sqlHourOffset();
|
||||||
|
|
||||||
|
for (let monitor of monitors) {
|
||||||
|
if (detailedLog) {
|
||||||
|
log.info("db", "Deleting non-important heartbeats for monitor " + monitor.id);
|
||||||
|
}
|
||||||
|
await R.exec(`
|
||||||
|
DELETE FROM heartbeat
|
||||||
|
WHERE monitor_id = ?
|
||||||
|
AND important = 0
|
||||||
|
AND time < ${sqlHourOffset}
|
||||||
|
AND id NOT IN (
|
||||||
|
SELECT id
|
||||||
|
FROM heartbeat
|
||||||
|
WHERE monitor_id = ?
|
||||||
|
ORDER BY time DESC
|
||||||
|
LIMIT ?
|
||||||
|
)
|
||||||
|
`, [
|
||||||
|
monitor.id,
|
||||||
|
-24,
|
||||||
|
monitor.id,
|
||||||
|
100,
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = Database;
|
module.exports = Database;
|
||||||
|
|
|
@ -1,21 +1,22 @@
|
||||||
const { R } = require("redbean-node");
|
const { R } = require("redbean-node");
|
||||||
const { log } = require("../../src/util");
|
const { log } = require("../../src/util");
|
||||||
const { setSetting, setting } = require("../util-server");
|
|
||||||
const Database = require("../database");
|
const Database = require("../database");
|
||||||
|
const { Settings } = require("../settings");
|
||||||
|
const dayjs = require("dayjs");
|
||||||
|
|
||||||
const DEFAULT_KEEP_PERIOD = 180;
|
const DEFAULT_KEEP_PERIOD = 365;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Clears old data from the heartbeat table of the database.
|
* Clears old data from the heartbeat table and the stat_daily of the database.
|
||||||
* @returns {Promise<void>} A promise that resolves when the data has been cleared.
|
* @returns {Promise<void>} A promise that resolves when the data has been cleared.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
const clearOldData = async () => {
|
const clearOldData = async () => {
|
||||||
let period = await setting("keepDataPeriodDays");
|
await Database.clearHeartbeatData();
|
||||||
|
let period = await Settings.get("keepDataPeriodDays");
|
||||||
|
|
||||||
// Set Default Period
|
// Set Default Period
|
||||||
if (period == null) {
|
if (period == null) {
|
||||||
await setSetting("keepDataPeriodDays", DEFAULT_KEEP_PERIOD, "general");
|
await Settings.set("keepDataPeriodDays", DEFAULT_KEEP_PERIOD, "general");
|
||||||
period = DEFAULT_KEEP_PERIOD;
|
period = DEFAULT_KEEP_PERIOD;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -25,23 +26,28 @@ const clearOldData = async () => {
|
||||||
parsedPeriod = parseInt(period);
|
parsedPeriod = parseInt(period);
|
||||||
} catch (_) {
|
} catch (_) {
|
||||||
log.warn("clearOldData", "Failed to parse setting, resetting to default..");
|
log.warn("clearOldData", "Failed to parse setting, resetting to default..");
|
||||||
await setSetting("keepDataPeriodDays", DEFAULT_KEEP_PERIOD, "general");
|
await Settings.set("keepDataPeriodDays", DEFAULT_KEEP_PERIOD, "general");
|
||||||
parsedPeriod = DEFAULT_KEEP_PERIOD;
|
parsedPeriod = DEFAULT_KEEP_PERIOD;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (parsedPeriod < 1) {
|
if (parsedPeriod < 1) {
|
||||||
log.info("clearOldData", `Data deletion has been disabled as period is less than 1. Period is ${parsedPeriod} days.`);
|
log.info("clearOldData", `Data deletion has been disabled as period is less than 1. Period is ${parsedPeriod} days.`);
|
||||||
} else {
|
} else {
|
||||||
|
|
||||||
log.debug("clearOldData", `Clearing Data older than ${parsedPeriod} days...`);
|
log.debug("clearOldData", `Clearing Data older than ${parsedPeriod} days...`);
|
||||||
|
|
||||||
const sqlHourOffset = Database.sqlHourOffset();
|
const sqlHourOffset = Database.sqlHourOffset();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await R.exec(
|
// Heartbeat
|
||||||
"DELETE FROM heartbeat WHERE time < " + sqlHourOffset,
|
await R.exec("DELETE FROM heartbeat WHERE time < " + sqlHourOffset, [
|
||||||
[ parsedPeriod * -24 ]
|
parsedPeriod * -24,
|
||||||
);
|
]);
|
||||||
|
|
||||||
|
let timestamp = dayjs().subtract(parsedPeriod, "day").utc().startOf("day").unix();
|
||||||
|
|
||||||
|
// stat_daily
|
||||||
|
await R.exec("DELETE FROM stat_daily WHERE timestamp < ? ", [
|
||||||
|
timestamp,
|
||||||
|
]);
|
||||||
|
|
||||||
if (Database.dbConfig.type === "sqlite") {
|
if (Database.dbConfig.type === "sqlite") {
|
||||||
await R.exec("PRAGMA optimize;");
|
await R.exec("PRAGMA optimize;");
|
||||||
|
@ -50,6 +56,8 @@ const clearOldData = async () => {
|
||||||
log.error("clearOldData", `Failed to clear old data: ${e.message}`);
|
log.error("clearOldData", `Failed to clear old data: ${e.message}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.debug("clearOldData", "Data cleared.");
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
|
@ -1604,18 +1604,20 @@ let needSetup = false;
|
||||||
|
|
||||||
await server.start();
|
await server.start();
|
||||||
|
|
||||||
server.httpServer.listen(port, hostname, () => {
|
server.httpServer.listen(port, hostname, async () => {
|
||||||
if (hostname) {
|
if (hostname) {
|
||||||
log.info("server", `Listening on ${hostname}:${port}`);
|
log.info("server", `Listening on ${hostname}:${port}`);
|
||||||
} else {
|
} else {
|
||||||
log.info("server", `Listening on ${port}`);
|
log.info("server", `Listening on ${port}`);
|
||||||
}
|
}
|
||||||
startMonitors();
|
await startMonitors();
|
||||||
|
|
||||||
|
// Put this here. Start background jobs after the db and server is ready to prevent clear up during db migration.
|
||||||
|
await initBackgroundJobs();
|
||||||
|
|
||||||
checkVersion.startInterval();
|
checkVersion.startInterval();
|
||||||
});
|
});
|
||||||
|
|
||||||
await initBackgroundJobs();
|
|
||||||
|
|
||||||
// Start cloudflared at the end if configured
|
// Start cloudflared at the end if configured
|
||||||
await cloudflaredAutoStart(cloudflaredToken);
|
await cloudflaredAutoStart(cloudflaredToken);
|
||||||
|
|
||||||
|
@ -1809,7 +1811,11 @@ async function startMonitors() {
|
||||||
}
|
}
|
||||||
|
|
||||||
for (let monitor of list) {
|
for (let monitor of list) {
|
||||||
await monitor.start(io);
|
try {
|
||||||
|
await monitor.start(io);
|
||||||
|
} catch (e) {
|
||||||
|
log.error("monitor", e);
|
||||||
|
}
|
||||||
// Give some delays, so all monitors won't make request at the same moment when just start the server.
|
// Give some delays, so all monitors won't make request at the same moment when just start the server.
|
||||||
await sleep(getRandomInt(300, 1000));
|
await sleep(getRandomInt(300, 1000));
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,7 +12,6 @@ class UptimeCalculator {
|
||||||
* @private
|
* @private
|
||||||
* @type {{string:UptimeCalculator}}
|
* @type {{string:UptimeCalculator}}
|
||||||
*/
|
*/
|
||||||
|
|
||||||
static list = {};
|
static list = {};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -55,6 +54,15 @@ class UptimeCalculator {
|
||||||
lastHourlyStatBean = null;
|
lastHourlyStatBean = null;
|
||||||
lastMinutelyStatBean = null;
|
lastMinutelyStatBean = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For migration purposes.
|
||||||
|
* @type {boolean}
|
||||||
|
*/
|
||||||
|
migrationMode = false;
|
||||||
|
|
||||||
|
statMinutelyKeepHour = 24;
|
||||||
|
statHourlyKeepDay = 30;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the uptime calculator for a monitor
|
* Get the uptime calculator for a monitor
|
||||||
* Initializes and returns the monitor if it does not exist
|
* Initializes and returns the monitor if it does not exist
|
||||||
|
@ -189,16 +197,19 @@ class UptimeCalculator {
|
||||||
/**
|
/**
|
||||||
* @param {number} status status
|
* @param {number} status status
|
||||||
* @param {number} ping Ping
|
* @param {number} ping Ping
|
||||||
|
* @param {dayjs.Dayjs} date Date (Only for migration)
|
||||||
* @returns {dayjs.Dayjs} date
|
* @returns {dayjs.Dayjs} date
|
||||||
* @throws {Error} Invalid status
|
* @throws {Error} Invalid status
|
||||||
*/
|
*/
|
||||||
async update(status, ping = 0) {
|
async update(status, ping = 0, date) {
|
||||||
let date = this.getCurrentDate();
|
if (!date) {
|
||||||
|
date = this.getCurrentDate();
|
||||||
|
}
|
||||||
|
|
||||||
let flatStatus = this.flatStatus(status);
|
let flatStatus = this.flatStatus(status);
|
||||||
|
|
||||||
if (flatStatus === DOWN && ping > 0) {
|
if (flatStatus === DOWN && ping > 0) {
|
||||||
log.warn("uptime-calc", "The ping is not effective when the status is DOWN");
|
log.debug("uptime-calc", "The ping is not effective when the status is DOWN");
|
||||||
}
|
}
|
||||||
|
|
||||||
let divisionKey = this.getMinutelyKey(date);
|
let divisionKey = this.getMinutelyKey(date);
|
||||||
|
@ -297,47 +308,61 @@ class UptimeCalculator {
|
||||||
}
|
}
|
||||||
await R.store(dailyStatBean);
|
await R.store(dailyStatBean);
|
||||||
|
|
||||||
let hourlyStatBean = await this.getHourlyStatBean(hourlyKey);
|
let currentDate = this.getCurrentDate();
|
||||||
hourlyStatBean.up = hourlyData.up;
|
|
||||||
hourlyStatBean.down = hourlyData.down;
|
// For migration mode, we don't need to store old hourly and minutely data, but we need 30-day's hourly data
|
||||||
hourlyStatBean.ping = hourlyData.avgPing;
|
// Run anyway for non-migration mode
|
||||||
hourlyStatBean.pingMin = hourlyData.minPing;
|
if (!this.migrationMode || date.isAfter(currentDate.subtract(this.statHourlyKeepDay, "day"))) {
|
||||||
hourlyStatBean.pingMax = hourlyData.maxPing;
|
let hourlyStatBean = await this.getHourlyStatBean(hourlyKey);
|
||||||
{
|
hourlyStatBean.up = hourlyData.up;
|
||||||
// eslint-disable-next-line no-unused-vars
|
hourlyStatBean.down = hourlyData.down;
|
||||||
const { up, down, avgPing, minPing, maxPing, timestamp, ...extras } = hourlyData;
|
hourlyStatBean.ping = hourlyData.avgPing;
|
||||||
if (Object.keys(extras).length > 0) {
|
hourlyStatBean.pingMin = hourlyData.minPing;
|
||||||
hourlyStatBean.extras = JSON.stringify(extras);
|
hourlyStatBean.pingMax = hourlyData.maxPing;
|
||||||
|
{
|
||||||
|
// eslint-disable-next-line no-unused-vars
|
||||||
|
const { up, down, avgPing, minPing, maxPing, timestamp, ...extras } = hourlyData;
|
||||||
|
if (Object.keys(extras).length > 0) {
|
||||||
|
hourlyStatBean.extras = JSON.stringify(extras);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
await R.store(hourlyStatBean);
|
||||||
}
|
}
|
||||||
await R.store(hourlyStatBean);
|
|
||||||
|
|
||||||
let minutelyStatBean = await this.getMinutelyStatBean(divisionKey);
|
// For migration mode, we don't need to store old hourly and minutely data, but we need 24-hour's minutely data
|
||||||
minutelyStatBean.up = minutelyData.up;
|
// Run anyway for non-migration mode
|
||||||
minutelyStatBean.down = minutelyData.down;
|
if (!this.migrationMode || date.isAfter(currentDate.subtract(this.statMinutelyKeepHour, "hour"))) {
|
||||||
minutelyStatBean.ping = minutelyData.avgPing;
|
let minutelyStatBean = await this.getMinutelyStatBean(divisionKey);
|
||||||
minutelyStatBean.pingMin = minutelyData.minPing;
|
minutelyStatBean.up = minutelyData.up;
|
||||||
minutelyStatBean.pingMax = minutelyData.maxPing;
|
minutelyStatBean.down = minutelyData.down;
|
||||||
{
|
minutelyStatBean.ping = minutelyData.avgPing;
|
||||||
// eslint-disable-next-line no-unused-vars
|
minutelyStatBean.pingMin = minutelyData.minPing;
|
||||||
const { up, down, avgPing, minPing, maxPing, timestamp, ...extras } = minutelyData;
|
minutelyStatBean.pingMax = minutelyData.maxPing;
|
||||||
if (Object.keys(extras).length > 0) {
|
{
|
||||||
minutelyStatBean.extras = JSON.stringify(extras);
|
// eslint-disable-next-line no-unused-vars
|
||||||
|
const { up, down, avgPing, minPing, maxPing, timestamp, ...extras } = minutelyData;
|
||||||
|
if (Object.keys(extras).length > 0) {
|
||||||
|
minutelyStatBean.extras = JSON.stringify(extras);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
await R.store(minutelyStatBean);
|
||||||
}
|
}
|
||||||
await R.store(minutelyStatBean);
|
|
||||||
|
|
||||||
// Remove the old data
|
// No need to remove old data in migration mode
|
||||||
log.debug("uptime-calc", "Remove old data");
|
if (!this.migrationMode) {
|
||||||
await R.exec("DELETE FROM stat_minutely WHERE monitor_id = ? AND timestamp < ?", [
|
// Remove the old data
|
||||||
this.monitorID,
|
// TODO: Improvement: Convert it to a job?
|
||||||
this.getMinutelyKey(date.subtract(24, "hour")),
|
log.debug("uptime-calc", "Remove old data");
|
||||||
]);
|
await R.exec("DELETE FROM stat_minutely WHERE monitor_id = ? AND timestamp < ?", [
|
||||||
|
this.monitorID,
|
||||||
|
this.getMinutelyKey(currentDate.subtract(this.statMinutelyKeepHour, "hour")),
|
||||||
|
]);
|
||||||
|
|
||||||
await R.exec("DELETE FROM stat_hourly WHERE monitor_id = ? AND timestamp < ?", [
|
await R.exec("DELETE FROM stat_hourly WHERE monitor_id = ? AND timestamp < ?", [
|
||||||
this.monitorID,
|
this.monitorID,
|
||||||
this.getHourlyKey(date.subtract(30, "day")),
|
this.getHourlyKey(currentDate.subtract(this.statHourlyKeepDay, "day")),
|
||||||
]);
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
return date;
|
return date;
|
||||||
}
|
}
|
||||||
|
@ -812,6 +837,14 @@ class UptimeCalculator {
|
||||||
return dayjs.utc();
|
return dayjs.utc();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For migration purposes.
|
||||||
|
* @param {boolean} value Migration mode on/off
|
||||||
|
* @returns {void}
|
||||||
|
*/
|
||||||
|
setMigrationMode(value) {
|
||||||
|
this.migrationMode = value;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class UptimeDataResult {
|
class UptimeDataResult {
|
||||||
|
|
Loading…
Reference in a new issue