Compare commits
2 commits
3a179e0c69
...
6670b57674
| Author | SHA1 | Date | |
|---|---|---|---|
| 6670b57674 | |||
| 8872d127a9 |
6 changed files with 39 additions and 32 deletions
|
|
@ -25,7 +25,7 @@
|
||||||
# uncomment this and let the build fail, then get the current hash
|
# uncomment this and let the build fail, then get the current hash
|
||||||
# very scuffed but endorsed!
|
# very scuffed but endorsed!
|
||||||
# npmDepsHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
|
# npmDepsHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
|
||||||
npmDepsHash = "sha256-11AayHpPu7ocBPRB5k4SU7b99Aqc/dufAy2Yg5oPvGE=";
|
npmDepsHash = "sha256-7DsCZ+wDLvpqBch2rVbXiSxIaYdITX3RCHfjNx0wUKs=";
|
||||||
|
|
||||||
nativeBuildInputs = with pkgs; [ makeWrapper ];
|
nativeBuildInputs = with pkgs; [ makeWrapper ];
|
||||||
|
|
||||||
|
|
|
||||||
13
package-lock.json
generated
13
package-lock.json
generated
|
|
@ -14,7 +14,6 @@
|
||||||
"callsites": "^4.2.0",
|
"callsites": "^4.2.0",
|
||||||
"chalk": "^5.4.1",
|
"chalk": "^5.4.1",
|
||||||
"data-uri-to-buffer": "^6.0.2",
|
"data-uri-to-buffer": "^6.0.2",
|
||||||
"dotenv": "^17.2.1",
|
|
||||||
"drizzle-orm": "^0.44.4",
|
"drizzle-orm": "^0.44.4",
|
||||||
"express": "^5.1.0",
|
"express": "^5.1.0",
|
||||||
"express-handlebars": "^8.0.3",
|
"express-handlebars": "^8.0.3",
|
||||||
|
|
@ -2797,18 +2796,6 @@
|
||||||
"node": ">=6.0.0"
|
"node": ">=6.0.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/dotenv": {
|
|
||||||
"version": "17.2.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.1.tgz",
|
|
||||||
"integrity": "sha512-kQhDYKZecqnM0fCnzI5eIv5L4cAe/iRI+HqMbO/hbRdTAeXDG+M9FjipUxNfbARuEg4iHIbhnhs78BCHNbSxEQ==",
|
|
||||||
"license": "BSD-2-Clause",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=12"
|
|
||||||
},
|
|
||||||
"funding": {
|
|
||||||
"url": "https://dotenvx.com"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/drizzle-kit": {
|
"node_modules/drizzle-kit": {
|
||||||
"version": "0.31.4",
|
"version": "0.31.4",
|
||||||
"resolved": "https://registry.npmjs.org/drizzle-kit/-/drizzle-kit-0.31.4.tgz",
|
"resolved": "https://registry.npmjs.org/drizzle-kit/-/drizzle-kit-0.31.4.tgz",
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,6 @@
|
||||||
"callsites": "^4.2.0",
|
"callsites": "^4.2.0",
|
||||||
"chalk": "^5.4.1",
|
"chalk": "^5.4.1",
|
||||||
"data-uri-to-buffer": "^6.0.2",
|
"data-uri-to-buffer": "^6.0.2",
|
||||||
"dotenv": "^17.2.1",
|
|
||||||
"drizzle-orm": "^0.44.4",
|
"drizzle-orm": "^0.44.4",
|
||||||
"express": "^5.1.0",
|
"express": "^5.1.0",
|
||||||
"express-handlebars": "^8.0.3",
|
"express-handlebars": "^8.0.3",
|
||||||
|
|
|
||||||
37
src/cache.ts
37
src/cache.ts
|
|
@ -6,6 +6,7 @@ import fsPromises from "fs/promises";
|
||||||
import { and, eq } from "drizzle-orm";
|
import { and, eq } from "drizzle-orm";
|
||||||
import * as log from "./log.js";
|
import * as log from "./log.js";
|
||||||
import prettyBytes from "pretty-bytes";
|
import prettyBytes from "pretty-bytes";
|
||||||
|
import process from "node:process";
|
||||||
|
|
||||||
// try creating cache if it doesn't exist
|
// try creating cache if it doesn't exist
|
||||||
// a bit scuffed but that ok
|
// a bit scuffed but that ok
|
||||||
|
|
@ -26,16 +27,21 @@ try {
|
||||||
let entriesClearedBytes = 0;
|
let entriesClearedBytes = 0;
|
||||||
log.debug("cache cleanup and expiry timers starting");
|
log.debug("cache cleanup and expiry timers starting");
|
||||||
|
|
||||||
await Promise.all((await db.select().from(fileCacheTable)).map(async ({ name, expiry }) => {
|
const results = await Promise.all((await db.select().from(fileCacheTable)).map(async ({ name, expiry }) => {
|
||||||
if (expiry < Date.now()) {
|
if (expiry < Date.now()) {
|
||||||
entriesCleared++;
|
return await dropFile(name);
|
||||||
entriesClearedBytes += (await fsPromises.stat(path.join(config.downloader.cache.directory, name))).size;
|
|
||||||
await dropFile(name);
|
|
||||||
} else {
|
} else {
|
||||||
await scheduleDeletion(name, expiry);
|
await scheduleDeletion(name, expiry);
|
||||||
}
|
}
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
for (const result of results) {
|
||||||
|
if (result !== undefined) {
|
||||||
|
entriesCleared += 1;
|
||||||
|
entriesClearedBytes += result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
log.debug("cache cleanup complete!");
|
log.debug("cache cleanup complete!");
|
||||||
log.debug(`cleared ${entriesCleared} entr${entriesCleared === 1 ? "y" : "ies"}, freeing up ${prettyBytes(entriesClearedBytes)}!`);
|
log.debug(`cleared ${entriesCleared} entr${entriesCleared === 1 ? "y" : "ies"}, freeing up ${prettyBytes(entriesClearedBytes)}!`);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|
@ -56,19 +62,26 @@ async function scheduleDeletion(name: string, expiry: number): Promise<void> {
|
||||||
timers.set(name, timeout);
|
timers.set(name, timeout);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function dropFile(name: string): Promise<void> {
|
// TODO: add behavior toggle: should we keep it in the database on failure or not ??
|
||||||
|
// current behavior: delete from db first, then try deleting files
|
||||||
|
// this is good if manual cleanup was already done (we can then ignore ENOENT)
|
||||||
|
// bad if they change the permissions instead of manual removal
|
||||||
|
async function dropFile(name: string): Promise<number | undefined> {
|
||||||
|
try {
|
||||||
|
await db.delete(fileCacheTable).where(eq(fileCacheTable.name, name));
|
||||||
const size = (await fsPromises.stat(path.join(config.downloader.cache.directory, name))).size;
|
const size = (await fsPromises.stat(path.join(config.downloader.cache.directory, name))).size;
|
||||||
await fsPromises.unlink(path.join(config.downloader.cache.directory, name)).catch((err) => {
|
await fsPromises.unlink(path.join(config.downloader.cache.directory, name));
|
||||||
if (err.code !== "ENOENT") {
|
|
||||||
|
log.debug(`deleted file ${name} from cache, freeing up ${prettyBytes(size)}`);
|
||||||
|
|
||||||
|
return size;
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof Error && err.message.includes("ENOENT")) { return; }
|
||||||
|
|
||||||
log.error(`failed to delete cached file ${name} for whatever reason!`);
|
log.error(`failed to delete cached file ${name} for whatever reason!`);
|
||||||
log.error("manual removal may be necessary!");
|
log.error("manual removal may be necessary!");
|
||||||
log.error(err);
|
log.error(err);
|
||||||
}
|
}
|
||||||
});
|
|
||||||
|
|
||||||
log.debug(`deleted file ${name} from cache, freeing up ${prettyBytes(size)}`);
|
|
||||||
|
|
||||||
await db.delete(fileCacheTable).where(eq(fileCacheTable.name, name));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function addFileToCache(fileName: string): Promise<void> {
|
export async function addFileToCache(fileName: string): Promise<void> {
|
||||||
|
|
|
||||||
|
|
@ -2,10 +2,17 @@ import fs from "node:fs";
|
||||||
import * as log from "./log.js";
|
import * as log from "./log.js";
|
||||||
import toml from "toml";
|
import toml from "toml";
|
||||||
import { z, ZodError, ZodObject } from "zod";
|
import { z, ZodError, ZodObject } from "zod";
|
||||||
import * as dotenv from "dotenv";
|
|
||||||
import { fromZodError } from "zod-validation-error";
|
import { fromZodError } from "zod-validation-error";
|
||||||
|
import process from "node:process";
|
||||||
|
|
||||||
dotenv.config({ quiet: true });
|
try {
|
||||||
|
process.loadEnvFile("./.env");
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof Error && !err.message.includes("ENOENT")) {
|
||||||
|
log.error("error reading or parsing evv file!");
|
||||||
|
log.error(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const configSchema = z.object({
|
const configSchema = z.object({
|
||||||
server: z.object({
|
server: z.object({
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ import { createClient } from "@libsql/client";
|
||||||
import { config, env } from "../config.js";
|
import { config, env } from "../config.js";
|
||||||
import { drizzle } from "drizzle-orm/libsql";
|
import { drizzle } from "drizzle-orm/libsql";
|
||||||
import { migrate } from "drizzle-orm/libsql/migrator";
|
import { migrate } from "drizzle-orm/libsql/migrator";
|
||||||
|
import process from "node:process";
|
||||||
import fsPromises from "fs/promises";
|
import fsPromises from "fs/promises";
|
||||||
import * as log from "../log.js";
|
import * as log from "../log.js";
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue