* Create parse endpoint in collector (#4212) * create parse endpoint in collector * revert cleanup temp util call * lint * remove unused cleanupTempDocuments function * revert slug change minor change for destinations --------- Co-authored-by: timothycarambat <rambat1010@gmail.com> * Add parsed files table and parse server endpoints (#4222) * add workspace_parsed_files table + parse endpoints/models * remove dev api parse endpoint * remove unneeded imports * iterate over all files + remove unneeded update function + update telemetry debounce * Upload UI/UX context window check + frontend alert (#4230) * prompt user to embed if exceeds prompt window + handle embed + handle cancel * add tokenCountEstimate to workspace_parsed_files + optimizations * use util for path locations + use safeJsonParse * add modal for user decision on overflow of context window * lint * dynamic fetching of provider/model combo + inject parsed documents * remove unneeded comments * popup ui for attaching/removing files + warning to embed + wip fetching states on update * remove prop drilling, fetch files/limits directly in attach files popup * rework ux of FE + BE optimizations * fix ux of FE + BE optimizations * Implement bidirectional sync for parsed file states linting small changes and comments * move parse support to another endpoint file simplify calls and loading of records * button borders * enable default users to upload parsed files but NOT embed * delete cascade on user/workspace/thread deletion to remove parsedFileRecord * enable bgworker with "always" jobs and optional document sync jobs orphan document job: Will find any broken reference files to prevent overpollution of the storage folder. This will run 10s after boot and every 12hr after * change run timeout for orphan job to 1m to allow settling before spawning a worker * linting and cleanup pr --------- Co-authored-by: Timothy Carambat <rambat1010@gmail.com> * dev build * fix tooltip hiding during embedding overflow files * prevent crash log from ERRNO on parse files * unused import * update docs link * Migrate parsed-files to GET endpoint patch logic for grabbing models names from utils better handling for undetermined context windows (null instead of Pos_INIFI) UI placeholder for null context windows * patch URL --------- Co-authored-by: Sean Hatfield <seanhatfield5@gmail.com>
97 lines
2.4 KiB
JavaScript
97 lines
2.4 KiB
JavaScript
const path = require("path");
|
|
const Graceful = require("@ladjs/graceful");
|
|
const Bree = require("@mintplex-labs/bree");
|
|
const setLogger = require("../logger");
|
|
|
|
class BackgroundService {
|
|
name = "BackgroundWorkerService";
|
|
static _instance = null;
|
|
documentSyncEnabled = false;
|
|
#root = path.resolve(__dirname, "../../jobs");
|
|
|
|
#alwaysRunJobs = [
|
|
{
|
|
name: "cleanup-orphan-documents",
|
|
timeout: "1m",
|
|
interval: "12hr",
|
|
},
|
|
];
|
|
|
|
#documentSyncJobs = [
|
|
// Job for auto-sync of documents
|
|
// https://github.com/breejs/bree
|
|
{
|
|
name: "sync-watched-documents",
|
|
interval: "1hr",
|
|
},
|
|
];
|
|
|
|
constructor() {
|
|
if (BackgroundService._instance) {
|
|
this.#log("SINGLETON LOCK: Using existing BackgroundService.");
|
|
return BackgroundService._instance;
|
|
}
|
|
|
|
this.logger = setLogger();
|
|
BackgroundService._instance = this;
|
|
}
|
|
|
|
#log(text, ...args) {
|
|
console.log(`\x1b[36m[${this.name}]\x1b[0m ${text}`, ...args);
|
|
}
|
|
|
|
async boot() {
|
|
const { DocumentSyncQueue } = require("../../models/documentSyncQueue");
|
|
this.documentSyncEnabled = await DocumentSyncQueue.enabled();
|
|
const jobsToRun = this.jobs();
|
|
|
|
this.#log("Starting...");
|
|
this.bree = new Bree({
|
|
logger: this.logger,
|
|
root: this.#root,
|
|
jobs: jobsToRun,
|
|
errorHandler: this.onError,
|
|
workerMessageHandler: this.onWorkerMessageHandler,
|
|
runJobsAs: "process",
|
|
});
|
|
this.graceful = new Graceful({ brees: [this.bree], logger: this.logger });
|
|
this.graceful.listen();
|
|
this.bree.start();
|
|
this.#log(
|
|
`Service started with ${jobsToRun.length} jobs`,
|
|
jobsToRun.map((j) => j.name)
|
|
);
|
|
}
|
|
|
|
async stop() {
|
|
this.#log("Stopping...");
|
|
if (!!this.graceful && !!this.bree) this.graceful.stopBree(this.bree, 0);
|
|
this.bree = null;
|
|
this.graceful = null;
|
|
this.#log("Service stopped");
|
|
}
|
|
|
|
/** @returns {import("@mintplex-labs/bree").Job[]} */
|
|
jobs() {
|
|
const activeJobs = [...this.#alwaysRunJobs];
|
|
if (this.documentSyncEnabled) activeJobs.push(...this.#documentSyncJobs);
|
|
return activeJobs;
|
|
}
|
|
|
|
onError(error, _workerMetadata) {
|
|
this.logger.error(`${error.message}`, {
|
|
service: "bg-worker",
|
|
origin: error.name,
|
|
});
|
|
}
|
|
|
|
onWorkerMessageHandler(message, _workerMetadata) {
|
|
this.logger.info(`${message.message}`, {
|
|
service: "bg-worker",
|
|
origin: message.name,
|
|
});
|
|
}
|
|
}
|
|
|
|
module.exports.BackgroundService = BackgroundService;
|