michael/merlyn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove race condition regression for FoundryLocal provider

Browse Source
This commit is contained in:
Timothy Carambat 2026-01-16 16:45:05 -08:00
parent 607b5faf74
commit 9191179c9e
1 changed files with 21 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
server/utils/AiProviders/foundry/index.js
View File

@ -25,17 +25,9 @@ class FoundryLLM {
this.embedder = embedder ?? new NativeEmbedder();
this.defaultTemp = 0.7;
FoundryLLM.cacheContextWindows(true).then(() => {
this.limits = {
history: this.promptWindowLimit() * 0.15,
system: this.promptWindowLimit() * 0.15,
user: this.promptWindowLimit() * 0.7,
};
this.#log(
`Loaded with model: ${this.model} with context window: ${this.promptWindowLimit()}`
);
});
this.limits = null;
FoundryLLM.cacheContextWindows(true);
this.#log(`Loaded with model: ${this.model}`);
}
static #slog(text, ...args) {
@ -46,6 +38,16 @@ class FoundryLLM {
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
}
async assertModelContextLimits() {
if (this.limits !== null) return;
await FoundryLLM.cacheContextWindows();
this.limits = {
history: this.promptWindowLimit() * 0.15,
system: this.promptWindowLimit() * 0.15,
user: this.promptWindowLimit() * 0.7,
};
}
#appendContext(contextTexts = []) {
if (!contextTexts || !contextTexts.length) return "";
return (
@ -116,6 +118,13 @@ class FoundryLLM {
}
static promptWindowLimit(modelName) {
if (Object.keys(FoundryLLM.modelContextWindows).length === 0) {
this.#slog(
"No context windows cached - Context window may be inaccurately reported."
);
return process.env.FOUNDRY_MODEL_TOKEN_LIMIT || 4096;
}
let userDefinedLimit = null;
const systemDefinedLimit =
Number(this.modelContextWindows[modelName]) || 4096;
@ -264,6 +273,7 @@ class FoundryLLM {
}
async compressMessages(promptArgs = {}, rawHistory = []) {
await this.assertModelContextLimits();
const { messageArrayCompressor } = require("../../helpers/chat");
const messageArray = this.constructPrompt(promptArgs);
return await messageArrayCompressor(this, messageArray, rawHistory);