michael/merlyn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update the timeout value on all stream-timeout providers: (#4412)

Browse Source 
- OpenRouter
- Novita
- CometAPI
updated to 3,000ms default with 500ms min
This commit is contained in:
Timothy Carambat 2025-09-19 08:52:20 -07:00 committed by GitHub
parent 1209606d9a
commit 9466f67162
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 35 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
frontend/src/components/LLMSelection/CometApiLLMOptions/index.jsx
View File

@ -59,7 +59,7 @@ function AdvancedControls({ settings }) {
name="CometApiLLMTimeout"
className="border-none bg-theme-settings-input-bg text-theme-text-primary placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="Timeout value between token responses to auto-timeout the stream"
defaultValue={settings?.CometApiLLMTimeout ?? 500}
defaultValue={settings?.CometApiLLMTimeout ?? 3_000}
autoComplete="off"
onScroll={(e) => e.target.blur()}
min={500}

2
frontend/src/components/LLMSelection/NovitaLLMOptions/index.jsx
View File

@ -59,7 +59,7 @@ function AdvancedControls({ settings }) {
name="NovitaLLMTimeout"
className="border-none bg-theme-settings-input-bg text-theme-text-primary placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="Timeout value between token responses to auto-timeout the stream"
defaultValue={settings?.NovitaLLMTimeout ?? 500}
defaultValue={settings?.NovitaLLMTimeout ?? 3_000}
autoComplete="off"
onScroll={(e) => e.target.blur()}
min={500}

2
frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx
View File

@ -57,7 +57,7 @@ function AdvancedControls({ settings }) {
name="OpenRouterTimeout"
className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="Timeout value between token responses to auto-timeout the stream"
defaultValue={settings?.OpenRouterTimeout ?? 500}
defaultValue={settings?.OpenRouterTimeout ?? 3_000}
autoComplete="off"
onScroll={(e) => e.target.blur()}
min={500}

9
server/utils/AiProviders/cometapi/index.js
View File

@ -19,6 +19,7 @@ const cacheFolder = path.resolve(
);
class CometApiLLM {
defaultTimeout = 3_000;
constructor(embedder = null, modelPreference = null) {
if (!process.env.COMETAPI_LLM_API_KEY)
throw new Error("No CometAPI API key was set.");
@ -61,10 +62,14 @@ class CometApiLLM {
* CometAPI has various models that never return `finish_reasons` and thus leave the stream open
* which causes issues in subsequent messages. This timeout value forces us to close the stream after
* x milliseconds. This is a configurable value via the COMETAPI_LLM_TIMEOUT_MS value
* @returns {number} The timeout value in milliseconds (default: 500)
* @returns {number} The timeout value in milliseconds (default: 3_000)
*/
#parseTimeout() {
if (isNaN(Number(process.env.COMETAPI_LLM_TIMEOUT_MS))) return 500;
this.log(
`CometAPI timeout is set to ${process.env.COMETAPI_LLM_TIMEOUT_MS ?? this.defaultTimeout}ms`
);
if (isNaN(Number(process.env.COMETAPI_LLM_TIMEOUT_MS)))
return this.defaultTimeout;
const setValue = Number(process.env.COMETAPI_LLM_TIMEOUT_MS);
if (setValue < 500) return 500;
return setValue;

14
server/utils/AiProviders/novita/index.js
View File

@ -18,6 +18,8 @@ const cacheFolder = path.resolve(
);
class NovitaLLM {
defaultTimeout = 3_000;
constructor(embedder = null, modelPreference = null) {
if (!process.env.NOVITA_LLM_API_KEY)
throw new Error("No Novita API key was set.");
@ -62,12 +64,16 @@ class NovitaLLM {
* Novita has various models that never return `finish_reasons` and thus leave the stream open
* which causes issues in subsequent messages. This timeout value forces us to close the stream after
* x milliseconds. This is a configurable value via the NOVITA_LLM_TIMEOUT_MS value
* @returns {number} The timeout value in milliseconds (default: 500)
* @returns {number} The timeout value in milliseconds (default: 3_000)
*/
#parseTimeout() {
if (isNaN(Number(process.env.NOVITA_LLM_TIMEOUT_MS))) return 500;
this.log(
`Novita timeout is set to ${process.env.NOVITA_LLM_TIMEOUT_MS ?? this.defaultTimeout}ms`
);
if (isNaN(Number(process.env.NOVITA_LLM_TIMEOUT_MS)))
return this.defaultTimeout;
const setValue = Number(process.env.NOVITA_LLM_TIMEOUT_MS);
if (setValue < 500) return 500;
if (setValue < 500) return 500; // 500ms is the minimum timeout
return setValue;
}
@ -318,7 +324,7 @@ class NovitaLLM {
});
}
if (message.finish_reason !== null) {
if (message?.finish_reason !== null) {
writeResponseChunk(response, {
uuid,
sources,

19
server/utils/AiProviders/openRouter/index.js
View File

@ -18,6 +18,16 @@ const cacheFolder = path.resolve(
);
class OpenRouterLLM {
/**
* Some openrouter models never send a finish_reason and thus leave the stream open in the UI.
* However, because OR is a middleware it can also wait an inordinately long time between chunks so we need
* to ensure that we dont accidentally close the stream too early. If the time between chunks is greater than this timeout
* we will close the stream and assume it to be complete. This is common for free models or slow providers they can
* possibly delegate to during invocation.
* @type {number}
*/
defaultTimeout = 3_000;
constructor(embedder = null, modelPreference = null) {
if (!process.env.OPENROUTER_API_KEY)
throw new Error("No OpenRouter API key was set.");
@ -85,15 +95,16 @@ class OpenRouterLLM {
* OpenRouter has various models that never return `finish_reasons` and thus leave the stream open
* which causes issues in subsequent messages. This timeout value forces us to close the stream after
* x milliseconds. This is a configurable value via the OPENROUTER_TIMEOUT_MS value
* @returns {number} The timeout value in milliseconds (default: 500)
* @returns {number} The timeout value in milliseconds (default: 3_000)
*/
#parseTimeout() {
this.log(
`OpenRouter timeout is set to ${process.env.OPENROUTER_TIMEOUT_MS ?? 500}ms`
`OpenRouter timeout is set to ${process.env.OPENROUTER_TIMEOUT_MS ?? this.defaultTimeout}ms`
);
if (isNaN(Number(process.env.OPENROUTER_TIMEOUT_MS))) return 500;
if (isNaN(Number(process.env.OPENROUTER_TIMEOUT_MS)))
return this.defaultTimeout;
const setValue = Number(process.env.OPENROUTER_TIMEOUT_MS);
if (setValue < 500) return 500;
if (setValue < 500) return 500; // 500ms is the minimum timeout
return setValue;
}