michael/merlyn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update LMStudio LLM & Embedder for API token (#4948)

Browse Source 
- Updates Option panels to be consistent for other providers
adds API key to all LMStudio API calls
This commit is contained in:
Timothy Carambat 2026-01-30 11:13:32 -08:00 committed by GitHub
parent 0032c4da22
commit 97b140b4b4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 233 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docker/.env.example
View File

@ -33,6 +33,7 @@ GID='1000'
# LMSTUDIO_BASE_PATH='http://your-server:1234/v1'
# LMSTUDIO_MODEL_PREF='Loaded from Chat UI' # this is a bug in LMStudio 0.2.17
# LMSTUDIO_MODEL_TOKEN_LIMIT=4096
# LMSTUDIO_AUTH_TOKEN='your-lmstudio-auth-token-here'
# LLM_PROVIDER='localai'
# LOCAL_AI_BASE_PATH='http://host.docker.internal:8080/v1'

108
frontend/src/components/EmbeddingSelection/LMStudioOptions/index.jsx
View File

@ -1,8 +1,7 @@
import React, { useEffect, useState } from "react";
import System from "@/models/system";
import PreLoader from "@/components/Preloader";
import { LMSTUDIO_COMMON_URLS } from "@/utils/constants";
import { CaretDown, CaretUp, Info } from "@phosphor-icons/react";
import { CaretDown, CaretUp, Info, CircleNotch } from "@phosphor-icons/react";
import { Tooltip } from "react-tooltip";
import useProviderEndpointAutoDiscovery from "@/hooks/useProviderEndpointAutoDiscovery";
@ -11,6 +10,8 @@ export default function LMStudioEmbeddingOptions({ settings }) {
autoDetecting: loading,
basePath,
basePathValue,
authToken,
authTokenValue,
showAdvancedControls,
setShowAdvancedControls,
handleAutoDetectClick,
@ -31,20 +32,24 @@ export default function LMStudioEmbeddingOptions({ settings }) {
return (
<div className="w-full flex flex-col gap-y-7">
<div className="w-full flex items-start gap-[36px] mt-1.5">
<LMStudioModelSelection settings={settings} basePath={basePath.value} />
<LMStudioModelSelection
settings={settings}
basePath={basePath.value}
apiKey={authTokenValue.value}
/>
<div className="flex flex-col w-60">
<div
data-tooltip-place="top"
data-tooltip-id="max-embedding-chunk-length-tooltip"
className="flex gap-x-1 items-center mb-3"
>
<label className="text-white text-sm font-semibold block">
Max embedding chunk length
</label>
<Info
size={16}
className="text-theme-text-secondary cursor-pointer"
/>
<label className="text-white text-sm font-semibold block">
Max embedding chunk length
</label>
<Tooltip id="max-embedding-chunk-length-tooltip">
Maximum length of text chunks, in characters, for embedding.
</Tooltip>
@ -61,9 +66,6 @@ export default function LMStudioEmbeddingOptions({ settings }) {
required={true}
autoComplete="off"
/>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Maximum length of text chunks, in characters, for embedding.
</p>
</div>
</div>
<div className="flex justify-start mt-4">
@ -85,19 +87,41 @@ export default function LMStudioEmbeddingOptions({ settings }) {
<div hidden={!showAdvancedControls}>
<div className="w-full flex items-start gap-4">
<div className="flex flex-col w-60">
<div className="flex flex-col w-[300px]">
<div className="flex justify-between items-center mb-2">
<label className="text-white text-sm font-semibold">
LM Studio Base URL
</label>
<div className="flex items-center gap-1">
<label className="text-white text-sm font-semibold">
LM Studio Base URL
</label>
<Info
size={18}
className="text-theme-text-secondary cursor-pointer"
data-tooltip-id="lmstudio-base-url"
data-tooltip-content="Enter the URL where LM Studio is running."
/>
<Tooltip
id="lmstudio-base-url"
place="top"
delayShow={300}
className="tooltip !text-xs !opacity-100"
style={{
maxWidth: "250px",
whiteSpace: "normal",
wordWrap: "break-word",
}}
/>
</div>
{loading ? (
<PreLoader size="6" />
<CircleNotch
size={16}
className="text-theme-text-secondary animate-spin"
/>
) : (
<>
{!basePathValue.value && (
<button
onClick={handleAutoDetectClick}
className="bg-primary-button text-xs font-medium px-2 py-1 rounded-lg hover:bg-secondary hover:text-white shadow-[0_4px_14px_rgba(0,0,0,0.25)]"
className="border-none bg-primary-button text-xs font-medium px-2 py-1 rounded-lg hover:bg-secondary hover:text-white shadow-[0_4px_14px_rgba(0,0,0,0.25)]"
>
Auto-Detect
</button>
@ -117,9 +141,51 @@ export default function LMStudioEmbeddingOptions({ settings }) {
onChange={basePath.onChange}
onBlur={basePath.onBlur}
/>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Enter the URL where LM Studio is running.
</p>
</div>
<div className="flex flex-col w-60">
<div className="flex items-center mb-2 gap-x-1">
<label className="text-white text-sm font-semibold">
Authentication Token
</label>
<Info
size={18}
className="text-theme-text-secondary cursor-pointer"
data-tooltip-id="lmstudio-authentication-token"
/>
<Tooltip
id="lmstudio-authentication-token"
place="top"
delayShow={300}
delayHide={400}
clickable={true}
className="tooltip !text-xs !opacity-100"
style={{
maxWidth: "250px",
whiteSpace: "normal",
wordWrap: "break-word",
}}
>
<p className="text-xs leading-[18px] font-base">
Enter a <code>Bearer</code> Auth Token for interacting with
your LM Studio server.
<br /> <br />
Useful if running LM Studio behind an authentication or proxy.
</p>
</Tooltip>
</div>
<input
type="password"
name="LMStudioAuthToken"
className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg outline-none block w-full p-2.5 focus:outline-primary-button active:outline-primary-button"
placeholder="LM Studio Auth Token"
defaultValue={settings?.LMStudioAuthToken ? "*".repeat(20) : ""}
value={authTokenValue.value}
onChange={authToken.onChange}
onBlur={authToken.onBlur}
required={false}
autoComplete="off"
spellCheck={false}
/>
</div>
</div>
</div>
@ -127,7 +193,7 @@ export default function LMStudioEmbeddingOptions({ settings }) {
);
}
function LMStudioModelSelection({ settings, basePath = null }) {
function LMStudioModelSelection({ settings, basePath = null, apiKey = null }) {
const [customModels, setCustomModels] = useState([]);
const [loading, setLoading] = useState(true);
@ -142,7 +208,7 @@ function LMStudioModelSelection({ settings, basePath = null }) {
try {
const { models } = await System.customModels(
"lmstudio",
null,
apiKey,
basePath
);
setCustomModels(models || []);
@ -153,7 +219,7 @@ function LMStudioModelSelection({ settings, basePath = null }) {
setLoading(false);
}
findCustomModels();
}, [basePath]);
}, [basePath, apiKey]);
if (loading || customModels.length == 0) {
return (

137
frontend/src/components/LLMSelection/LMStudioOptions/index.jsx
View File

@ -1,16 +1,18 @@
import { useEffect, useState } from "react";
import { Info, CaretDown, CaretUp } from "@phosphor-icons/react";
import { Info, CaretDown, CaretUp, CircleNotch } from "@phosphor-icons/react";
import paths from "@/utils/paths";
import System from "@/models/system";
import PreLoader from "@/components/Preloader";
import { LMSTUDIO_COMMON_URLS } from "@/utils/constants";
import useProviderEndpointAutoDiscovery from "@/hooks/useProviderEndpointAutoDiscovery";
import { Tooltip } from "react-tooltip";
export default function LMStudioOptions({ settings, showAlert = false }) {
const {
autoDetecting: loading,
basePath,
basePathValue,
authToken,
authTokenValue,
showAdvancedControls,
setShowAdvancedControls,
handleAutoDetectClick,
@ -48,7 +50,11 @@ export default function LMStudioOptions({ settings, showAlert = false }) {
</div>
)}
<div className="w-full flex items-start gap-[36px] mt-1.5">
<LMStudioModelSelection settings={settings} basePath={basePath.value} />
<LMStudioModelSelection
settings={settings}
basePath={basePath.value}
apiKey={authTokenValue.value}
/>
</div>
<div className="flex justify-start mt-4">
<button
@ -69,19 +75,41 @@ export default function LMStudioOptions({ settings, showAlert = false }) {
<div hidden={!showAdvancedControls}>
<div className="w-full flex items-start gap-4">
<div className="flex flex-col w-60">
<div className="flex flex-col w-[300px]">
<div className="flex justify-between items-center mb-2">
<label className="text-white text-sm font-semibold">
LM Studio Base URL
</label>
<div className="flex items-center gap-1">
<label className="text-white text-sm font-semibold">
LM Studio Base URL
</label>
<Info
size={18}
className="text-theme-text-secondary cursor-pointer"
data-tooltip-id="lmstudio-base-url"
data-tooltip-content="Enter the URL where LM Studio is running."
/>
<Tooltip
id="lmstudio-base-url"
place="top"
delayShow={300}
className="tooltip !text-xs !opacity-100"
style={{
maxWidth: "250px",
whiteSpace: "normal",
wordWrap: "break-word",
}}
/>
</div>
{loading ? (
<PreLoader size="6" />
<CircleNotch
size={16}
className="text-theme-text-secondary animate-spin"
/>
) : (
<>
{!basePathValue.value && (
<button
onClick={handleAutoDetectClick}
className="bg-primary-button text-xs font-medium px-2 py-1 rounded-lg hover:bg-secondary hover:text-white shadow-[0_4px_14px_rgba(0,0,0,0.25)]"
className="border-none bg-primary-button text-xs font-medium px-2 py-1 rounded-lg hover:bg-secondary hover:text-white shadow-[0_4px_14px_rgba(0,0,0,0.25)]"
>
Auto-Detect
</button>
@ -101,14 +129,28 @@ export default function LMStudioOptions({ settings, showAlert = false }) {
onChange={basePath.onChange}
onBlur={basePath.onBlur}
/>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Enter the URL where LM Studio is running.
</p>
</div>
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-2">
Max Tokens (Optional)
</label>
<div className="flex items-center mb-2 gap-x-1">
<label className="text-white text-sm font-semibold">
Model Context Window
</label>
<Info
size={18}
className="text-theme-text-secondary cursor-pointer"
data-tooltip-id="lmstudio-max-tokens"
data-tooltip-content="Override the context window limit. Leave empty to auto-detect from the model (defaults to 4096 if detection fails)."
/>
<Tooltip
id="lmstudio-max-tokens"
className="tooltip !text-xs !opacity-100"
style={{
maxWidth: "250px",
whiteSpace: "normal",
wordWrap: "break-word",
}}
/>
</div>
<input
type="number"
name="LMStudioTokenLimit"
@ -121,10 +163,54 @@ export default function LMStudioOptions({ settings, showAlert = false }) {
required={false}
autoComplete="off"
/>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Override the context window limit. Leave empty to auto-detect from
the model (defaults to 4096 if detection fails).
</p>
</div>
</div>
<div className="flex items-start gap-4 mt-4">
<div className="flex flex-col w-60">
<div className="flex items-center mb-2 gap-x-1">
<label className="text-white text-sm font-semibold">
Authentication Token
</label>
<Info
size={18}
className="text-theme-text-secondary cursor-pointer"
data-tooltip-id="lmstudio-authentication-token"
/>
<Tooltip
id="lmstudio-authentication-token"
place="top"
delayShow={300}
delayHide={400}
clickable={true}
className="tooltip !text-xs !opacity-100"
style={{
maxWidth: "250px",
whiteSpace: "normal",
wordWrap: "break-word",
}}
>
<p className="text-xs leading-[18px] font-base">
Enter a <code>Bearer</code> Auth Token for interacting with
your LM Studio server.
<br /> <br />
Useful if running LM Studio behind an authentication or proxy.
</p>
</Tooltip>
</div>
<input
type="password"
name="LMStudioAuthToken"
className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg outline-none block w-full p-2.5 focus:outline-primary-button active:outline-primary-button"
placeholder="LM Studio Auth Token"
defaultValue={settings?.LMStudioAuthToken ? "*".repeat(20) : ""}
value={authTokenValue.value}
onChange={authToken.onChange}
onBlur={authToken.onBlur}
required={false}
autoComplete="off"
spellCheck={false}
/>
</div>
</div>
</div>
@ -132,7 +218,7 @@ export default function LMStudioOptions({ settings, showAlert = false }) {
);
}
function LMStudioModelSelection({ settings, basePath = null }) {
function LMStudioModelSelection({ settings, basePath = null, apiKey = null }) {
const [customModels, setCustomModels] = useState([]);
const [loading, setLoading] = useState(true);
@ -147,7 +233,7 @@ function LMStudioModelSelection({ settings, basePath = null }) {
try {
const { models } = await System.customModels(
"lmstudio",
null,
apiKey,
basePath
);
setCustomModels(models || []);
@ -158,7 +244,7 @@ function LMStudioModelSelection({ settings, basePath = null }) {
setLoading(false);
}
findCustomModels();
}, [basePath]);
}, [basePath, apiKey]);
if (loading || customModels.length === 0) {
return (
@ -177,10 +263,6 @@ function LMStudioModelSelection({ settings, basePath = null }) {
: "Enter LM Studio URL first"}
</option>
</select>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Select the LM Studio model you want to use. Models will load after
entering a valid LM Studio URL.
</p>
</div>
);
}
@ -211,9 +293,6 @@ function LMStudioModelSelection({ settings, basePath = null }) {
</optgroup>
)}
</select>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Choose the LM Studio model you want to use for your conversations.
</p>
</div>
);
}

1
server/.env.example
View File

@ -30,6 +30,7 @@ SIG_SALT='salt' # Please generate random string at least 32 chars long.
# LMSTUDIO_BASE_PATH='http://your-server:1234/v1'
# LMSTUDIO_MODEL_PREF='Loaded from Chat UI' # this is a bug in LMStudio 0.2.17
# LMSTUDIO_MODEL_TOKEN_LIMIT=4096
# LMSTUDIO_AUTH_TOKEN='your-lmstudio-auth-token-here'
# LLM_PROVIDER='localai'
# LOCAL_AI_BASE_PATH='http://localhost:8080/v1'

1
server/models/systemSettings.js
View File

@ -530,6 +530,7 @@ const SystemSettings = {
LMStudioBasePath: process.env.LMSTUDIO_BASE_PATH,
LMStudioTokenLimit: process.env.LMSTUDIO_MODEL_TOKEN_LIMIT || null,
LMStudioModelPref: process.env.LMSTUDIO_MODEL_PREF,
LMStudioAuthToken: !!process.env.LMSTUDIO_AUTH_TOKEN,
// LocalAI Keys
LocalAiApiKey: !!process.env.LOCAL_AI_API_KEY,

17
server/utils/AiProviders/lmStudio/index.js
View File

@ -17,9 +17,10 @@ class LMStudioLLM {
if (!process.env.LMSTUDIO_BASE_PATH)
throw new Error("No LMStudio API Base Path was set.");
const apiKey = process.env.LMSTUDIO_AUTH_TOKEN ?? null;
this.lmstudio = new OpenAIApi({
baseURL: parseLMStudioBasePath(process.env.LMSTUDIO_BASE_PATH), // here is the URL to your LMStudio instance
apiKey: null,
apiKey,
});
// Prior to LMStudio 0.2.17 the `model` param was not required and you could pass anything
@ -28,10 +29,8 @@ class LMStudioLLM {
// and any other value will crash inferencing. So until this is patched we will
// try to fetch the `/models` and have the user set it, or just fallback to "Loaded from Chat UI"
// which will not impact users with <v0.2.17 and should work as well once the bug is fixed.
this.model =
modelPreference ||
process.env.LMSTUDIO_MODEL_PREF ||
"Loaded from Chat UI";
this.model = modelPreference || process.env.LMSTUDIO_MODEL_PREF;
if (!this.model) throw new Error("LMStudio must have a valid model set.");
this.embedder = embedder ?? new NativeEmbedder();
this.defaultTemp = 0.7;
@ -76,11 +75,17 @@ class LMStudioLLM {
if (Object.keys(LMStudioLLM.modelContextWindows).length > 0 && !force)
return;
const apiKey = process.env.LMSTUDIO_AUTH_TOKEN ?? null;
const endpoint = new URL(
parseLMStudioBasePath(process.env.LMSTUDIO_BASE_PATH)
);
endpoint.pathname = "/api/v0/models";
await fetch(endpoint.toString())
await fetch(endpoint.toString(), {
headers: {
"Content-Type": "application/json",
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
},
})
.then((res) => {
if (!res.ok)
throw new Error(`LMStudio:cacheContextWindows - ${res.statusText}`);

3
server/utils/EmbeddingEngines/lmstudio/index.js
View File

@ -8,11 +8,12 @@ class LMStudioEmbedder {
if (!process.env.EMBEDDING_MODEL_PREF)
throw new Error("No embedding model was set.");
const apiKey = process.env.LMSTUDIO_AUTH_TOKEN ?? null;
this.className = "LMStudioEmbedder";
const { OpenAI: OpenAIApi } = require("openai");
this.lmstudio = new OpenAIApi({
baseURL: parseLMStudioBasePath(process.env.EMBEDDING_BASE_PATH),
apiKey: null,
apiKey,
});
this.model = process.env.EMBEDDING_MODEL_PREF;

6
server/utils/agents/aibitat/providers/ai-provider.js
View File

@ -273,14 +273,16 @@ class Provider {
// });
case "ollama":
return OllamaLangchainChatModel.create(config);
case "lmstudio":
case "lmstudio": {
const apiKey = process.env.LMSTUDIO_AUTH_TOKEN ?? null;
return new ChatOpenAI({
configuration: {
baseURL: parseLMStudioBasePath(process.env.LMSTUDIO_BASE_PATH),
},
apiKey: "not-used", // Needs to be specified or else will assume OpenAI
apiKey: apiKey || "not-used",
...config,
});
}
case "koboldcpp":
return new ChatOpenAI({
configuration: {

8
server/utils/agents/aibitat/providers/lmstudio.js
View File

@ -19,11 +19,13 @@ class LMStudioProvider extends InheritMultiple([Provider, UnTooled]) {
*/
constructor(config = {}) {
super();
const model =
config?.model || process.env.LMSTUDIO_MODEL_PREF || "Loaded from Chat UI";
const model = config?.model || process.env.LMSTUDIO_MODEL_PREF;
if (!model) throw new Error("LMStudio must have a valid model set.");
const apiKey = process.env.LMSTUDIO_AUTH_TOKEN ?? null;
const client = new OpenAI({
baseURL: parseLMStudioBasePath(process.env.LMSTUDIO_BASE_PATH),
apiKey: null,
apiKey,
maxRetries: 3,
});

2
server/utils/agents/index.js
View File

@ -253,7 +253,7 @@ class AgentHandler {
case "anthropic":
return process.env.ANTHROPIC_MODEL_PREF ?? "claude-3-sonnet-20240229";
case "lmstudio":
return process.env.LMSTUDIO_MODEL_PREF ?? "server-default";
return process.env.LMSTUDIO_MODEL_PREF ?? null;
case "ollama":
return process.env.OLLAMA_MODEL_PREF ?? "llama3:latest";
case "groq":

11
server/utils/helpers/customModels.js
View File

@ -77,7 +77,7 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) {
case "openrouter":
return await getOpenRouterModels();
case "lmstudio":
return await getLMStudioModels(basePath);
return await getLMStudioModels(basePath, apiKey);
case "koboldcpp":
return await getKoboldCPPModels(basePath);
case "litellm":
@ -331,14 +331,19 @@ async function liteLLMModels(basePath = null, apiKey = null) {
return { models, error: null };
}
async function getLMStudioModels(basePath = null) {
async function getLMStudioModels(basePath = null, _apiKey = null) {
try {
const apiKey =
_apiKey === true
? process.env.LMSTUDIO_AUTH_TOKEN
: _apiKey || process.env.LMSTUDIO_AUTH_TOKEN || null;
const { OpenAI: OpenAIApi } = require("openai");
const openai = new OpenAIApi({
baseURL: parseLMStudioBasePath(
basePath || process.env.LMSTUDIO_BASE_PATH
),
apiKey: null,
apiKey: apiKey || null,
});
const models = await openai.models
.list()

4
server/utils/helpers/updateENV.js
View File

@ -94,6 +94,10 @@ const KEY_MAPPING = {
envKey: "LMSTUDIO_MODEL_TOKEN_LIMIT",
checks: [],
},
LMStudioAuthToken: {
envKey: "LMSTUDIO_AUTH_TOKEN",
checks: [],
},
// LocalAI Settings
LocalAiBasePath: {