From 157e3e4b38163f9c78eaa86c1b230acd3b0bfd47 Mon Sep 17 00:00:00 2001 From: Colin Perry <55003831+17ColinMiPerry@users.noreply.github.com> Date: Tue, 25 Nov 2025 11:16:16 -0800 Subject: [PATCH] Feat/add openrouter embedding models (#4682) * implemented openrouter embedding model support * ran yarn lint * data handling entry --------- Co-authored-by: timothycarambat --- docker/.env.example | 4 + .../OpenRouterOptions/index.jsx | 96 +++++++++++++ .../EmbeddingPreference/index.jsx | 23 +++- .../Steps/DataHandling/index.jsx | 8 ++ server/.env.example | 4 + .../EmbeddingEngines/openRouter/index.js | 126 ++++++++++++++++++ server/utils/helpers/customModels.js | 21 +++ server/utils/helpers/index.js | 3 + server/utils/helpers/updateENV.js | 1 + 9 files changed, 279 insertions(+), 7 deletions(-) create mode 100644 frontend/src/components/EmbeddingSelection/OpenRouterOptions/index.jsx create mode 100644 server/utils/EmbeddingEngines/openRouter/index.js diff --git a/docker/.env.example b/docker/.env.example index 4db7aeff..0cf05f3c 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -215,6 +215,10 @@ GID='1000' # GEMINI_EMBEDDING_API_KEY= # EMBEDDING_MODEL_PREF='text-embedding-004' +# EMBEDDING_ENGINE='openrouter' +# EMBEDDING_MODEL_PREF='baai/bge-m3' +# OPENROUTER_API_KEY='' + ########################################### ######## Vector Database Selection ######## ########################################### diff --git a/frontend/src/components/EmbeddingSelection/OpenRouterOptions/index.jsx b/frontend/src/components/EmbeddingSelection/OpenRouterOptions/index.jsx new file mode 100644 index 00000000..852667a0 --- /dev/null +++ b/frontend/src/components/EmbeddingSelection/OpenRouterOptions/index.jsx @@ -0,0 +1,96 @@ +import { useState, useEffect } from "react"; +import System from "@/models/system"; + +export default function OpenRouterOptions({ settings }) { + return ( +
+
+
+ + +
+ +
+
+ ); +} + +function OpenRouterEmbeddingModelSelection({ settings }) { + const [models, setModels] = useState([]); + const [loading, setLoading] = useState(true); + const [selectedModel, setSelectedModel] = useState( + settings?.EmbeddingModelPref || "" + ); + + useEffect(() => { + async function fetchModels() { + setLoading(true); + const response = await System.customModels("openrouter-embedder"); + const fetchedModels = response?.models || []; + setModels(fetchedModels); + + if ( + settings?.EmbeddingModelPref && + fetchedModels.some((m) => m.id === settings.EmbeddingModelPref) + ) { + setSelectedModel(settings.EmbeddingModelPref); + } else if (fetchedModels.length > 0) { + setSelectedModel(fetchedModels[0].id); + } + + setLoading(false); + } + fetchModels(); + }, [settings?.EmbeddingModelPref]); + + if (loading) { + return ( +
+ + +
+ ); + } + + return ( +
+ + +
+ ); +} diff --git a/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx b/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx index de27acb8..e0066ae4 100644 --- a/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx +++ b/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx @@ -15,6 +15,7 @@ import VoyageAiLogo from "@/media/embeddingprovider/voyageai.png"; import LiteLLMLogo from "@/media/llmprovider/litellm.png"; import GenericOpenAiLogo from "@/media/llmprovider/generic-openai.png"; import MistralAiLogo from "@/media/llmprovider/mistral.jpeg"; +import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg"; import PreLoader from "@/components/Preloader"; import ChangeWarningModal from "@/components/ChangeWarning"; @@ -29,6 +30,7 @@ import CohereEmbeddingOptions from "@/components/EmbeddingSelection/CohereOption import VoyageAiOptions from "@/components/EmbeddingSelection/VoyageAiOptions"; import LiteLLMOptions from "@/components/EmbeddingSelection/LiteLLMOptions"; import GenericOpenAiEmbeddingOptions from "@/components/EmbeddingSelection/GenericOpenAiOptions"; +import OpenRouterOptions from "@/components/EmbeddingSelection/OpenRouterOptions"; import EmbedderItem from "@/components/EmbeddingSelection/EmbedderItem"; import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react"; @@ -90,6 +92,20 @@ const EMBEDDERS = [ description: "Discover, download, and run thousands of cutting edge LLMs in a few clicks.", }, + { + name: "OpenRouter", + value: "openrouter", + logo: OpenRouterLogo, + options: (settings) => , + description: "Run embedding models from OpenRouter.", + }, + { + name: "LiteLLM", + value: "litellm", + logo: LiteLLMLogo, + options: (settings) => , + description: "Run powerful embedding models from LiteLLM.", + }, { name: "Cohere", value: "cohere", @@ -104,13 +120,6 @@ const EMBEDDERS = [ options: (settings) => , description: "Run powerful embedding models from Voyage AI.", }, - { - name: "LiteLLM", - value: "litellm", - logo: LiteLLMLogo, - options: (settings) => , - description: "Run powerful embedding models from LiteLLM.", - }, { name: "Mistral AI", value: "mistral", diff --git a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx index fbb767c8..3c804ec8 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx @@ -403,6 +403,14 @@ export const EMBEDDING_ENGINE_PRIVACY = { ], logo: LMStudioLogo, }, + openrouter: { + name: "OpenRouter", + description: [ + "Your document text is sent to OpenRouter's servers for processing", + "Your document text is stored or managed according to the terms of service of OpenRouter API Terms of Service", + ], + logo: OpenRouterLogo, + }, cohere: { name: "Cohere", description: [ diff --git a/server/.env.example b/server/.env.example index aecd0e14..c7c02a0a 100644 --- a/server/.env.example +++ b/server/.env.example @@ -214,6 +214,10 @@ SIG_SALT='salt' # Please generate random string at least 32 chars long. # GEMINI_EMBEDDING_API_KEY= # EMBEDDING_MODEL_PREF='text-embedding-004' +# EMBEDDING_ENGINE='openrouter' +# EMBEDDING_MODEL_PREF='baai/bge-m3' +# OPENROUTER_API_KEY='' + ########################################### ######## Vector Database Selection ######## ########################################### diff --git a/server/utils/EmbeddingEngines/openRouter/index.js b/server/utils/EmbeddingEngines/openRouter/index.js new file mode 100644 index 00000000..22dd365b --- /dev/null +++ b/server/utils/EmbeddingEngines/openRouter/index.js @@ -0,0 +1,126 @@ +const { toChunks } = require("../../helpers"); + +class OpenRouterEmbedder { + constructor() { + if (!process.env.OPENROUTER_API_KEY) + throw new Error("No OpenRouter API key was set."); + this.className = "OpenRouterEmbedder"; + const { OpenAI: OpenAIApi } = require("openai"); + this.openai = new OpenAIApi({ + baseURL: "https://openrouter.ai/api/v1", + apiKey: process.env.OPENROUTER_API_KEY, + defaultHeaders: { + "HTTP-Referer": "https://anythingllm.com", + "X-Title": "AnythingLLM", + }, + }); + this.model = process.env.EMBEDDING_MODEL_PREF || "baai/bge-m3"; + + // Limit of how many strings we can process in a single pass to stay with resource or network limits + this.maxConcurrentChunks = 500; + + // https://openrouter.ai/docs/api/reference/embeddings + this.embeddingMaxChunkLength = 8_191; + } + + log(text, ...args) { + console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args); + } + + async embedTextInput(textInput) { + const result = await this.embedChunks( + Array.isArray(textInput) ? textInput : [textInput] + ); + return result?.[0] || []; + } + + async embedChunks(textChunks = []) { + this.log(`Embedding ${textChunks.length} document chunks...`); + const embeddingRequests = []; + for (const chunk of toChunks(textChunks, this.maxConcurrentChunks)) { + embeddingRequests.push( + new Promise((resolve) => { + this.openai.embeddings + .create({ + model: this.model, + input: chunk, + }) + .then((result) => { + resolve({ data: result?.data, error: null }); + }) + .catch((e) => { + e.type = + e?.response?.data?.error?.code || + e?.response?.status || + "failed_to_embed"; + e.message = e?.response?.data?.error?.message || e.message; + resolve({ data: [], error: e }); + }); + }) + ); + } + + const { data = [], error = null } = await Promise.all( + embeddingRequests + ).then((results) => { + // If any errors were returned from OpenAI abort the entire sequence because the embeddings + // will be incomplete. + const errors = results + .filter((res) => !!res.error) + .map((res) => res.error) + .flat(); + if (errors.length > 0) { + let uniqueErrors = new Set(); + errors.map((error) => + uniqueErrors.add(`[${error.type}]: ${error.message}`) + ); + + return { + data: [], + error: Array.from(uniqueErrors).join(", "), + }; + } + return { + data: results.map((res) => res?.data || []).flat(), + error: null, + }; + }); + + if (!!error) throw new Error(`OpenRouter Failed to embed: ${error}`); + return data.length > 0 && + data.every((embd) => embd.hasOwnProperty("embedding")) + ? data.map((embd) => embd.embedding) + : null; + } +} + +async function fetchOpenRouterEmbeddingModels() { + return await fetch(`https://openrouter.ai/api/v1/embeddings/models`, { + method: "GET", + headers: { "Content-Type": "application/json" }, + }) + .then((res) => res.json()) + .then(({ data = [] }) => { + const models = {}; + data.forEach((model) => { + models[model.id] = { + id: model.id, + name: model.name || model.id, + organization: + model.id.split("/")[0].charAt(0).toUpperCase() + + model.id.split("/")[0].slice(1), + maxLength: model.context_length, + }; + }); + return models; + }) + .catch((e) => { + console.error("OpenRouter:fetchEmbeddingModels", e.message); + return {}; + }); +} + +module.exports = { + OpenRouterEmbedder, + fetchOpenRouterEmbeddingModels, +}; diff --git a/server/utils/helpers/customModels.js b/server/utils/helpers/customModels.js index 5c8d1962..27371909 100644 --- a/server/utils/helpers/customModels.js +++ b/server/utils/helpers/customModels.js @@ -1,4 +1,7 @@ const { fetchOpenRouterModels } = require("../AiProviders/openRouter"); +const { + fetchOpenRouterEmbeddingModels, +} = require("../EmbeddingEngines/openRouter"); const { fetchApiPieModels } = require("../AiProviders/apipie"); const { perplexityModels } = require("../AiProviders/perplexity"); const { fireworksAiModels } = require("../AiProviders/fireworksAi"); @@ -42,6 +45,7 @@ const SUPPORT_CUSTOM_MODELS = [ // Embedding Engines "native-embedder", "cohere-embedder", + "openrouter-embedder", ]; async function getCustomModels(provider = "", apiKey = null, basePath = null) { @@ -107,6 +111,8 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) { return await getNativeEmbedderModels(); case "cohere-embedder": return await getCohereModels(apiKey, "embed"); + case "openrouter-embedder": + return await getOpenRouterEmbeddingModels(); default: return { models: [], error: "Invalid provider for custom models" }; } @@ -824,6 +830,21 @@ async function getZAiModels(_apiKey = null) { return { models, error: null }; } +async function getOpenRouterEmbeddingModels() { + const knownModels = await fetchOpenRouterEmbeddingModels(); + if (!Object.keys(knownModels).length === 0) + return { models: [], error: null }; + + const models = Object.values(knownModels).map((model) => { + return { + id: model.id, + organization: model.organization, + name: model.name, + }; + }); + return { models, error: null }; +} + module.exports = { getCustomModels, SUPPORT_CUSTOM_MODELS, diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index 765415c3..f1cc1fde 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -279,6 +279,9 @@ function getEmbeddingEngineSelection() { case "gemini": const { GeminiEmbedder } = require("../EmbeddingEngines/gemini"); return new GeminiEmbedder(); + case "openrouter": + const { OpenRouterEmbedder } = require("../EmbeddingEngines/openRouter"); + return new OpenRouterEmbedder(); default: return new NativeEmbedder(); } diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 3f34c1d7..ea73fe2f 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -920,6 +920,7 @@ function supportedEmbeddingModel(input = "") { "litellm", "generic-openai", "mistral", + "openrouter", ]; return supported.includes(input) ? null