Feat/add openrouter embedding models (#4682)

* implemented openrouter embedding model support

* ran yarn lint

* data handling entry

---------

Co-authored-by: timothycarambat <rambat1010@gmail.com>

docker/.env.example

@@ -215,6 +215,10 @@ GID='1000'
 # GEMINI_EMBEDDING_API_KEY=
 # EMBEDDING_MODEL_PREF='text-embedding-004'
 
+# EMBEDDING_ENGINE='openrouter'
+# EMBEDDING_MODEL_PREF='baai/bge-m3'
+# OPENROUTER_API_KEY=''
+
 ###########################################
 ######## Vector Database Selection ########
 ###########################################

frontend/src/components/EmbeddingSelection/OpenRouterOptions/index.jsx

@@ -0,0 +1,96 @@
+import { useState, useEffect } from "react";
+import System from "@/models/system";
+
+export default function OpenRouterOptions({ settings }) {
+  return (
+    <div className="w-full flex flex-col gap-y-4">
+      <div className="w-full flex items-center gap-[36px] mt-1.5">
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            API Key
+          </label>
+          <input
+            type="password"
+            name="OpenRouterApiKey"
+            className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="OpenRouter API Key"
+            defaultValue={settings?.OpenRouterApiKey ? "*".repeat(20) : ""}
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+        <OpenRouterEmbeddingModelSelection settings={settings} />
+      </div>
+    </div>
+  );
+}
+
+function OpenRouterEmbeddingModelSelection({ settings }) {
+  const [models, setModels] = useState([]);
+  const [loading, setLoading] = useState(true);
+  const [selectedModel, setSelectedModel] = useState(
+    settings?.EmbeddingModelPref || ""
+  );
+
+  useEffect(() => {
+    async function fetchModels() {
+      setLoading(true);
+      const response = await System.customModels("openrouter-embedder");
+      const fetchedModels = response?.models || [];
+      setModels(fetchedModels);
+
+      if (
+        settings?.EmbeddingModelPref &&
+        fetchedModels.some((m) => m.id === settings.EmbeddingModelPref)
+      ) {
+        setSelectedModel(settings.EmbeddingModelPref);
+      } else if (fetchedModels.length > 0) {
+        setSelectedModel(fetchedModels[0].id);
+      }
+
+      setLoading(false);
+    }
+    fetchModels();
+  }, [settings?.EmbeddingModelPref]);
+
+  if (loading) {
+    return (
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-3">
+          Model Preference
+        </label>
+        <select
+          name="EmbeddingModelPref"
+          disabled={true}
+          className="border-none bg-theme-settings-input-bg border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+        >
+          <option disabled={true} selected={true}>
+            -- loading available models --
+          </option>
+        </select>
+      </div>
+    );
+  }
+
+  return (
+    <div className="flex flex-col w-60">
+      <label className="text-white text-sm font-semibold block mb-3">
+        Model Preference
+      </label>
+      <select
+        name="EmbeddingModelPref"
+        required={true}
+        value={selectedModel}
+        onChange={(e) => setSelectedModel(e.target.value)}
+        className="border-none bg-theme-settings-input-bg border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+      >
+        {models.map((model) => (
+          <option key={model.id} value={model.id}>
+            {model.name}
+          </option>
+        ))}
+      </select>
+    </div>
+  );
+}

frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx

@@ -15,6 +15,7 @@ import VoyageAiLogo from "@/media/embeddingprovider/voyageai.png";
 import LiteLLMLogo from "@/media/llmprovider/litellm.png";
 import GenericOpenAiLogo from "@/media/llmprovider/generic-openai.png";
 import MistralAiLogo from "@/media/llmprovider/mistral.jpeg";
+import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
 
 import PreLoader from "@/components/Preloader";
 import ChangeWarningModal from "@/components/ChangeWarning";
@@ -29,6 +30,7 @@ import CohereEmbeddingOptions from "@/components/EmbeddingSelection/CohereOption
 import VoyageAiOptions from "@/components/EmbeddingSelection/VoyageAiOptions";
 import LiteLLMOptions from "@/components/EmbeddingSelection/LiteLLMOptions";
 import GenericOpenAiEmbeddingOptions from "@/components/EmbeddingSelection/GenericOpenAiOptions";
+import OpenRouterOptions from "@/components/EmbeddingSelection/OpenRouterOptions";
 
 import EmbedderItem from "@/components/EmbeddingSelection/EmbedderItem";
 import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
@@ -90,6 +92,20 @@ const EMBEDDERS = [
     description:
       "Discover, download, and run thousands of cutting edge LLMs in a few clicks.",
   },
+  {
+    name: "OpenRouter",
+    value: "openrouter",
+    logo: OpenRouterLogo,
+    options: (settings) => <OpenRouterOptions settings={settings} />,
+    description: "Run embedding models from OpenRouter.",
+  },
+  {
+    name: "LiteLLM",
+    value: "litellm",
+    logo: LiteLLMLogo,
+    options: (settings) => <LiteLLMOptions settings={settings} />,
+    description: "Run powerful embedding models from LiteLLM.",
+  },
   {
     name: "Cohere",
     value: "cohere",
@@ -104,13 +120,6 @@ const EMBEDDERS = [
     options: (settings) => <VoyageAiOptions settings={settings} />,
     description: "Run powerful embedding models from Voyage AI.",
   },
-  {
-    name: "LiteLLM",
-    value: "litellm",
-    logo: LiteLLMLogo,
-    options: (settings) => <LiteLLMOptions settings={settings} />,
-    description: "Run powerful embedding models from LiteLLM.",
-  },
   {
     name: "Mistral AI",
     value: "mistral",

frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx

@@ -403,6 +403,14 @@ export const EMBEDDING_ENGINE_PRIVACY = {
     ],
     logo: LMStudioLogo,
   },
+  openrouter: {
+    name: "OpenRouter",
+    description: [
+      "Your document text is sent to OpenRouter's servers for processing",
+      "Your document text is stored or managed according to the terms of service of OpenRouter API Terms of Service",
+    ],
+    logo: OpenRouterLogo,
+  },
   cohere: {
     name: "Cohere",
     description: [

server/.env.example

@@ -214,6 +214,10 @@ SIG_SALT='salt' # Please generate random string at least 32 chars long.
 # GEMINI_EMBEDDING_API_KEY=
 # EMBEDDING_MODEL_PREF='text-embedding-004'
 
+# EMBEDDING_ENGINE='openrouter'
+# EMBEDDING_MODEL_PREF='baai/bge-m3'
+# OPENROUTER_API_KEY=''
+
 ###########################################
 ######## Vector Database Selection ########
 ###########################################

server/utils/EmbeddingEngines/openRouter/index.js

@@ -0,0 +1,126 @@
+const { toChunks } = require("../../helpers");
+
+class OpenRouterEmbedder {
+  constructor() {
+    if (!process.env.OPENROUTER_API_KEY)
+      throw new Error("No OpenRouter API key was set.");
+    this.className = "OpenRouterEmbedder";
+    const { OpenAI: OpenAIApi } = require("openai");
+    this.openai = new OpenAIApi({
+      baseURL: "https://openrouter.ai/api/v1",
+      apiKey: process.env.OPENROUTER_API_KEY,
+      defaultHeaders: {
+        "HTTP-Referer": "https://anythingllm.com",
+        "X-Title": "AnythingLLM",
+      },
+    });
+    this.model = process.env.EMBEDDING_MODEL_PREF || "baai/bge-m3";
+
+    // Limit of how many strings we can process in a single pass to stay with resource or network limits
+    this.maxConcurrentChunks = 500;
+
+    // https://openrouter.ai/docs/api/reference/embeddings
+    this.embeddingMaxChunkLength = 8_191;
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
+  }
+
+  async embedTextInput(textInput) {
+    const result = await this.embedChunks(
+      Array.isArray(textInput) ? textInput : [textInput]
+    );
+    return result?.[0] || [];
+  }
+
+  async embedChunks(textChunks = []) {
+    this.log(`Embedding ${textChunks.length} document chunks...`);
+    const embeddingRequests = [];
+    for (const chunk of toChunks(textChunks, this.maxConcurrentChunks)) {
+      embeddingRequests.push(
+        new Promise((resolve) => {
+          this.openai.embeddings
+            .create({
+              model: this.model,
+              input: chunk,
+            })
+            .then((result) => {
+              resolve({ data: result?.data, error: null });
+            })
+            .catch((e) => {
+              e.type =
+                e?.response?.data?.error?.code ||
+                e?.response?.status ||
+                "failed_to_embed";
+              e.message = e?.response?.data?.error?.message || e.message;
+              resolve({ data: [], error: e });
+            });
+        })
+      );
+    }
+
+    const { data = [], error = null } = await Promise.all(
+      embeddingRequests
+    ).then((results) => {
+      // If any errors were returned from OpenAI abort the entire sequence because the embeddings
+      // will be incomplete.
+      const errors = results
+        .filter((res) => !!res.error)
+        .map((res) => res.error)
+        .flat();
+      if (errors.length > 0) {
+        let uniqueErrors = new Set();
+        errors.map((error) =>
+          uniqueErrors.add(`[${error.type}]: ${error.message}`)
+        );
+
+        return {
+          data: [],
+          error: Array.from(uniqueErrors).join(", "),
+        };
+      }
+      return {
+        data: results.map((res) => res?.data || []).flat(),
+        error: null,
+      };
+    });
+
+    if (!!error) throw new Error(`OpenRouter Failed to embed: ${error}`);
+    return data.length > 0 &&
+      data.every((embd) => embd.hasOwnProperty("embedding"))
+      ? data.map((embd) => embd.embedding)
+      : null;
+  }
+}
+
+async function fetchOpenRouterEmbeddingModels() {
+  return await fetch(`https://openrouter.ai/api/v1/embeddings/models`, {
+    method: "GET",
+    headers: { "Content-Type": "application/json" },
+  })
+    .then((res) => res.json())
+    .then(({ data = [] }) => {
+      const models = {};
+      data.forEach((model) => {
+        models[model.id] = {
+          id: model.id,
+          name: model.name || model.id,
+          organization:
+            model.id.split("/")[0].charAt(0).toUpperCase() +
+            model.id.split("/")[0].slice(1),
+          maxLength: model.context_length,
+        };
+      });
+      return models;
+    })
+    .catch((e) => {
+      console.error("OpenRouter:fetchEmbeddingModels", e.message);
+      return {};
+    });
+}
+
+module.exports = {
+  OpenRouterEmbedder,
+  fetchOpenRouterEmbeddingModels,
+};

server/utils/helpers/customModels.js

@@ -1,4 +1,7 @@
 const { fetchOpenRouterModels } = require("../AiProviders/openRouter");
+const {
+  fetchOpenRouterEmbeddingModels,
+} = require("../EmbeddingEngines/openRouter");
 const { fetchApiPieModels } = require("../AiProviders/apipie");
 const { perplexityModels } = require("../AiProviders/perplexity");
 const { fireworksAiModels } = require("../AiProviders/fireworksAi");
@@ -42,6 +45,7 @@ const SUPPORT_CUSTOM_MODELS = [
   // Embedding Engines
   "native-embedder",
   "cohere-embedder",
+  "openrouter-embedder",
 ];
 
 async function getCustomModels(provider = "", apiKey = null, basePath = null) {
@@ -107,6 +111,8 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) {
       return await getNativeEmbedderModels();
     case "cohere-embedder":
       return await getCohereModels(apiKey, "embed");
+    case "openrouter-embedder":
+      return await getOpenRouterEmbeddingModels();
     default:
       return { models: [], error: "Invalid provider for custom models" };
   }
@@ -824,6 +830,21 @@ async function getZAiModels(_apiKey = null) {
   return { models, error: null };
 }
 
+async function getOpenRouterEmbeddingModels() {
+  const knownModels = await fetchOpenRouterEmbeddingModels();
+  if (!Object.keys(knownModels).length === 0)
+    return { models: [], error: null };
+
+  const models = Object.values(knownModels).map((model) => {
+    return {
+      id: model.id,
+      organization: model.organization,
+      name: model.name,
+    };
+  });
+  return { models, error: null };
+}
+
 module.exports = {
   getCustomModels,
   SUPPORT_CUSTOM_MODELS,

server/utils/helpers/index.js

@@ -279,6 +279,9 @@ function getEmbeddingEngineSelection() {
     case "gemini":
       const { GeminiEmbedder } = require("../EmbeddingEngines/gemini");
       return new GeminiEmbedder();
+    case "openrouter":
+      const { OpenRouterEmbedder } = require("../EmbeddingEngines/openRouter");
+      return new OpenRouterEmbedder();
     default:
       return new NativeEmbedder();
   }

server/utils/helpers/updateENV.js

@@ -920,6 +920,7 @@ function supportedEmbeddingModel(input = "") {
     "litellm",
     "generic-openai",
     "mistral",
+    "openrouter",
   ];
   return supported.includes(input)
     ? null