From 2aeb4c2961d3beaca42e428919e3e394ab42e640 Mon Sep 17 00:00:00 2001 From: cnJasonZ Date: Fri, 28 Feb 2025 02:53:00 +0800 Subject: [PATCH] Add new model provider PPIO (#3211) * feat: add new model provider PPIO * fix: fix ppio model fetching * fix: code lint * reorder LLM update interface for streaming and chats to use valid keys linting --------- Co-authored-by: timothycarambat --- .vscode/settings.json | 1 + README.md | 1 + docker/.env.example | 4 + .../LLMSelection/PPIOLLMOptions/index.jsx | 100 +++++++ frontend/src/hooks/useGetProvidersModels.js | 1 + frontend/src/media/llmprovider/ppio.png | Bin 0 -> 3590 bytes .../GeneralSettings/LLMPreference/index.jsx | 11 + .../Steps/DataHandling/index.jsx | 10 +- .../Steps/LLMPreference/index.jsx | 11 + .../AgentConfig/AgentLLMSelection/index.jsx | 1 + locales/README.fa-IR.md | 1 + locales/README.ja-JP.md | 1 + locales/README.tr-TR.md | 1 + locales/README.zh-CN.md | 1 + server/.env.example | 4 + server/models/systemSettings.js | 4 + server/storage/models/.gitignore | 2 +- server/utils/AiProviders/ppio/index.js | 266 ++++++++++++++++++ server/utils/agents/aibitat/index.js | 3 +- .../agents/aibitat/providers/ai-provider.js | 8 + .../utils/agents/aibitat/providers/index.js | 2 + server/utils/agents/aibitat/providers/ppio.js | 115 ++++++++ server/utils/agents/index.js | 6 + server/utils/helpers/customModels.js | 17 ++ server/utils/helpers/index.js | 6 + server/utils/helpers/updateENV.js | 11 + 26 files changed, 585 insertions(+), 3 deletions(-) create mode 100644 frontend/src/components/LLMSelection/PPIOLLMOptions/index.jsx create mode 100644 frontend/src/media/llmprovider/ppio.png create mode 100644 server/utils/AiProviders/ppio/index.js create mode 100644 server/utils/agents/aibitat/providers/ppio.js diff --git a/.vscode/settings.json b/.vscode/settings.json index 307bbe6c..e6b76c9e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -41,6 +41,7 @@ "opendocument", "openrouter", "pagerender", + "ppio", "Qdrant", "royalblue", "SearchApi", diff --git a/README.md b/README.md index f5782936..ae82df2c 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ AnythingLLM divides your documents into objects called `workspaces`. A Workspace - [Apipie](https://apipie.ai/) - [xAI](https://x.ai/) - [Novita AI (chat models)](https://novita.ai/model-api/product/llm-api?utm_source=github_anything-llm&utm_medium=github_readme&utm_campaign=link) +- [PPIO](https://ppinfra.com?utm_source=github_anything-llm) **Embedder models:** diff --git a/docker/.env.example b/docker/.env.example index d3cc68e1..40acac84 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -126,6 +126,10 @@ GID='1000' # DEEPSEEK_API_KEY='your-deepseek-api-key-here' # DEEPSEEK_MODEL_PREF='deepseek-chat' +# LLM_PROVIDER='ppio' +# PPIO_API_KEY='your-ppio-api-key-here' +# PPIO_MODEL_PREF=deepseek/deepseek-v3/community + ########################################### ######## Embedding API SElECTION ########## ########################################### diff --git a/frontend/src/components/LLMSelection/PPIOLLMOptions/index.jsx b/frontend/src/components/LLMSelection/PPIOLLMOptions/index.jsx new file mode 100644 index 00000000..99b617f1 --- /dev/null +++ b/frontend/src/components/LLMSelection/PPIOLLMOptions/index.jsx @@ -0,0 +1,100 @@ +import System from "@/models/system"; +import { useState, useEffect } from "react"; + +export default function PPIOLLMOptions({ settings }) { + return ( +
+
+
+ + +
+ {!settings?.credentialsOnly && ( + + )} +
+
+ ); +} + +function PPIOModelSelection({ settings }) { + const [groupedModels, setGroupedModels] = useState({}); + const [loading, setLoading] = useState(true); + + useEffect(() => { + async function fetchModels() { + setLoading(true); + const { models } = await System.customModels("ppio"); + if (models?.length > 0) { + const modelsByOrganization = models.reduce((acc, model) => { + acc[model.organization] = acc[model.organization] || []; + acc[model.organization].push(model); + return acc; + }, {}); + setGroupedModels(modelsByOrganization); + } + setLoading(false); + } + fetchModels(); + }, []); + + if (loading || Object.keys(groupedModels).length === 0) { + return ( +
+ + +
+ ); + } + + return ( +
+ + +
+ ); +} diff --git a/frontend/src/hooks/useGetProvidersModels.js b/frontend/src/hooks/useGetProvidersModels.js index 8c1b998c..2019a19d 100644 --- a/frontend/src/hooks/useGetProvidersModels.js +++ b/frontend/src/hooks/useGetProvidersModels.js @@ -79,6 +79,7 @@ const groupedProviders = [ "openai", "novita", "openrouter", + "ppio", ]; export default function useGetProviderModels(provider = null) { const [defaultModels, setDefaultModels] = useState([]); diff --git a/frontend/src/media/llmprovider/ppio.png b/frontend/src/media/llmprovider/ppio.png new file mode 100644 index 0000000000000000000000000000000000000000..37b131a54404718a78c102edfb1e21dfbf200e03 GIT binary patch literal 3590 zcmcgv`9IT-AAfHRvpKe6Kic*ts~+j;~hf)044ojfCx5HTSq1+#Kr^*)P5FOJ`xabj5!7X8ZzPg?rcYs3JJy- z+epxQu|a^X1FCoPd9$Pa*<@dHI9s+yo=&b!3(*O7CNVeV3?9;SL6X}%YfU2~_e?3q z!zYmQF`55^k*~=`d#aOV3k~ok?1i`Q#$5HzZ_wNM{AP0^y1iuHP1a`Xb;R4~!-*kjZ8YxbMfdFW93*9w4Z0;QGF|8+LGm zc?(w#niYD7uUpCrvKW7*U5CgV2>+?He?1c35Y>pwTjr9ZsJe~1PKhv?97=HUlxWi? zo=2t7l=p$#yvNeEma0OE!~3#f)am>Q0hS717w2~zP{Qg6(MgLFq!EN!O`>Y~{2PX;Wzg@_J`p`fBocxcIBdxx*J~(p2BC7OwRA{W1A)EwV6Y* zD>3-ktc$UGDg&Alrbznc+eYhq_a7Q(IL{@m8PfqxMJ<>F8tKDtI3}Utbgr}Cx%%dX z$N#Eq#)WKjy(gdt;4!FcSMiGJbWd=dKsTs##sBX=1(EU(tV- zt)YFmUo9TE8Iuz}B6kBhv8QxMT55ZVs?)JE^Fpu~o*`F#Nx|6y4&)3bh5T51|rR2e{ZM#8b{t45I z<<8U3r1RA4bL+XvI(D$7MdNJl1nTt8@K&1>Nz|d?tdBhV#}AV@Duu>SJ}x7g;9~K( zoT+F~^NJ$SIWze@sA}Pg>^07;13C!>TXPYP*m;{fKTP6BCxyz+pem0WNuQ=L7D2fM zctf9EIq6q&CnKO^;xNMuMv2yu;xvutpt@;7P;BG^1`0G{7~Op2ZJFnRp|~Y)#L6V0 z+yyc@sd5d;N9U9%VN*J1gf&uI!PPe}bZ_6%J~CaP5@5&mh6Fz(9hS6Zm(5-Y=(7z# z1}+G#j~IrziOhNm-onxJ$_PAD#W6ROT($T7w6nOZy~-T+4^m{mlYwKiG98-0h<>D#!`2KY}ozWq4gV6AbU5J-4beW<&>KR1smBh9xdJg-1NoA+w2)Ze zCZphVeMKmQ!O^{ydB^WDk0bPzf^!%Q{dC=e7$!-#kT{8m7@v7@=35ZyN9`q=q68d1 z(<&~O-8m3HlBADZ)3U}656s~_c1O3cZv>4Fl)k*_ntavjEEy)5rI;Ph56$NO0cL~_ z7Av2N<1cLy8$b-$zyd-GoF(lf4axF*WUP8@G{~noEWVIt1jU>cuKXlzBTX6A(^9+r zJS9|dI;d?#f9-S<0wIj|qN-oJ9qI_~So>ImyMfO75;!W6M7CtSAfXF{%Pv5=UaE&> z_xk`tUn(Sl%qE0X-}#EE2+^U`>Ad!$8*9)h?#q))-*b#@ppjKMptGd0k$52_K6kvi zuCEm<5kyN(s6gZ{E&oGSvg^*a&=7|&#ZJ0r_LBK~ruxcNgo1jgd1rondUpL{6St=M=iy%Q@2(wAXE}U|bq#AMcrvukqc>X(8)JD>jm4AWKP!uryVn6kUbK zl@cAGLOd-M40q0ZqOb9&R9_Doy*>j`@!3ZGbu_<`_D5|#>6-9|zt8CvTiT>6L?t4M z+9Dla3Q+F7EV6bBvm=7Je_lv%6sd6smo+XfctYgSKfj1B@0B(6SF2J9$>MA{ za{2`%U3Z#9gq(ji8bVcj>N@!CuX3vQvZ3KODonRy zWF7xiKGkg8_VaW3^7WU7pR_Q~Z|TdcrW_=*k5*gG24k7|XHWhq_PKt3ICVOOBI?eU zAA0x4gN~+S8XizgpUz`x8Xz1oc!%5GKUMQ+2RBoz6NQjI7KB-)M>1b-g(b9E@9y%* zL%6pAtJC21N|>NaUXD}J)JtvdT1%+J^zPsfT|0ip8947q76cC3$-49b)scMH*>eH+ znxYfo^v}ukv&g02ZyKQOZz>J+4ONuGL@PP29nZNGB*g(5!QF>SZMO6Ftm7sV&O+jQ zPx_9WUnubTI2h@kJ%YIp-6E^E8H?3mBgZ*U1HS0ebSgvJ`_S6Gd?Jg|8Rc3Ig? zw5OsNSYqS3&_l zm+%wCr_u$u`^eD^b{dsN{RV6N3;l^{8c+VpXXk(55|9kPh$Ll&X8x+iqJbeQpsa`U zSK*>~$Hd02&F&QniKp_OV#x(haL!wL6;7y@tiNx=T(e9pZ%i0`!Vkia38i=CiP^%j zQ=f^gm_iGY8hhYLdrY6xvEv+2VDu?|z*}z8sK<`B5>z_)I&mI?(obO~)BFj4b!?Tv zmlHLQUTXjPAbRIdv}z75jW2AwNDqtWSAInp9(J=iUC7DsD|ei znb5+clmE^VkUx6^iGej7S05c6H^+WIW3(|~lYgUQryee?f%rW~QiW2ws3IiBV1_Am z-q=dxwE%2h#CgDb43?V>OBj@J&tJ6(VH#caly#ni0(fbiAwWL2uQ5un-~X zQwGCd|Kt=qrYL+OzFe+{mDHg~foj~U^vRA*>Llw^dm8u~uG%+Xm@RZMVf>(-{CCQ+ zw4Q4NDb#bWDxiUB4T8Or^&)33BFj4rP=qvK+W5y}SGBzXdp(5cNeqzVpbeAsPk>8U zVRhrbu6j4eMEib}QU=H2#LeOY+!9>4QLbaU`mUUkBL@XU=9|QordRR;+$F*Z-JgHSscHX2*k9wzz3oKiRgm2Hj*tUY4 z*zy`^SwHaDs@x?>D#lDrcU@t5SV1meHlORcq=PGffd-5@S8Z+g|! z7i*h7mJ~j`mW%#xI??G%y&(to5TnND-;xynUIzU?$VJwJ^ruP5vByl0VjDofSs2%1 H-6;P7!b)F7 literal 0 HcmV?d00001 diff --git a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx index 96ec360f..8f137805 100644 --- a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx +++ b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx @@ -30,6 +30,7 @@ import DeepSeekLogo from "@/media/llmprovider/deepseek.png"; import APIPieLogo from "@/media/llmprovider/apipie.png"; import XAILogo from "@/media/llmprovider/xai.png"; import NvidiaNimLogo from "@/media/llmprovider/nvidia-nim.png"; +import PPIOLogo from "@/media/llmprovider/ppio.png"; import PreLoader from "@/components/Preloader"; import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; @@ -57,6 +58,7 @@ import DeepSeekOptions from "@/components/LLMSelection/DeepSeekOptions"; import ApiPieLLMOptions from "@/components/LLMSelection/ApiPieOptions"; import XAILLMOptions from "@/components/LLMSelection/XAiLLMOptions"; import NvidiaNimOptions from "@/components/LLMSelection/NvidiaNimOptions"; +import PPIOLLMOptions from "@/components/LLMSelection/PPIOLLMOptions"; import LLMItem from "@/components/LLMSelection/LLMItem"; import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react"; @@ -246,6 +248,15 @@ export const AVAILABLE_LLM_PROVIDERS = [ description: "Run DeepSeek's powerful LLMs.", requiredConfig: ["DeepSeekApiKey"], }, + { + name: "PPIO", + value: "ppio", + logo: PPIOLogo, + options: (settings) => , + description: + "Run stable and cost-efficient open-source LLM APIs, such as DeepSeek, Llama, Qwen etc.", + requiredConfig: ["PPIOApiKey"], + }, { name: "AWS Bedrock", value: "bedrock", diff --git a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx index cab4d667..76372e83 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx @@ -25,7 +25,6 @@ import AWSBedrockLogo from "@/media/llmprovider/bedrock.png"; import DeepSeekLogo from "@/media/llmprovider/deepseek.png"; import APIPieLogo from "@/media/llmprovider/apipie.png"; import XAILogo from "@/media/llmprovider/xai.png"; - import CohereLogo from "@/media/llmprovider/cohere.png"; import ZillizLogo from "@/media/vectordbs/zilliz.png"; import AstraDBLogo from "@/media/vectordbs/astraDB.png"; @@ -36,6 +35,7 @@ import WeaviateLogo from "@/media/vectordbs/weaviate.png"; import QDrantLogo from "@/media/vectordbs/qdrant.png"; import MilvusLogo from "@/media/vectordbs/milvus.png"; import VoyageAiLogo from "@/media/embeddingprovider/voyageai.png"; +import PPIOLogo from "@/media/llmprovider/ppio.png"; import React, { useState, useEffect } from "react"; import paths from "@/utils/paths"; @@ -226,6 +226,14 @@ export const LLM_SELECTION_PRIVACY = { ], logo: XAILogo, }, + ppio: { + name: "PPIO", + description: [ + "Your chats will not be used for training", + "Your prompts and document text used in response creation are visible to PPIO", + ], + logo: PPIOLogo, + }, }; export const VECTOR_DB_PRIVACY = { diff --git a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx index 9ea733b9..16f15675 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx @@ -25,6 +25,8 @@ import NovitaLogo from "@/media/llmprovider/novita.png"; import XAILogo from "@/media/llmprovider/xai.png"; import NvidiaNimLogo from "@/media/llmprovider/nvidia-nim.png"; import CohereLogo from "@/media/llmprovider/cohere.png"; +import PPIOLogo from "@/media/llmprovider/ppio.png"; + import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions"; import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions"; @@ -50,6 +52,7 @@ import ApiPieLLMOptions from "@/components/LLMSelection/ApiPieOptions"; import NovitaLLMOptions from "@/components/LLMSelection/NovitaLLMOptions"; import XAILLMOptions from "@/components/LLMSelection/XAiLLMOptions"; import NvidiaNimOptions from "@/components/LLMSelection/NvidiaNimOptions"; +import PPIOLLMOptions from "@/components/LLMSelection/PPIOLLMOptions"; import LLMItem from "@/components/LLMSelection/LLMItem"; import System from "@/models/system"; @@ -213,6 +216,14 @@ const LLMS = [ options: (settings) => , description: "Run DeepSeek's powerful LLMs.", }, + { + name: "PPIO", + value: "ppio", + logo: PPIOLogo, + options: (settings) => , + description: + "Run stable and cost-efficient open-source LLM APIs, such as DeepSeek, Llama, Qwen etc.", + }, { name: "APIpie", value: "apipie", diff --git a/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx b/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx index ca936a83..10d98683 100644 --- a/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx +++ b/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx @@ -25,6 +25,7 @@ const ENABLED_PROVIDERS = [ "bedrock", "fireworksai", "deepseek", + "ppio", "litellm", "apipie", "xai", diff --git a/locales/README.fa-IR.md b/locales/README.fa-IR.md index dcb71f9d..33305af9 100644 --- a/locales/README.fa-IR.md +++ b/locales/README.fa-IR.md @@ -103,6 +103,7 @@ AnythingLLM اسناد شما را به اشیایی به نام `workspaces` ت - [Apipie](https://apipie.ai/) - [xAI](https://x.ai/) - [Novita AI (chat models)](https://novita.ai/model-api/product/llm-api?utm_source=github_anything-llm&utm_medium=github_readme&utm_campaign=link) +- [PPIO](https://ppinfra.com?utm_source=github_anything-llm)
diff --git a/locales/README.ja-JP.md b/locales/README.ja-JP.md index 9ba566eb..5c7ca1e8 100644 --- a/locales/README.ja-JP.md +++ b/locales/README.ja-JP.md @@ -90,6 +90,7 @@ AnythingLLMのいくつかのクールな機能 - [Groq](https://groq.com/) - [Cohere](https://cohere.com/) - [KoboldCPP](https://github.com/LostRuins/koboldcpp) +- [PPIO](https://ppinfra.com?utm_source=github_anything-llm) **埋め込みモデル：** diff --git a/locales/README.tr-TR.md b/locales/README.tr-TR.md index 026bc7d4..4bd9fe33 100644 --- a/locales/README.tr-TR.md +++ b/locales/README.tr-TR.md @@ -100,6 +100,7 @@ AnythingLLM, belgelerinizi **"çalışma alanları" (workspaces)** adı verilen - [Apipie](https://apipie.ai/) - [xAI](https://x.ai/) - [Novita AI (chat models)](https://novita.ai/model-api/product/llm-api?utm_source=github_anything-llm&utm_medium=github_readme&utm_campaign=link) +- [PPIO](https://ppinfra.com?utm_source=github_anything-llm) **Embedder modelleri:** diff --git a/locales/README.zh-CN.md b/locales/README.zh-CN.md index 095116fc..f6ecdc08 100644 --- a/locales/README.zh-CN.md +++ b/locales/README.zh-CN.md @@ -86,6 +86,7 @@ AnythingLLM的一些酷炫特性 - [Groq](https://groq.com/) - [Cohere](https://cohere.com/) - [KoboldCPP](https://github.com/LostRuins/koboldcpp) +- [PPIO (聊天模型)](https://ppinfra.com?utm_source=github_anything-llm) **支持的嵌入模型：** diff --git a/server/.env.example b/server/.env.example index 1975ada8..f5fa69a3 100644 --- a/server/.env.example +++ b/server/.env.example @@ -116,6 +116,10 @@ SIG_SALT='salt' # Please generate random string at least 32 chars long. # NVIDIA_NIM_LLM_BASE_PATH='http://127.0.0.1:8000' # NVIDIA_NIM_LLM_MODEL_PREF='meta/llama-3.2-3b-instruct' +# LLM_PROVIDER='ppio' +# PPIO_API_KEY='your-ppio-api-key-here' +# PPIO_MODEL_PREF='deepseek/deepseek-v3/community' + ########################################### ######## Embedding API SElECTION ########## ########################################### diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 58087f7b..bd811af1 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -560,6 +560,10 @@ const SystemSettings = { NvidiaNimLLMBasePath: process.env.NVIDIA_NIM_LLM_BASE_PATH, NvidiaNimLLMModelPref: process.env.NVIDIA_NIM_LLM_MODEL_PREF, NvidiaNimLLMTokenLimit: process.env.NVIDIA_NIM_LLM_MODEL_TOKEN_LIMIT, + + // PPIO API keys + PPIOApiKey: !!process.env.PPIO_API_KEY, + PPIOModelPref: process.env.PPIO_MODEL_PREF, }; }, diff --git a/server/storage/models/.gitignore b/server/storage/models/.gitignore index 12436b21..f5c78ac8 100644 --- a/server/storage/models/.gitignore +++ b/server/storage/models/.gitignore @@ -8,4 +8,4 @@ mixedbread-ai* gemini togetherAi tesseract -ppio \ No newline at end of file +ppio diff --git a/server/utils/AiProviders/ppio/index.js b/server/utils/AiProviders/ppio/index.js new file mode 100644 index 00000000..677cd4cd --- /dev/null +++ b/server/utils/AiProviders/ppio/index.js @@ -0,0 +1,266 @@ +const { NativeEmbedder } = require("../../EmbeddingEngines/native"); +const { + handleDefaultStreamResponseV2, +} = require("../../helpers/chat/responses"); +const fs = require("fs"); +const path = require("path"); +const { safeJsonParse } = require("../../http"); +const { + LLMPerformanceMonitor, +} = require("../../helpers/chat/LLMPerformanceMonitor"); +const cacheFolder = path.resolve( + process.env.STORAGE_DIR + ? path.resolve(process.env.STORAGE_DIR, "models", "ppio") + : path.resolve(__dirname, `../../../storage/models/ppio`) +); + +class PPIOLLM { + constructor(embedder = null, modelPreference = null) { + if (!process.env.PPIO_API_KEY) throw new Error("No PPIO API key was set."); + + const { OpenAI: OpenAIApi } = require("openai"); + this.basePath = "https://api.ppinfra.com/v3/openai/"; + this.openai = new OpenAIApi({ + baseURL: this.basePath, + apiKey: process.env.PPIO_API_KEY ?? null, + defaultHeaders: { + "HTTP-Referer": "https://anythingllm.com", + "X-API-Source": "anythingllm", + }, + }); + this.model = + modelPreference || + process.env.PPIO_MODEL_PREF || + "qwen/qwen2.5-32b-instruct"; + this.limits = { + history: this.promptWindowLimit() * 0.15, + system: this.promptWindowLimit() * 0.15, + user: this.promptWindowLimit() * 0.7, + }; + + this.embedder = embedder ?? new NativeEmbedder(); + this.defaultTemp = 0.7; + + if (!fs.existsSync(cacheFolder)) + fs.mkdirSync(cacheFolder, { recursive: true }); + this.cacheModelPath = path.resolve(cacheFolder, "models.json"); + this.cacheAtPath = path.resolve(cacheFolder, ".cached_at"); + + this.log(`Loaded with model: ${this.model}`); + } + + log(text, ...args) { + console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args); + } + + async #syncModels() { + if (fs.existsSync(this.cacheModelPath) && !this.#cacheIsStale()) + return false; + + this.log("Model cache is not present or stale. Fetching from PPIO API."); + await fetchPPIOModels(); + return; + } + + #cacheIsStale() { + const MAX_STALE = 6.048e8; // 1 Week in MS + if (!fs.existsSync(this.cacheAtPath)) return true; + const now = Number(new Date()); + const timestampMs = Number(fs.readFileSync(this.cacheAtPath)); + return now - timestampMs > MAX_STALE; + } + + #appendContext(contextTexts = []) { + if (!contextTexts || !contextTexts.length) return ""; + return ( + "\nContext:\n" + + contextTexts + .map((text, i) => { + return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`; + }) + .join("") + ); + } + + models() { + if (!fs.existsSync(this.cacheModelPath)) return {}; + return safeJsonParse( + fs.readFileSync(this.cacheModelPath, { encoding: "utf-8" }), + {} + ); + } + + streamingEnabled() { + return "streamGetChatCompletion" in this; + } + + promptWindowLimit() { + const model = this.models()[this.model]; + if (!model) return 4096; // Default to 4096 if we cannot find the model + return model?.maxLength || 4096; + } + + async isValidChatCompletionModel(model = "") { + await this.#syncModels(); + const availableModels = this.models(); + return Object.prototype.hasOwnProperty.call(availableModels, model); + } + + /** + * Generates appropriate content array for a message + attachments. + * @param {{userPrompt:string, attachments: import("../../helpers").Attachment[]}} + * @returns {string|object[]} + */ + #generateContent({ userPrompt, attachments = [] }) { + if (!attachments.length) { + return userPrompt; + } + + const content = [{ type: "text", text: userPrompt }]; + for (let attachment of attachments) { + content.push({ + type: "image_url", + image_url: { + url: attachment.contentString, + detail: "auto", + }, + }); + } + return content.flat(); + } + + constructPrompt({ + systemPrompt = "", + contextTexts = [], + chatHistory = [], + userPrompt = "", + // attachments = [], - not supported + }) { + const prompt = { + role: "system", + content: `${systemPrompt}${this.#appendContext(contextTexts)}`, + }; + return [prompt, ...chatHistory, { role: "user", content: userPrompt }]; + } + + async getChatCompletion(messages = null, { temperature = 0.7 }) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `PPIO chat: ${this.model} is not valid for chat completion!` + ); + + const result = await LLMPerformanceMonitor.measureAsyncFunction( + this.openai.chat.completions + .create({ + model: this.model, + messages, + temperature, + }) + .catch((e) => { + throw new Error(e.message); + }) + ); + + if ( + !Object.prototype.hasOwnProperty.call(result.output, "choices") || + result.output.choices.length === 0 + ) + return null; + + return { + textResponse: result.output.choices[0].message.content, + metrics: { + prompt_tokens: result.output.usage.prompt_tokens || 0, + completion_tokens: result.output.usage.completion_tokens || 0, + total_tokens: result.output.usage.total_tokens || 0, + outputTps: result.output.usage.completion_tokens / result.duration, + duration: result.duration, + }, + }; + } + + async streamGetChatCompletion(messages = null, { temperature = 0.7 }) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `PPIO chat: ${this.model} is not valid for chat completion!` + ); + + const measuredStreamRequest = await LLMPerformanceMonitor.measureStream( + this.openai.chat.completions.create({ + model: this.model, + stream: true, + messages, + temperature, + }), + messages + ); + return measuredStreamRequest; + } + + handleStream(response, stream, responseProps) { + return handleDefaultStreamResponseV2(response, stream, responseProps); + } + + async embedTextInput(textInput) { + return await this.embedder.embedTextInput(textInput); + } + async embedChunks(textChunks = []) { + return await this.embedder.embedChunks(textChunks); + } + + async compressMessages(promptArgs = {}, rawHistory = []) { + const { messageArrayCompressor } = require("../../helpers/chat"); + const messageArray = this.constructPrompt(promptArgs); + return await messageArrayCompressor(this, messageArray, rawHistory); + } +} + +async function fetchPPIOModels() { + return await fetch(`https://api.ppinfra.com/v3/openai/models`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${process.env.PPIO_API_KEY}`, + }, + }) + .then((res) => res.json()) + .then(({ data = [] }) => { + const models = {}; + data.forEach((model) => { + const organization = model.id?.split("/")?.[0] || "PPIO"; + models[model.id] = { + id: model.id, + name: model.display_name || model.title || model.id, + organization, + maxLength: model.context_size || 4096, + }; + }); + + if (!fs.existsSync(cacheFolder)) + fs.mkdirSync(cacheFolder, { recursive: true }); + fs.writeFileSync( + path.resolve(cacheFolder, "models.json"), + JSON.stringify(models), + { + encoding: "utf-8", + } + ); + fs.writeFileSync( + path.resolve(cacheFolder, ".cached_at"), + String(Number(new Date())), + { + encoding: "utf-8", + } + ); + return models; + }) + .catch((e) => { + console.error(e); + return {}; + }); +} + +module.exports = { + PPIOLLM, + fetchPPIOModels, +}; diff --git a/server/utils/agents/aibitat/index.js b/server/utils/agents/aibitat/index.js index f5af2e90..83bc736e 100644 --- a/server/utils/agents/aibitat/index.js +++ b/server/utils/agents/aibitat/index.js @@ -795,7 +795,8 @@ ${this.getHistory({ to: route.to }) return new Providers.XAIProvider({ model: config.model }); case "novita": return new Providers.NovitaProvider({ model: config.model }); - + case "ppio": + return new Providers.PPIOProvider({ model: config.model }); default: throw new Error( `Unknown provider: ${config.provider}. Please use a valid provider.` diff --git a/server/utils/agents/aibitat/providers/ai-provider.js b/server/utils/agents/aibitat/providers/ai-provider.js index bd792061..1d7a37fa 100644 --- a/server/utils/agents/aibitat/providers/ai-provider.js +++ b/server/utils/agents/aibitat/providers/ai-provider.js @@ -163,6 +163,14 @@ class Provider { apiKey: process.env.NOVITA_LLM_API_KEY ?? null, ...config, }); + case "ppio": + return new ChatOpenAI({ + configuration: { + baseURL: "https://api.ppinfra.com/v3/openai", + }, + apiKey: process.env.PPIO_API_KEY ?? null, + ...config, + }); // OSS Model Runners // case "anythingllm_ollama": diff --git a/server/utils/agents/aibitat/providers/index.js b/server/utils/agents/aibitat/providers/index.js index e5c01123..3b24ed20 100644 --- a/server/utils/agents/aibitat/providers/index.js +++ b/server/utils/agents/aibitat/providers/index.js @@ -20,6 +20,7 @@ const ApiPieProvider = require("./apipie.js"); const XAIProvider = require("./xai.js"); const NovitaProvider = require("./novita.js"); const NvidiaNimProvider = require("./nvidiaNim.js"); +const PPIOProvider = require("./ppio.js"); module.exports = { OpenAIProvider, @@ -44,4 +45,5 @@ module.exports = { XAIProvider, NovitaProvider, NvidiaNimProvider, + PPIOProvider, }; diff --git a/server/utils/agents/aibitat/providers/ppio.js b/server/utils/agents/aibitat/providers/ppio.js new file mode 100644 index 00000000..2baf895b --- /dev/null +++ b/server/utils/agents/aibitat/providers/ppio.js @@ -0,0 +1,115 @@ +const OpenAI = require("openai"); +const Provider = require("./ai-provider.js"); +const InheritMultiple = require("./helpers/classes.js"); +const UnTooled = require("./helpers/untooled.js"); + +/** + * The agent provider for the PPIO AI provider. + */ +class PPIOProvider extends InheritMultiple([Provider, UnTooled]) { + model; + + constructor(config = {}) { + const { model = "qwen/qwen2.5-32b-instruct" } = config; + super(); + const client = new OpenAI({ + baseURL: "https://api.ppinfra.com/v3/openai", + apiKey: process.env.PPIO_API_KEY, + maxRetries: 3, + defaultHeaders: { + "HTTP-Referer": "https://anythingllm.com", + "X-API-Source": "anythingllm", + }, + }); + + this._client = client; + this.model = model; + this.verbose = true; + } + + get client() { + return this._client; + } + + async #handleFunctionCallChat({ messages = [] }) { + return await this.client.chat.completions + .create({ + model: this.model, + temperature: 0, + messages, + }) + .then((result) => { + if (!Object.prototype.hasOwnProperty.call(result, "choices")) + throw new Error("PPIO chat: No results!"); + if (result.choices.length === 0) + throw new Error("PPIO chat: No results length!"); + return result.choices[0].message.content; + }) + .catch((_) => { + return null; + }); + } + + /** + * Create a completion based on the received messages. + * + * @param messages A list of messages to send to the API. + * @param functions + * @returns The completion. + */ + async complete(messages, functions = null) { + let completion; + if (functions.length > 0) { + const { toolCall, text } = await this.functionCall( + messages, + functions, + this.#handleFunctionCallChat.bind(this) + ); + + if (toolCall !== null) { + this.providerLog(`Valid tool call found - running ${toolCall.name}.`); + this.deduplicator.trackRun(toolCall.name, toolCall.arguments); + return { + result: null, + functionCall: { + name: toolCall.name, + arguments: toolCall.arguments, + }, + cost: 0, + }; + } + completion = { content: text }; + } + + if (!completion?.content) { + this.providerLog("Will assume chat completion without tool call inputs."); + const response = await this.client.chat.completions.create({ + model: this.model, + messages: this.cleanMsgs(messages), + }); + completion = response.choices[0].message; + } + + // The UnTooled class inherited Deduplicator is mostly useful to prevent the agent + // from calling the exact same function over and over in a loop within a single chat exchange + // _but_ we should enable it to call previously used tools in a new chat interaction. + this.deduplicator.reset("runs"); + return { + result: completion.content, + cost: 0, + }; + } + + /** + * Get the cost of the completion. + * + * @param _usage The completion to get the cost for. + * @returns The cost of the completion. + * Stubbed since PPIO has no cost basis. + */ + getCost() { + return 0; + } +} + +module.exports = PPIOProvider; diff --git a/server/utils/agents/index.js b/server/utils/agents/index.js index 531b2866..c06751c6 100644 --- a/server/utils/agents/index.js +++ b/server/utils/agents/index.js @@ -184,6 +184,10 @@ class AgentHandler { "NVIDIA NIM base path must be provided to use agents." ); break; + case "ppio": + if (!process.env.PPIO_API_KEY) + throw new Error("PPIO API Key must be provided to use agents."); + break; default: throw new Error( @@ -249,6 +253,8 @@ class AgentHandler { return process.env.NOVITA_LLM_MODEL_PREF ?? "deepseek/deepseek-r1"; case "nvidia-nim": return process.env.NVIDIA_NIM_LLM_MODEL_PREF ?? null; + case "ppio": + return process.env.PPIO_MODEL_PREF ?? "qwen/qwen2.5-32b-instruct"; default: return null; } diff --git a/server/utils/helpers/customModels.js b/server/utils/helpers/customModels.js index 32cb977d..f0da28bf 100644 --- a/server/utils/helpers/customModels.js +++ b/server/utils/helpers/customModels.js @@ -7,6 +7,7 @@ const { ElevenLabsTTS } = require("../TextToSpeech/elevenLabs"); const { fetchNovitaModels } = require("../AiProviders/novita"); const { parseLMStudioBasePath } = require("../AiProviders/lmStudio"); const { parseNvidiaNimBasePath } = require("../AiProviders/nvidiaNim"); +const { fetchPPIOModels } = require("../AiProviders/ppio"); const { GeminiLLM } = require("../AiProviders/gemini"); const SUPPORT_CUSTOM_MODELS = [ @@ -29,6 +30,7 @@ const SUPPORT_CUSTOM_MODELS = [ "novita", "xai", "gemini", + "ppio", ]; async function getCustomModels(provider = "", apiKey = null, basePath = null) { @@ -74,6 +76,8 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) { return await getNvidiaNimModels(basePath); case "gemini": return await getGeminiModels(apiKey); + case "ppio": + return await getPPIOModels(apiKey); default: return { models: [], error: "Invalid provider for custom models" }; } @@ -571,6 +575,19 @@ async function getGeminiModels(_apiKey = null) { return { models, error: null }; } +async function getPPIOModels() { + const ppioModels = await fetchPPIOModels(); + if (!Object.keys(ppioModels).length === 0) return { models: [], error: null }; + const models = Object.values(ppioModels).map((model) => { + return { + id: model.id, + organization: model.organization, + name: model.name, + }; + }); + return { models, error: null }; +} + module.exports = { getCustomModels, }; diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index 842d48e4..6fc7f00e 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -197,6 +197,9 @@ function getLLMProvider({ provider = null, model = null } = {}) { case "nvidia-nim": const { NvidiaNimLLM } = require("../AiProviders/nvidiaNim"); return new NvidiaNimLLM(embedder, model); + case "ppio": + const { PPIOLLM } = require("../AiProviders/ppio"); + return new PPIOLLM(embedder, model); default: throw new Error( `ENV: No valid LLM_PROVIDER value found in environment! Using ${process.env.LLM_PROVIDER}` @@ -338,6 +341,9 @@ function getLLMProviderClass({ provider = null } = {}) { case "nvidia-nim": const { NvidiaNimLLM } = require("../AiProviders/nvidiaNim"); return NvidiaNimLLM; + case "ppio": + const { PPIOLLM } = require("../AiProviders/ppio"); + return PPIOLLM; default: return null; } diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 41216c96..ab76cb15 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -615,6 +615,16 @@ const KEY_MAPPING = { }, ], }, + + // PPIO Options + PPIOApiKey: { + envKey: "PPIO_API_KEY", + checks: [isNotEmpty], + }, + PPIOModelPref: { + envKey: "PPIO_MODEL_PREF", + checks: [isNotEmpty], + }, }; function isNotEmpty(input = "") { @@ -721,6 +731,7 @@ function supportedLLM(input = "") { "apipie", "xai", "nvidia-nim", + "ppio", ].includes(input); return validSelection ? null : `${input} is not a valid LLM provider.`; }