From c2c8fe97562202ef028d9f57a220e84655502f19 Mon Sep 17 00:00:00 2001 From: Sean Hatfield Date: Wed, 17 Jan 2024 14:42:05 -0800 Subject: [PATCH] add support for mistral api (#610) * add support for mistral api * update docs to show support for Mistral * add default temp to all providers, suggest different results per provider --------- Co-authored-by: timothycarambat --- README.md | 1 + docker/.env.example | 4 + .../LLMSelection/MistralOptions/index.jsx | 103 ++++++++++ .../Modals/MangeWorkspace/Settings/index.jsx | 18 +- frontend/src/media/llmprovider/mistral.jpeg | Bin 0 -> 4542 bytes .../GeneralSettings/LLMPreference/index.jsx | 11 +- .../Steps/DataHandling/index.jsx | 8 + .../Steps/LLMPreference/index.jsx | 9 + server/.env.example | 4 + server/models/systemSettings.js | 12 ++ server/utils/AiProviders/anthropic/index.js | 1 + server/utils/AiProviders/azureOpenAi/index.js | 5 +- server/utils/AiProviders/gemini/index.js | 1 + server/utils/AiProviders/lmStudio/index.js | 5 +- server/utils/AiProviders/localAi/index.js | 5 +- server/utils/AiProviders/mistral/index.js | 184 ++++++++++++++++++ server/utils/AiProviders/native/index.js | 5 +- server/utils/AiProviders/ollama/index.js | 5 +- server/utils/AiProviders/openAi/index.js | 5 +- server/utils/AiProviders/togetherAi/index.js | 5 +- server/utils/chats/index.js | 2 +- server/utils/chats/stream.js | 4 +- server/utils/helpers/customModels.js | 23 +++ server/utils/helpers/index.js | 4 + server/utils/helpers/updateENV.js | 10 + 25 files changed, 412 insertions(+), 22 deletions(-) create mode 100644 frontend/src/components/LLMSelection/MistralOptions/index.jsx create mode 100644 frontend/src/media/llmprovider/mistral.jpeg create mode 100644 server/utils/AiProviders/mistral/index.js diff --git a/README.md b/README.md index 4249c42b..6e3df0df 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Some cool features of AnythingLLM - [LM Studio (all models)](https://lmstudio.ai) - [LocalAi (all models)](https://localai.io/) - [Together AI (chat models)](https://www.together.ai/) +- [Mistral](https://mistral.ai/) **Supported Embedding models:** diff --git a/docker/.env.example b/docker/.env.example index 5bd909af..8d33a809 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -44,6 +44,10 @@ GID='1000' # TOGETHER_AI_API_KEY='my-together-ai-key' # TOGETHER_AI_MODEL_PREF='mistralai/Mixtral-8x7B-Instruct-v0.1' +# LLM_PROVIDER='mistral' +# MISTRAL_API_KEY='example-mistral-ai-api-key' +# MISTRAL_MODEL_PREF='mistral-tiny' + ########################################### ######## Embedding API SElECTION ########## ########################################### diff --git a/frontend/src/components/LLMSelection/MistralOptions/index.jsx b/frontend/src/components/LLMSelection/MistralOptions/index.jsx new file mode 100644 index 00000000..d5c66641 --- /dev/null +++ b/frontend/src/components/LLMSelection/MistralOptions/index.jsx @@ -0,0 +1,103 @@ +import { useState, useEffect } from "react"; +import System from "@/models/system"; + +export default function MistralOptions({ settings }) { + const [inputValue, setInputValue] = useState(settings?.MistralApiKey); + const [mistralKey, setMistralKey] = useState(settings?.MistralApiKey); + + return ( +
+
+ + setInputValue(e.target.value)} + onBlur={() => setMistralKey(inputValue)} + /> +
+ +
+ ); +} + +function MistralModelSelection({ apiKey, settings }) { + const [customModels, setCustomModels] = useState([]); + const [loading, setLoading] = useState(true); + + useEffect(() => { + async function findCustomModels() { + if (!apiKey) { + setCustomModels([]); + setLoading(false); + return; + } + setLoading(true); + const { models } = await System.customModels( + "mistral", + typeof apiKey === "boolean" ? null : apiKey + ); + setCustomModels(models || []); + setLoading(false); + } + findCustomModels(); + }, [apiKey]); + + if (loading || customModels.length == 0) { + return ( +
+ + +
+ ); + } + + return ( +
+ + +
+ ); +} diff --git a/frontend/src/components/Modals/MangeWorkspace/Settings/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Settings/index.jsx index a3089d68..da0e7b9f 100644 --- a/frontend/src/components/Modals/MangeWorkspace/Settings/index.jsx +++ b/frontend/src/components/Modals/MangeWorkspace/Settings/index.jsx @@ -27,11 +27,21 @@ function castToType(key, value) { return definitions[key].cast(value); } +function recommendedSettings(provider = null) { + switch (provider) { + case "mistral": + return { temp: 0 }; + default: + return { temp: 0.7 }; + } +} + export default function WorkspaceSettings({ active, workspace, settings }) { const { slug } = useParams(); const formEl = useRef(null); const [saving, setSaving] = useState(false); const [hasChanges, setHasChanges] = useState(false); + const defaults = recommendedSettings(settings?.LLMProvider); const handleUpdate = async (e) => { setSaving(true); @@ -143,20 +153,20 @@ export default function WorkspaceSettings({ active, workspace, settings }) { This setting controls how "random" or dynamic your chat responses will be.
- The higher the number (2.0 maximum) the more random and + The higher the number (1.0 maximum) the more random and incoherent.
- Recommended: 0.7 + Recommended: {defaults.temp}

e.target.blur()} - defaultValue={workspace?.openAiTemp ?? 0.7} + defaultValue={workspace?.openAiTemp ?? defaults.temp} className="bg-zinc-900 text-white text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5" placeholder="0.7" required={true} diff --git a/frontend/src/media/llmprovider/mistral.jpeg b/frontend/src/media/llmprovider/mistral.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..1019f495d4d690dd639aa9f4e5751c403b2eff27 GIT binary patch literal 4542 zcmb7Hc|4Te+rMWu%*f0z_F}9tmSJp(7$R%7QrXHH8A7DRz9ppW*|Vh3gzPDW?6Rd4 zDoF}ik~Q0VPtViueSV+k{pUS%eLi#D=eo{y&iy^#`#RUzeY-mjU=4KjbpZ$j03hHG z*c}D50T>j@5U{~PL7)%_I2^&k#Dqj)STGng3mVP32ajdlgWH2fV>z)nb`Ao8fMMg} z=H%eUa}YQfK_D=23>?9XKrnN#qFFiq-)*-Uz@dN+paTXG0-!hu3%UU_Zn74>JPE1cyOUklhgg0|UomVOa20s|Ejz{^vi5tB^UVY1V)2 zCr{;yx;W(HxAHu%8vx3Zv`o*HEtI(&Wv#j|UnyaJz^-{00OJJAcD60YeXH|ugf;l8 zZy7%ACvFZS7CdguMa#Ex9Vm%=$2San)9t@c*Y9mqMa+njkD8&BEBO@cXc+(qsJ`$& zUdVkh4n|?4eC%?%n|3ty{N+M%@8XwjeRLk-LsdZfr|UTzR-L?xrU0-p{B=6EY4j{_ z>cP^vjD(uifyQTv0m9Et3JVn%*xj6dSH1tk(@JxtSFPR^X$J9U8V5B2w(Lvw>uR=d z2yvaO{Au^jziMe#7+WY)zS!o~Jteo<7%w9l+p@W&XLJ_cW*kSiG=FdOlIsdL0Qp&F zTQTgTcT`(xOqEa6nwHW|DH!0d$MwFJGVSpXc0X|P`XR6R#w$+ zv>~HUp1mR9v{$lg!gI{sFU*|y=7t|U#uw<+NxH`Dleny+c!2nzJgz-;F7AGHQMYCM z+iv{VA&pmi5p{)YmKRjs#qK%Dz_tJAl`3$` zoN&pr{DRn08(LveaE9nl@;`U+UpgDe3@LE8U`Qksr1js_fx?jh1cqP|#By*dC@K*s zczGO|op{CzWC{u-5Cj3;1xyP|#YK`{ytX``ma&mgo6e`1!6M~8Jllf$iJd+5>1NN3 zCmo|bH$r4@$(~-5418BE`na)n;m*~H@ad<)W5sJO&r@f9eQXS!Z(}Q3*#&&4Y~uwl zpH`lCr7$&JFV-DN_CRQJNcHOV6Jplq-8`Po?o7rA{OGHENSn=6 zo$9TxeCSHy(z@}$ru(wX!;GZEYW3wjzJh0M-VH1Hz0AIF)}~cpgTx3fyz#7GpuF)- z(z_4O=Jj^YFEf3Ytu9>kFU#8NkaP)hpa)S2aj@@AA4I0>D}|)p<~oI=2#`5@5TX3C z{2~=e*c7q6!u)zK>H@=)_WWep>QTl54VKo$H&P#{KQ!z7#55}Nc{b>mKqa1!Z+N?t z)IFX%WmUdfe?&Y&fYxx6zWFVvM~`;?^qR=f5v`2NSmgwmI2%-CF0E?G;a(1|kUgwj z!E9>a$Zb?*9*>?$h-K}G$U<-g4vEFH69mo7y*cFMDGF2z`-HqEPGO%4O)b;k zGy;>624u%cP*pdqUWBjYvDo27ZpF(CjeCN%gSTp8BSY@mE=+3kE!<8!al0!tNgAVsg!;&X^W@9|=>3&WWK@&#_hkp!? zxhUIJ-Q|kr)UJ^db;HE(0_xTp)vX(tlNKYV*0)MLpJwSwNq*DK?2Bc2TvwNEEp^%! zb+Bc%UdkQgNb~cKn%_!p=TuSgmn|l(-qf*n8#)mpHgEhDo!D01E7s0&;GjtxJ!7@q z8vVf&(z06hGW&Dwkd$;u*uxTK&Baei?_bO)XGv4eUYc7$qf3iRK1Wz|?DbY_-zozE zr{Dkp0f93y!4Q8wX9x&54uNDB#Nr7QGw(bOqMW?Bg}qPHfPyAf`!Bem47j%^*Eohw z*ihFJxJOrWp9E?^=Zu1Aeh;;(<8IE~+CqM;+WtJf&gIgdqUf0%IcuKoA+v50C`KNV znJeJ!vv<8!!jt<}>|^DSQ{rju0h#`r$lEh_sUOgXa)P2jr!Zp~bvs1rz&77A{b|Q5 z@y_`x!FHL&Py7>{bB@P+73S9C)?@aNP|>ju6YqDYS+iNQK~ID}65yQq zk*8Oeoc3H1Y1nExV0zK1Su)dw@yxo`kZG*jojG^A`834;mYa3Tdbg}sqm`6#SN*lx zb6@gb{FD-iie>A0@`#1Lzmds>b}CwI)J#0u%$nKS(jwDin$7^Su+X34yQ+ zocW^vcaMCqM4T|n;QI;FG4UZ!_8Dyaim8n1H}ZybES=5sB%ZO>xU zi>G*#t22@xO3=oK?mc+%rSGup(??1%Ewu6FK?Oe}eXk12h093zZsCdUzzHdrQ7ds(~uMc7VVN2jz( zXYnP^>+P$=bCF_BEbQAH$NHsjzZzGGC)Su%7^=opJfZokl)C$LO4f9r(bm^~lj=us zlf6Ho60yD>p?Xy5@S`ef)Wn?B2Vwc|G0y-U`(0pzVU8!pRV~%stqc=YCs(>|i$vQr z58jkRC+BE^z(NC7h)b`!z=fcamz!S}EGH5#pbgYs$e3SdkL2z}g~W&9e;X9>?H@gIaL`I+pfN0GQ}1C2Xv!}meaA9c6i(w5GR!RpKDbToz@ za%71jU?j2aI|__dgc@8qb8p%kw1iX><&zjQBP6E7a__cs3H^c&MRb{vqSrevW+>VD z3>v)`g4go8ygdDy*l>P~uV%Xmvw5GEsa2|)<_)cb6ZPFBMXhafLu8xIe)!`7I%1R_ z=y^zJm@G36_x)G^0)-cuXF_Pxg zb?&aB<{eyv>Llq_ftt9$33U`~x`>(f+@_$)g80^_qt)f@f<1q<#$dvF7P5u!*Ih;P zjACPp_=7xzmr<%8#%Cthpii|4Gyi@J3v7Ah;%A{+oPDgf-mll2+;wR*)wX(;c-!jbGw*QAs z`j!!U#s2bbBJ(kJvQKJQ6*k%*yBva=5UGQ!ab0gCAKW_@o##Dq3Qe_Q>DQe<$cP3= zIbE^tv~qkCQ${9@S*C<_B6Tj7Cd`uznw-Lg>dQ5;x71krNt!v7 z)had<{xqygaP-?V_l%}{b^$KRmhLQYDEN9mec73{F#Bi=x zPIeR3j)-}Fa8{&jQODEPb8YCc1MtY^J5l>?jb!bOtHY1$4^C(q%5|KV`POWxwxt8@ z*gtL>l4+ed9Hh-yaE`W;6 z+DDu4kFMaT;r9oY0|FcR)RuC&V13C;WokdW5B=!dhRp{2>{D&o1=JR48$O8#t*(d#961 z<^3E;;h8Gx2^9ftEAP`)E5$9zS+lmO#1y9g%?c$k zyf~=2@4(n}^UDLCUyTi`nl`7qE=t97T(l6~=}Xs>v(^es>qU<_8=T40e<)_S9b2$B zQ}}tLbqK=p!U9=al7Lc_@Qc8#Wo`C&?$tr2uvq4nfB;DR1^|hG|Ea$J006!})huDa zJ2dZyrfJis-=8Ss6vu9L@n`IdR>-9?RZ=-`n=YYQ@g{e|Yas+^iq$x$0nD|hKIo)j z>dYSPL19%oX2Ek`ZfqL$hUVHA`7f%0^z5X9_DeHftqY$Y`O!k&x5yVALn_}eajO`Q z$rc(POpCqx@A!*R%}8Jw+YjD8z|H+DK|yc;k!)ta_ya3PO~@M%tndzP`rn&}1{S<|^z^$% zB%}CTqw&fP&-G@lb86L%<0AB(n{@=20Bp_Kk%VTilU>|q5Sf)ohEdL5L{(2&Xoa>% z6_c5AWr{5-J#5mB9@Wk2` z;A~6>C<^CxQj>07I6NpP9TCm_C7XRa(cs+u=*^7RO?T%4HI{obP?wa9j7(3AeAn_>w#$rT%uRbT$kkdRXe35 zbUj1`=Xg!6IB^OkB^2<-&7&kVny4W*X;+!jcYU)=crDn=F6nPI(EHqI2)3^fO)?`F znYh1qT7BA2Y31PLd~2g}Wg3=8i^vZ?CVujk2tKB(bMEScoZ-h(`S9rUts%Eg1x;^l zqnO8<+6f0Pn^;{JW&TRn^EhZXD_%YS=&Ic}-N`Ud>J?9$fHb}g-94%@;#h;_Cx)+^ zm&fWhWRw-+`2B2-3Z_h7N-K@*8VI8d*07y>t9~Cfg1UC~nvt=Kefh

GOhkykkrx z5<$gl4%{{1&%57c5Qv$XoY%TcILb6UdoL+VK0U}ckiTqHx51ZU zmwjXW&11_L&eEgCFq@TP2dU4yzzlh^iUi9p_NpSM8p1Mj^7zmjK?2}Op8*u3nr2;h Kf#<=y@BRlu0mfYb literal 0 HcmV?d00001 diff --git a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx index bd6ae511..1efa818d 100644 --- a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx +++ b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx @@ -12,6 +12,7 @@ import OllamaLogo from "@/media/llmprovider/ollama.png"; import LMStudioLogo from "@/media/llmprovider/lmstudio.png"; import LocalAiLogo from "@/media/llmprovider/localai.png"; import TogetherAILogo from "@/media/llmprovider/togetherai.png"; +import MistralLogo from "@/media/llmprovider/mistral.jpeg"; import PreLoader from "@/components/Preloader"; import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions"; @@ -21,9 +22,10 @@ import LocalAiOptions from "@/components/LLMSelection/LocalAiOptions"; import NativeLLMOptions from "@/components/LLMSelection/NativeLLMOptions"; import GeminiLLMOptions from "@/components/LLMSelection/GeminiLLMOptions"; import OllamaLLMOptions from "@/components/LLMSelection/OllamaLLMOptions"; +import TogetherAiOptions from "@/components/LLMSelection/TogetherAiOptions"; +import MistralOptions from "@/components/LLMSelection/MistralOptions"; import LLMItem from "@/components/LLMSelection/LLMItem"; import { MagnifyingGlass } from "@phosphor-icons/react"; -import TogetherAiOptions from "@/components/LLMSelection/TogetherAiOptions"; export default function GeneralLLMPreference() { const [saving, setSaving] = useState(false); @@ -134,6 +136,13 @@ export default function GeneralLLMPreference() { options: , description: "Run open source models from Together AI.", }, + { + name: "Mistral", + value: "mistral", + logo: MistralLogo, + options: , + description: "Run open source models from Mistral AI.", + }, { name: "Native", value: "native", diff --git a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx index 281f1e8c..3b004638 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx @@ -9,6 +9,7 @@ import OllamaLogo from "@/media/llmprovider/ollama.png"; import TogetherAILogo from "@/media/llmprovider/togetherai.png"; import LMStudioLogo from "@/media/llmprovider/lmstudio.png"; import LocalAiLogo from "@/media/llmprovider/localai.png"; +import MistralLogo from "@/media/llmprovider/mistral.jpeg"; import ChromaLogo from "@/media/vectordbs/chroma.png"; import PineconeLogo from "@/media/vectordbs/pinecone.png"; import LanceDbLogo from "@/media/vectordbs/lancedb.png"; @@ -91,6 +92,13 @@ const LLM_SELECTION_PRIVACY = { ], logo: TogetherAILogo, }, + mistral: { + name: "Mistral", + description: [ + "Your prompts and document text used in response creation are visible to Mistral", + ], + logo: MistralLogo, + }, }; const VECTOR_DB_PRIVACY = { diff --git a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx index dc060594..9e8ab84a 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx @@ -9,6 +9,7 @@ import LMStudioLogo from "@/media/llmprovider/lmstudio.png"; import LocalAiLogo from "@/media/llmprovider/localai.png"; import TogetherAILogo from "@/media/llmprovider/togetherai.png"; import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png"; +import MistralLogo from "@/media/llmprovider/mistral.jpeg"; import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions"; import AnthropicAiOptions from "@/components/LLMSelection/AnthropicAiOptions"; @@ -17,6 +18,7 @@ import LocalAiOptions from "@/components/LLMSelection/LocalAiOptions"; import NativeLLMOptions from "@/components/LLMSelection/NativeLLMOptions"; import GeminiLLMOptions from "@/components/LLMSelection/GeminiLLMOptions"; import OllamaLLMOptions from "@/components/LLMSelection/OllamaLLMOptions"; +import MistralOptions from "@/components/LLMSelection/MistralOptions"; import LLMItem from "@/components/LLMSelection/LLMItem"; import System from "@/models/system"; import paths from "@/utils/paths"; @@ -109,6 +111,13 @@ export default function LLMPreference({ options: , description: "Run open source models from Together AI.", }, + { + name: "Mistral", + value: "mistral", + logo: MistralLogo, + options: , + description: "Run open source models from Mistral AI.", + }, { name: "Native", value: "native", diff --git a/server/.env.example b/server/.env.example index d060e0ab..26c51927 100644 --- a/server/.env.example +++ b/server/.env.example @@ -41,6 +41,10 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea # TOGETHER_AI_API_KEY='my-together-ai-key' # TOGETHER_AI_MODEL_PREF='mistralai/Mixtral-8x7B-Instruct-v0.1' +# LLM_PROVIDER='mistral' +# MISTRAL_API_KEY='example-mistral-ai-api-key' +# MISTRAL_MODEL_PREF='mistral-tiny' + ########################################### ######## Embedding API SElECTION ########## ########################################### diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index cd008d42..53d42f2e 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -159,6 +159,18 @@ const SystemSettings = { AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF, } : {}), + ...(llmProvider === "mistral" + ? { + MistralApiKey: !!process.env.MISTRAL_API_KEY, + MistralModelPref: process.env.MISTRAL_MODEL_PREF, + + // For embedding credentials when mistral is selected. + OpenAiKey: !!process.env.OPEN_AI_KEY, + AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT, + AzureOpenAiKey: !!process.env.AZURE_OPENAI_KEY, + AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF, + } + : {}), ...(llmProvider === "native" ? { NativeLLMModelPref: process.env.NATIVE_LLM_MODEL_PREF, diff --git a/server/utils/AiProviders/anthropic/index.js b/server/utils/AiProviders/anthropic/index.js index 17f2abc4..56d3a80f 100644 --- a/server/utils/AiProviders/anthropic/index.js +++ b/server/utils/AiProviders/anthropic/index.js @@ -26,6 +26,7 @@ class AnthropicLLM { ); this.embedder = embedder; this.answerKey = v4().split("-")[0]; + this.defaultTemp = 0.7; } streamingEnabled() { diff --git a/server/utils/AiProviders/azureOpenAi/index.js b/server/utils/AiProviders/azureOpenAi/index.js index f59fc51f..639ac102 100644 --- a/server/utils/AiProviders/azureOpenAi/index.js +++ b/server/utils/AiProviders/azureOpenAi/index.js @@ -25,6 +25,7 @@ class AzureOpenAiLLM { "No embedding provider defined for AzureOpenAiLLM - falling back to AzureOpenAiEmbedder for embedding!" ); this.embedder = !embedder ? new AzureOpenAiEmbedder() : embedder; + this.defaultTemp = 0.7; } #appendContext(contextTexts = []) { @@ -93,7 +94,7 @@ class AzureOpenAiLLM { ); const textResponse = await this.openai .getChatCompletions(this.model, messages, { - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, }) .then((res) => { @@ -130,7 +131,7 @@ class AzureOpenAiLLM { this.model, messages, { - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, } ); diff --git a/server/utils/AiProviders/gemini/index.js b/server/utils/AiProviders/gemini/index.js index 348c8f5e..63549fb8 100644 --- a/server/utils/AiProviders/gemini/index.js +++ b/server/utils/AiProviders/gemini/index.js @@ -22,6 +22,7 @@ class GeminiLLM { "INVALID GEMINI LLM SETUP. No embedding engine has been set. Go to instance settings and set up an embedding interface to use Gemini as your LLM." ); this.embedder = embedder; + this.defaultTemp = 0.7; // not used for Gemini } #appendContext(contextTexts = []) { diff --git a/server/utils/AiProviders/lmStudio/index.js b/server/utils/AiProviders/lmStudio/index.js index 61480803..08950a7b 100644 --- a/server/utils/AiProviders/lmStudio/index.js +++ b/server/utils/AiProviders/lmStudio/index.js @@ -25,6 +25,7 @@ class LMStudioLLM { "INVALID LM STUDIO SETUP. No embedding engine has been set. Go to instance settings and set up an embedding interface to use LMStudio as your LLM." ); this.embedder = embedder; + this.defaultTemp = 0.7; } #appendContext(contextTexts = []) { @@ -85,7 +86,7 @@ class LMStudioLLM { const textResponse = await this.lmstudio .createChatCompletion({ model: this.model, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, messages: await this.compressMessages( { @@ -122,7 +123,7 @@ class LMStudioLLM { const streamRequest = await this.lmstudio.createChatCompletion( { model: this.model, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, stream: true, messages: await this.compressMessages( diff --git a/server/utils/AiProviders/localAi/index.js b/server/utils/AiProviders/localAi/index.js index 6623ac88..6d265cf8 100644 --- a/server/utils/AiProviders/localAi/index.js +++ b/server/utils/AiProviders/localAi/index.js @@ -27,6 +27,7 @@ class LocalAiLLM { "INVALID LOCAL AI SETUP. No embedding engine has been set. Go to instance settings and set up an embedding interface to use LocalAI as your LLM." ); this.embedder = embedder; + this.defaultTemp = 0.7; } #appendContext(contextTexts = []) { @@ -85,7 +86,7 @@ class LocalAiLLM { const textResponse = await this.openai .createChatCompletion({ model: this.model, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, messages: await this.compressMessages( { @@ -123,7 +124,7 @@ class LocalAiLLM { { model: this.model, stream: true, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, messages: await this.compressMessages( { diff --git a/server/utils/AiProviders/mistral/index.js b/server/utils/AiProviders/mistral/index.js new file mode 100644 index 00000000..a25185c7 --- /dev/null +++ b/server/utils/AiProviders/mistral/index.js @@ -0,0 +1,184 @@ +const { chatPrompt } = require("../../chats"); + +class MistralLLM { + constructor(embedder = null, modelPreference = null) { + const { Configuration, OpenAIApi } = require("openai"); + if (!process.env.MISTRAL_API_KEY) + throw new Error("No Mistral API key was set."); + + const config = new Configuration({ + basePath: "https://api.mistral.ai/v1", + apiKey: process.env.MISTRAL_API_KEY, + }); + this.openai = new OpenAIApi(config); + this.model = + modelPreference || process.env.MISTRAL_MODEL_PREF || "mistral-tiny"; + this.limits = { + history: this.promptWindowLimit() * 0.15, + system: this.promptWindowLimit() * 0.15, + user: this.promptWindowLimit() * 0.7, + }; + + if (!embedder) + console.warn( + "No embedding provider defined for MistralLLM - falling back to OpenAiEmbedder for embedding!" + ); + this.embedder = embedder; + this.defaultTemp = 0.0; + } + + #appendContext(contextTexts = []) { + if (!contextTexts || !contextTexts.length) return ""; + return ( + "\nContext:\n" + + contextTexts + .map((text, i) => { + return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`; + }) + .join("") + ); + } + + streamingEnabled() { + return "streamChat" in this && "streamGetChatCompletion" in this; + } + + promptWindowLimit() { + return 32000; + } + + async isValidChatCompletionModel(modelName = "") { + return true; + } + + constructPrompt({ + systemPrompt = "", + contextTexts = [], + chatHistory = [], + userPrompt = "", + }) { + const prompt = { + role: "system", + content: `${systemPrompt}${this.#appendContext(contextTexts)}`, + }; + return [prompt, ...chatHistory, { role: "user", content: userPrompt }]; + } + + async isSafe(_ = "") { + return { safe: true, reasons: [] }; + } + + async sendChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `Mistral chat: ${this.model} is not valid for chat completion!` + ); + + const textResponse = await this.openai + .createChatCompletion({ + model: this.model, + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), + messages: await this.compressMessages( + { + systemPrompt: chatPrompt(workspace), + userPrompt: prompt, + chatHistory, + }, + rawHistory + ), + }) + .then((json) => { + const res = json.data; + if (!res.hasOwnProperty("choices")) + throw new Error("Mistral chat: No results!"); + if (res.choices.length === 0) + throw new Error("Mistral chat: No results length!"); + return res.choices[0].message.content; + }) + .catch((error) => { + throw new Error( + `Mistral::createChatCompletion failed with: ${error.message}` + ); + }); + + return textResponse; + } + + async streamChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `Mistral chat: ${this.model} is not valid for chat completion!` + ); + + const streamRequest = await this.openai.createChatCompletion( + { + model: this.model, + stream: true, + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), + messages: await this.compressMessages( + { + systemPrompt: chatPrompt(workspace), + userPrompt: prompt, + chatHistory, + }, + rawHistory + ), + }, + { responseType: "stream" } + ); + + return streamRequest; + } + + async getChatCompletion(messages = null, { temperature = 0.7 }) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `Mistral chat: ${this.model} is not valid for chat completion!` + ); + + const { data } = await this.openai.createChatCompletion({ + model: this.model, + messages, + temperature, + }); + + if (!data.hasOwnProperty("choices")) return null; + return data.choices[0].message.content; + } + + async streamGetChatCompletion(messages = null, { temperature = 0.7 }) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `Mistral chat: ${this.model} is not valid for chat completion!` + ); + + const streamRequest = await this.openai.createChatCompletion( + { + model: this.model, + stream: true, + messages, + temperature, + }, + { responseType: "stream" } + ); + return streamRequest; + } + + // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations + async embedTextInput(textInput) { + return await this.embedder.embedTextInput(textInput); + } + async embedChunks(textChunks = []) { + return await this.embedder.embedChunks(textChunks); + } + + async compressMessages(promptArgs = {}, rawHistory = []) { + const { messageArrayCompressor } = require("../../helpers/chat"); + const messageArray = this.constructPrompt(promptArgs); + return await messageArrayCompressor(this, messageArray, rawHistory); + } +} + +module.exports = { + MistralLLM, +}; diff --git a/server/utils/AiProviders/native/index.js b/server/utils/AiProviders/native/index.js index 66cc84d0..fff904c4 100644 --- a/server/utils/AiProviders/native/index.js +++ b/server/utils/AiProviders/native/index.js @@ -29,6 +29,7 @@ class NativeLLM { // Make directory when it does not exist in existing installations if (!fs.existsSync(this.cacheDir)) fs.mkdirSync(this.cacheDir); + this.defaultTemp = 0.7; } async #initializeLlamaModel(temperature = 0.7) { @@ -132,7 +133,7 @@ class NativeLLM { ); const model = await this.#llamaClient({ - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), }); const response = await model.call(messages); return response.content; @@ -145,7 +146,7 @@ class NativeLLM { async streamChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) { const model = await this.#llamaClient({ - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), }); const messages = await this.compressMessages( { diff --git a/server/utils/AiProviders/ollama/index.js b/server/utils/AiProviders/ollama/index.js index fce96f36..af7fe821 100644 --- a/server/utils/AiProviders/ollama/index.js +++ b/server/utils/AiProviders/ollama/index.js @@ -20,6 +20,7 @@ class OllamaAILLM { "INVALID OLLAMA SETUP. No embedding engine has been set. Go to instance settings and set up an embedding interface to use Ollama as your LLM." ); this.embedder = embedder; + this.defaultTemp = 0.7; } #ollamaClient({ temperature = 0.07 }) { @@ -113,7 +114,7 @@ class OllamaAILLM { ); const model = this.#ollamaClient({ - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), }); const textResponse = await model .pipe(new StringOutputParser()) @@ -136,7 +137,7 @@ class OllamaAILLM { ); const model = this.#ollamaClient({ - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), }); const stream = await model .pipe(new StringOutputParser()) diff --git a/server/utils/AiProviders/openAi/index.js b/server/utils/AiProviders/openAi/index.js index 038d201d..582bc054 100644 --- a/server/utils/AiProviders/openAi/index.js +++ b/server/utils/AiProviders/openAi/index.js @@ -23,6 +23,7 @@ class OpenAiLLM { "No embedding provider defined for OpenAiLLM - falling back to OpenAiEmbedder for embedding!" ); this.embedder = !embedder ? new OpenAiEmbedder() : embedder; + this.defaultTemp = 0.7; } #appendContext(contextTexts = []) { @@ -127,7 +128,7 @@ class OpenAiLLM { const textResponse = await this.openai .createChatCompletion({ model: this.model, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, messages: await this.compressMessages( { @@ -165,7 +166,7 @@ class OpenAiLLM { { model: this.model, stream: true, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, messages: await this.compressMessages( { diff --git a/server/utils/AiProviders/togetherAi/index.js b/server/utils/AiProviders/togetherAi/index.js index 44061dd0..341661f8 100644 --- a/server/utils/AiProviders/togetherAi/index.js +++ b/server/utils/AiProviders/togetherAi/index.js @@ -28,6 +28,7 @@ class TogetherAiLLM { "INVALID TOGETHER AI SETUP. No embedding engine has been set. Go to instance settings and set up an embedding interface to use Together AI as your LLM." ); this.embedder = embedder; + this.defaultTemp = 0.7; } #appendContext(contextTexts = []) { @@ -89,7 +90,7 @@ class TogetherAiLLM { const textResponse = await this.openai .createChatCompletion({ model: this.model, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, messages: await this.compressMessages( { @@ -127,7 +128,7 @@ class TogetherAiLLM { { model: this.model, stream: true, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, messages: await this.compressMessages( { diff --git a/server/utils/chats/index.js b/server/utils/chats/index.js index d63de47d..764c7795 100644 --- a/server/utils/chats/index.js +++ b/server/utils/chats/index.js @@ -171,7 +171,7 @@ async function chatWithWorkspace( // Send the text completion. const textResponse = await LLMConnector.getChatCompletion(messages, { - temperature: workspace?.openAiTemp ?? 0.7, + temperature: workspace?.openAiTemp ?? LLMConnector.defaultTemp, }); if (!textResponse) { diff --git a/server/utils/chats/stream.js b/server/utils/chats/stream.js index ceea8d7d..cff565ed 100644 --- a/server/utils/chats/stream.js +++ b/server/utils/chats/stream.js @@ -141,7 +141,7 @@ async function streamChatWithWorkspace( `\x1b[31m[STREAMING DISABLED]\x1b[0m Streaming is not available for ${LLMConnector.constructor.name}. Will use regular chat method.` ); completeText = await LLMConnector.getChatCompletion(messages, { - temperature: workspace?.openAiTemp ?? 0.7, + temperature: workspace?.openAiTemp ?? LLMConnector.defaultTemp, }); writeResponseChunk(response, { uuid, @@ -153,7 +153,7 @@ async function streamChatWithWorkspace( }); } else { const stream = await LLMConnector.streamGetChatCompletion(messages, { - temperature: workspace?.openAiTemp ?? 0.7, + temperature: workspace?.openAiTemp ?? LLMConnector.defaultTemp, }); completeText = await handleStreamResponses(response, stream, { uuid, diff --git a/server/utils/helpers/customModels.js b/server/utils/helpers/customModels.js index 87fe976e..53c641e7 100644 --- a/server/utils/helpers/customModels.js +++ b/server/utils/helpers/customModels.js @@ -5,6 +5,7 @@ const SUPPORT_CUSTOM_MODELS = [ "ollama", "native-llm", "togetherai", + "mistral", ]; async function getCustomModels(provider = "", apiKey = null, basePath = null) { @@ -20,6 +21,8 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) { return await ollamaAIModels(basePath); case "togetherai": return await getTogetherAiModels(); + case "mistral": + return await getMistralModels(apiKey); case "native-llm": return nativeLLMModels(); default: @@ -117,6 +120,26 @@ async function getTogetherAiModels() { return { models, error: null }; } +async function getMistralModels(apiKey = null) { + const { Configuration, OpenAIApi } = require("openai"); + const config = new Configuration({ + apiKey: apiKey || process.env.MISTRAL_API_KEY, + basePath: "https://api.mistral.ai/v1", + }); + const openai = new OpenAIApi(config); + const models = await openai + .listModels() + .then((res) => res.data.data.filter((model) => !model.id.includes("embed"))) + .catch((e) => { + console.error(`Mistral:listModels`, e.message); + return []; + }); + + // Api Key was successful so lets save it for future uses + if (models.length > 0 && !!apiKey) process.env.MISTRAL_API_KEY = apiKey; + return { models, error: null }; +} + function nativeLLMModels() { const fs = require("fs"); const path = require("path"); diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index 2b1f3dac..2eed9057 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -52,6 +52,9 @@ function getLLMProvider(modelPreference = null) { case "togetherai": const { TogetherAiLLM } = require("../AiProviders/togetherAi"); return new TogetherAiLLM(embedder, modelPreference); + case "mistral": + const { MistralLLM } = require("../AiProviders/mistral"); + return new MistralLLM(embedder, modelPreference); case "native": const { NativeLLM } = require("../AiProviders/native"); return new NativeLLM(embedder, modelPreference); @@ -76,6 +79,7 @@ function getEmbeddingEngineSelection() { return new LocalAiEmbedder(); case "native": const { NativeEmbedder } = require("../EmbeddingEngines/native"); + console.log("\x1b[34m[INFO]\x1b[0m Using Native Embedder"); return new NativeEmbedder(); default: return null; diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 5c43da51..54e68402 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -95,6 +95,15 @@ const KEY_MAPPING = { checks: [nonZero], }, + MistralApiKey: { + envKey: "MISTRAL_API_KEY", + checks: [isNotEmpty], + }, + MistralModelPref: { + envKey: "MISTRAL_MODEL_PREF", + checks: [isNotEmpty], + }, + // Native LLM Settings NativeLLMModelPref: { envKey: "NATIVE_LLM_MODEL_PREF", @@ -268,6 +277,7 @@ function supportedLLM(input = "") { "ollama", "native", "togetherai", + "mistral", ].includes(input); return validSelection ? null : `${input} is not a valid LLM provider.`; }