From b8dd7bc97e1f7094fd934a25a144de3815bf4823 Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Thu, 29 Jan 2026 12:01:11 -0800 Subject: [PATCH] Support PrivateModeAI Integration (#4937) * Support PrivateModeAI Integration * tooltip for proxy --- docker/.env.example | 4 + .../LLMSelection/PrivateModeOptions/index.jsx | 129 +++++++++++ .../components/ProviderPrivacy/constants.js | 6 + .../src/media/llmprovider/privatemode.png | Bin 0 -> 9454 bytes .../GeneralSettings/LLMPreference/index.jsx | 10 + .../Steps/LLMPreference/index.jsx | 9 + .../AgentConfig/AgentLLMSelection/index.jsx | 1 + server/.env.example | 4 + server/endpoints/utils.js | 3 + server/models/systemSettings.js | 4 + server/utils/AiProviders/privatemode/index.js | 218 ++++++++++++++++++ server/utils/agents/aibitat/index.js | 2 + .../agents/aibitat/providers/ai-provider.js | 8 + .../utils/agents/aibitat/providers/index.js | 2 + .../agents/aibitat/providers/privatemode.js | 98 ++++++++ server/utils/agents/index.js | 8 + server/utils/helpers/customModels.js | 51 ++++ server/utils/helpers/index.js | 8 + server/utils/helpers/updateENV.js | 11 + 19 files changed, 576 insertions(+) create mode 100644 frontend/src/components/LLMSelection/PrivateModeOptions/index.jsx create mode 100644 frontend/src/media/llmprovider/privatemode.png create mode 100644 server/utils/AiProviders/privatemode/index.js create mode 100644 server/utils/agents/aibitat/providers/privatemode.js diff --git a/docker/.env.example b/docker/.env.example index b17517e5..e20fa82e 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -167,6 +167,10 @@ GID='1000' # DOCKER_MODEL_RUNNER_LLM_MODEL_PREF='phi-3.5-mini' # DOCKER_MODEL_RUNNER_LLM_MODEL_TOKEN_LIMIT=4096 +# LLM_PROVIDER='privatemode' +# PRIVATEMODE_LLM_BASE_PATH='http://127.0.0.1:8080' +# PRIVATEMODE_LLM_MODEL_PREF='gemma-3-27b' + ########################################### ######## Embedding API SElECTION ########## ########################################### diff --git a/frontend/src/components/LLMSelection/PrivateModeOptions/index.jsx b/frontend/src/components/LLMSelection/PrivateModeOptions/index.jsx new file mode 100644 index 00000000..5238e03b --- /dev/null +++ b/frontend/src/components/LLMSelection/PrivateModeOptions/index.jsx @@ -0,0 +1,129 @@ +import { useEffect, useState } from "react"; +import { Info } from "@phosphor-icons/react"; +import { Tooltip } from "react-tooltip"; +import System from "@/models/system"; +import { Link } from "react-router-dom"; + +export default function PrivateModeOptions({ settings }) { + const [models, setModels] = useState([]); + const [loading, setLoading] = useState(!!settings?.PrivateModeBasePath); + const [basePath, setBasePath] = useState(settings?.PrivateModeBasePath); + const [model, setModel] = useState(settings?.PrivateModeModelPref || ""); + + useEffect(() => { + setModel(settings?.PrivateModeModelPref || ""); + }, [settings?.PrivateModeModelPref]); + + useEffect(() => { + async function fetchModels() { + try { + setLoading(true); + if (!basePath) throw new Error("Base path is required"); + const { models, error } = await System.customModels( + "privatemode", + null, + basePath + ); + if (error) throw new Error(error); + setModels(models); + } catch (error) { + console.error("Error fetching Private Mode models:", error); + setModels([]); + } finally { + setLoading(false); + } + } + fetchModels(); + }, [basePath]); + + return ( +
+
+
+
+ + + + Enter the URL where Privatemode Proxy is running. +
+
+ + Learn more → + + +
+ setBasePath(e.target.value)} + /> +
+
+ + {loading ? ( + + ) : ( + + )} +
+
+
+ ); +} diff --git a/frontend/src/components/ProviderPrivacy/constants.js b/frontend/src/components/ProviderPrivacy/constants.js index 2e6a32d0..1d99fff7 100644 --- a/frontend/src/components/ProviderPrivacy/constants.js +++ b/frontend/src/components/ProviderPrivacy/constants.js @@ -42,6 +42,7 @@ import CometApiLogo from "@/media/llmprovider/cometapi.png"; import FoundryLogo from "@/media/llmprovider/foundry-local.png"; import GiteeAILogo from "@/media/llmprovider/giteeai.png"; import DockerModelRunnerLogo from "@/media/llmprovider/docker-model-runner.png"; +import PrivateModeLogo from "@/media/llmprovider/privatemode.png"; const LLM_PROVIDER_PRIVACY_MAP = { openai: { @@ -232,6 +233,11 @@ const LLM_PROVIDER_PRIVACY_MAP = { ], logo: DockerModelRunnerLogo, }, + privatemode: { + name: "Privatemode", + policyUrl: "https://docs.privatemode.ai/getting-started/faq#q2", + logo: PrivateModeLogo, + }, }; const VECTOR_DB_PROVIDER_PRIVACY_MAP = { diff --git a/frontend/src/media/llmprovider/privatemode.png b/frontend/src/media/llmprovider/privatemode.png new file mode 100644 index 0000000000000000000000000000000000000000..e4bcf382ea0a5dec8b242198d560b3ae28dec038 GIT binary patch literal 9454 zcmd6NcQl+`*EiBcf)qiNkgE4ST6BpJEfU>ebTf<@!w3?BM8Bg36A|thL?mj2i6DAs zM6@AF#E2F~FW(utzx#RaXT5)X>wVX|)_0a!*LAM5uYK)v&Mv>bk0*u(I*be)3^X(} zj5l;OjcI6TZ&H8sbl}OV-b3(!=D3%xxepBu(`V|BHc5zylZNJqwX5kp)II%M5C?>( zDAW;Q4-@tC^a9j0G)k&|UQh>j7>d^(=Ijbr=3A_);p266ROY)Yqc5)Sr2%ts)x{!V zCRhVg2duk;f+L@*3WJg#1TgS~p`g5eo*r-?h@Udw0bdBXr#=?r14KwiCy248_Fp8R zr_ASqLU}>N#4s3)C`L*Yfpiv=P*6}16PFZ|loSCJB0m0b6x2@y?sFM%VBkIAp$YSG zK)QONToG_yDo?0A0*zAU<743ci?tdGg><$5n>*Y`6ngLkO?^caOj-=;=p!Z}Doz!P zmsjaGJw(mXhuZ!95fhmALFaeR!}Z`xUQnbD%+%itrp#vy^Fg4I4zL3b)R+EZqJe}# zQ7}gpNpVS85pf9-Nd+ab{}`fv<6kdmLQznafNS1p1WM5n;qdzg^82ExrtR(NFC*a% zzw`Tn@b3%S2ZsCy;T<(iIeS@SP0hPn{~)wSLg9}8pj3nVqg-6!&bMyq^P0k-p1%hJ z{}%EHw97OEpB^*SgVNx&=1xI-&5h*zdNe3B81t*y7zoz>8 zm4C9)wg+5fBxNLJWfUZ&CFB%jiX^n#*X?LA<;BD`KmS6`q&o(MCvvfh(Ze2N(bollp67pa)Syqg)V36;oGFUJWP`>S+%} zDN)J(?*OpPgZ2M!tub&Oe=qVsf~(cn|DB`9EzyN$fv4|Ch7*Jwf2XfG-pK+v9=G-#!xvqRfW` z9ub$(gazKz0Cw+&lfFKUAh@TeIZVqza|GPcf}aX4=im3*wEQ%O4xR(YO?URMdn(oM zW(O|R-oX`fWNe0p=ECR=O*K=$jQLUjd!_@sN0&~tM>o5L>l@W7UwS{zdiXRZkwb=E zF5f;|tHDm1MZf=RSfkQVV;Wx7$ibNS!uSu4NmdyxSk#Hg)=Pr)kA;du>&5GL_Qdti z7I+_BAdCetFV4g%-TrlDrn=T&0kh!Se)8y{V>H5&xlA-P>S_eq)G+KJ@e`DzcC2K^ zPr^jb2j+M=ni4|V_lHU^V#f#AWz1^8kTy|%#6_|U5O5j$h zYu`~u4o#dE@^b-3zY@BS^$c>#mhbcCa`x=fH-Ci<6n|5@bG>K)NPd5NZC)pq>-5&i zXB*hHpXXUwSvffJ%?k2Lbi;HhL8e6ZDs9VU-JfD~PEI35to#wT61yp?XVw=+hbw(% zV!0I~qoVRk=-YaGk*$dWP$;xMoH4so{dP2VtdkgguCTD^EcSWIFWhT(Ra;lmrP~!a z)!@Cw;Qj4~hmK8A`af)LQRXU8v%Tk~oJ!sLBOxcw*Us(Y@!rFWWEq(dgI{hAGlF!l z1fFO-*3Ip^dhHtg>)VUU$~ef0tIp0d4N+`jR`0t92cJPs&_d^|t>KT&&BXR3HczB_ z_q|LEj3^0Cd^0cygx#J|-QWD)kxn$vsRKK-JkeleWMp7qKs499YDd-ePUh0E!3mt+ zHjLqFg`I}bz#CznY*is&I9wEcN}%}NyC;tyKaP$@d3n9)ymW?kF3_()cK2Sn|Bd}gIlqurCgujR z29pL%_O(S>KG4@=%olPxFG*FvN7KZibzk2(to-_gFQM0s$7T)|2?g^$>~j_KzVb>f zHwmtE`6?gZ`us3veyEH>p%Bfd$Wtu|yIYj);jz6&lD-s%p*8pB`l4oyi&!TP#_otU9i>q-39E@+0Qr z6+XTIEEYag`UFC6@^ODJ0D)Mf%ng_v`*yyI0{56QX1=j%xqe$&P;w-3Sk4P)P=TCo zeei&$Mdhu4yG5aIL*%*o2hK zzI`hmqpgAavvT7#CiF{uf@nH+uGGpPjo9Z_Hng;~WV^pLAR|M>%vXpj`u<#6SrOvr zf5Fr%eE&<8e1%c&w6{|2v~9xbLpT)r4VWre_M6-}VXe*0OKWS%lu;><1Xx2M*`f^20!sf)+_Z$+ZwPfa!&23 z#l@0j-cR0K6;2&#-QC^cN6%ZG!7^o1f{?c(?`*>2qOPtB%}&OtK3DL0#ie@dve(+`mk2Q>qlpYR3|S_I5Uf?!L)kq7(BX69sH-$DTetL*~6U zUP!WtQH#CnD8!@pCkZV2*6J*9+f=$F98THe>s!kTxY02QRs`jX-j?i2id%f+Bs&_7 zKFeotOIP;=6J1e!X6EAJVrD{u2KQ(F>V8YUuA)dv*=hmi&Yk}E-jfCy@@m|NqkoOp z9tsZLkB&Y`<}GTR!5|basXn}O_bvzxpFjE1iL@s0-PVq~fJ;i!;+{I^S5Tq;98vD- zslRD|t3Db2OQwY{kc);!0>62tCH! z7=8ZoG+(>L7Fqw^B8KFDY^HZ=0l4;Tow($mjJiO;!PfSn=S8)jHOlUIp-`)DtgCci z9Ss_N@9pqws-^e-Rn|-%__4d21oe@;L$#R2_wV1!xPN)9Dn>swQt9I})A^^tGTnXF zVr#Z%1XU|oOPRmoO1Ay3Ph$$A8k@dNXRoOXXD%r+OEt+QoV4yBXX;H(Q7$*VRdyCm~qdmLl#YO zJhoN7B4~!#Ndn2&O^TW4e1@F~{(VrkCZrqx<_**F<2|!eo4VZ6Ej)Wm5sNkZ&mmzn zRf*B50-=*DwUg({i=Lp;c$Ji_7Q{vlo6MpizGt8+RxL@qVb79IN-JWoKEFLJY3`#NmM`C9tSl!SMDZ}yb!RyAY%32@X?d& zX}*gjTxzN=`}KINB3hhhtz472huf-y?p0Kx$fZj!^Yd?JR9;WJbm8AsdM2Jcd8Lyu{t(&xO+$$7Lin(B14R<(CD&=h=^)Z{_(mSdU`9%%cp~FXuBRNRHfHc>Z;q;D*8W*qdzUsl_ulf z?YYqta^!^Wn`nlB$be7x8t~>%H_njCqXu8g2zp=OZRzYJmAWRdOFg`0MZltm%RMK$ zgvM((<6=GBb(J;!O(H@I%)gFV43?grwffZj$L0Oq#o#C@w&pO4!jJwduPh2#eT*fN zOJ0cosXF>(wcJ{B-dA0n?gQg~!Ryzzb})XA-(gjKMAa}8QAgi7ZL9H#t(d!;7J6!u zeFFmn<1Q#iMn-;q#x&JXS2qC^%Erd#Wd}cL9=2z~Qe$3F?63cNv;BRR3CrSrVVeqt z*x`?XiXRxovIpk0Y(6zu5;F5drFQ0|JQw_-6_-melIdJoESui&80|g~$hOxPqZ9<1 zJClU&ZmrD=bz0dEY(E=W*}6Q;(R%nRu1~#%kD18}laqmZ&_zwJiV*<7fDhYQ`lhj*d+poL zpIq$h?59qR0|YlaJ6NiAQ%|q2ug}ET7`P_y&0nyfvo{C9RJPU^#p*O2DM5|h&C~ab zQ~~0gLpmgcR)6{8vE&y;`;ft1v3S@{b$69IapMlrG~fB5j#LSUp;FiCu~0O?TpI_C zNHh)7Nq*6iz!$ZGKkiKkHr;!%Og0Y;a&mzBX8*DN?gL74u6N+&4DSkj_lYfU5Ag4V zfc3gM^&$2;qde1#7ccsJZ$C~?e`;&G7(hs{A`%i39+^4S-jl#ZDnELzB}m!(<4vt0 zcEE72oBC(z+U@o}9IKK2+@yTMw6I;cuu`_u$e8bb)b3eW zh+4fD7Zhym@4wUT&&k0742$49Z(&xG84(ut#EU0=iY%f7z(8AjdyYY^RMJffKry1W zA4eJ*!URJNa-*W6wwLOe2Q6Ff2;TAbUiRVR_2-)%$ZxL8Gs*orw+5g@NlA(7=G56b zxDQvZA^-RGXFijSF>xwec=X(tM>qU(``xts#AypKK7`%1VPVhJ*@=l_FjPUpMnRfa z7#JA7cjAD1Femd?yAHgu_V#`k7WTk7_AOSf4Z!G{y)Ab#OZ79AK&+Wr^5LouZ!3Uy zyD}732Awk{N}|VVg7<&)WQQ0$#LDS;cnEWHhUb~)fBpJ3aC=QbUf$4o^Xlo-r_Y~1 ze<{O2{~^705)X6Cq&5i2rF$Mn=xVruG-nHe4K&-XxWrna`$xice+sn?1j zY+_-wstz~)i9Ct|dBRE5@bK{VL_?G$S+>dBt!xWSeyZp^Tm_&R_W{vwjXH#MQ5BWv z5OtpzE(J*ig~5)F7|7$6#GBIqdJYuX-saA!Tw5Fy8yN=@Tq4+vy>rkpSLKmKPBkMw zZfY+$=kIL|)PU8`fM}JBRQs2Jc&r{B-Cy+t#5BG+I5Rh7b?I9@auIwU;8!ZD5J=N{ zUw@rg^~8~ojsRK-RK^Vk)8c-_Q*gH^Haa@Tpa)eKnC#_6(ge8|7=3Bb!vd`B_Kh3f zNaRbVx{|>X=QfO@+{4(ecUZXwpzV%Bo{EZ9mvv&_4tT~+`YwzpI5wXmv*@FKewDZw zu0=?n0ND=lbec=ob7jCO*We<3(oOXEC)%O(sq|YfZnKa_S=7h$_~QX$mK(dD0110w zWo;eyk@*%cFm(!LXK^45oBsOc%S%)sAJ(Fzqy*AH9h|zCnwnY^tH?_OXz_;+kc;p- z4T64hVj}S?|Ic(Ae`h%SqtA@NQPI%z4x=B?AlJz;fJ$EUt+@`U0AS#*bp+`^@}euB z$>7ise6TnS@P4#m^{pqMD|R@6WFQi5TaKKm7Ag z*qu8`#w|0~%I^{ML1qlh<~puU+WpIGSJ%<+@(T&KRkC93IAXMYW_zy&HE9y`eZJKl zc^Z?J{#qk~38~aiybsJ80JLjphCTp>R}^{4o@pyHT_!r>9T7Z1c;h_0N%zJi&%Q2q ztu#=_9j3?F^lL&wQIL@185tSs8X9>f@v}YIO*uCbfS}_;W$p(g;p?h2Y9bb;_&Wt= zL;?O5_qkM{w}obHlMm#5;2c6i1-Z8~!4x7ZEB!(G6~V-f4?PvIOBtYM1Saf4Qc@+i zGga=#g!d00_V@Qk&H~*5>-+;;fONUBc633^%tnZ_Q&UrhI^wmj%}el?bwjGxA|}@7 z1MwBdfqmjHv$r)iI!%0jLKL`?1?-BJmX;_G-LR;g2nZHNDmA!k?>RI^yI#bka%yOI zwJcXXdO^At-$UkRXV>LcQ+HhL8^;`1Bk1#HW8*kvA|WA<{QUio6WI7{7f6J*$ubGz z@ZpAr2IA@H-f~aNhc>T;Y(IYd2)cKSjq3&s@fGh-5Nj>1BWQ+f;B^gk;xpt(exZ<% zch8mle~-7ewgNE=FpDJO=_?UU0a}m3v1j>3ME;Dm6DDp4`(v@jI^wKk{Rd4vAG+@a zvbp)Zuc~UBUJD?mU-$P{Kp%F&$H z17<8S+VwynId?8@?!3#oiB>_9?-%s1i{9=W8Xh*$iRUk~(C4n@$IDeGo=UfVOG^n) zd8+3rC?s@1Y;Z$bT6#%_h3qL@X|)c@1`_DQPm15ZeG$tmOB}xU{rj&C^A>pMm7XO( zZ3{!CJ&vbMI%W)0zp8jBLC=6q0O5@IS?)820r9j8Dy&qRE&(7c^qO>d${4|9TVp%X zldS`mEu=r4rY4~3cs?Hg7!tzLGHq?4v>+G?-V$ZL;`M8jD@s^ZH4_qIG2m_plMu)j;r4`WPF>2&yVTf`&;wYa@g#P7xCecKZiqYii1o%+g36sg$zS z$Rc?_e1Ec|4hOE3AtkR@xHlsl@5BgH>|105a9&WX?eq*W-~G- z25cLhn3&iF9Sy>>I~Epc09fKeK>cg5%$*I*aCWIGoq^m@419u-4sUtDI{ZB0g-w>> zD!D(ZiPi{_4ASSeAHGbaS;`g&TSmF!wR1%^or?*2c)3ZHtsj@k^p3!(2Ls?jX1Vrb zpopjD&`VOZB6ih9Tl*-Qfunf4Ulw@KpktbZ!3z5Z1~UC&dj|&)HS{3oA7ad-b zM4AynE9umvlckPaPlOJ{XJq;gLzF5q565TxVD;FE%0NGTaPl- zv)JX}8K*91lSGW}-TOM*mz!s@^s2N}nx9|yqor8P1?99RO!__u8fQSk>vo3xA2GT5 zf`Wq7)NT&P39{ph-OWHMkUq-!{=8RN^o{v~($e~R3MF7P>E<*D!ACf_DDX-*clXec zkiE^WVB=%1FRQDSK(%6SpvdHyYx0X1ya53L50B~)ETs$06g`JZ898MQk4c!jx#eeO zW?K0k**Q|;)NuyxMdWsWs^kzSX$7xdGpQh>q@!4dEtk+8F>=Hc=~iza=|!Tbh7aARkCg zUYNWfEmfB|cL7CH1xXi!T%=3c5SRsO?u#eLF=Aq3h~67l>O@L`<9T3HH1+}Q4xrRP zp=EB53*P?9l`DQ&thTncnVFd}?36A5P`S6BU0M|e)TU@OQmdX(s%DbNiIq!?m|plTvtnr>)wOjbyC3MlWtF@f{JB}4f1Zy>({T3GGF*PIVt5e zejT7U0L?gqk5_=rRa8`zl9Cb;5xII*OMCmUOr}MlC_n$(l9IDf(buGIO;Eu&$TDtp z?V077`hudfbJuf#hRXM$4h|sJ1Mln(R3Sef1QGbi`#;jtg|Zc={MM_+b`3J%K6tsM zKeMtf^6}+{aaqef%g%Pt)O_U9_pM#wfnTziE!@H3@(>Cy_ds`KWaP8e2L&SMb!kpn zk5e;`Hse4YUm}`aLjBP&@p&lr5PfG|ojoX)Pjp3r4jdRZ0ZTa(Yy)1o2kPPg&|NXf zCehRlm3yXxsMA6OY#$t$0MnJm>jc_I6F+`{GK+Q{4Hz&4Uc}nUivG|G5={ueTJ#JI zFFPJ!4=ojefDcqZ7eL}eOC3W)5K&QssqNdg^pvCHgDJK_v$L}|a3PaGwjeESZBS`W zf;2FL9tlWl51fUnPxB3W^Tt8YO=L>ULkFTfBoKeS3G#DZtys|8F>o112;Z3Sp z$c){Z0Ilt{+3tbS6fgsQ3Cx1k(}CP)hbc$J&Hel~fS@;U6a7H0zMZ}Gnz3To2@wA6 z;v{I!zP7DV5fqGsguMf!+5AbZU8?){J2MoBYPZ1rtJ4UCU6A^Nvi1$`&(e6gPeLHp z^C)8z9eF~W>hfeB*h)G!$FHuU6nxV)2(&@9AfbQy^eI?$J?@+*US3Q~bb?84ZOI~3 zfCQ?5fWry|v%tm11`Y-^0EVZMNCBh@YV{n4M!&zs9@@bJ6!`AlyW*-X$_dc-;%eGm ztEeFa?Lzbp2;Ns#1}-!NC)(!f%U<2;sIIQA5p$VJD*uNt_)o z)vN)g;^ifdj=T#ForHvirT4~)ii%W0=3OxswEeR!->Y_`7nJip9kh22P35kA;gHyz zEK;L=8A zLO}>$?F+sK0=}W47g%+^+t0x{9FRUSIXO8cWq^3lzI*|Nx`8iW%H8@~8yf-6SR1sR zpP%n{D+48AFE7J_eh^a7V4-|{Uv1esaFQAI9i5#ADiF5iVmWQxZvy^T&MSc>{|`?* z|9|A5r^bC<7Q}^v^sYKBH1rVqa6E%lXzg_zEzRMFp+5hslip_*BD2mFO8CoyLvNZJ MS_YbL)a^q44>l0ZLjV8( literal 0 HcmV?d00001 diff --git a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx index 7eb71f03..dd4c58ce 100644 --- a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx +++ b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx @@ -38,6 +38,7 @@ import CometApiLogo from "@/media/llmprovider/cometapi.png"; import FoundryLogo from "@/media/llmprovider/foundry-local.png"; import GiteeAILogo from "@/media/llmprovider/giteeai.png"; import DockerModelRunnerLogo from "@/media/llmprovider/docker-model-runner.png"; +import PrivateModeLogo from "@/media/llmprovider/privatemode.png"; import PreLoader from "@/components/Preloader"; import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; @@ -73,6 +74,7 @@ import MoonshotAiOptions from "@/components/LLMSelection/MoonshotAiOptions"; import FoundryOptions from "@/components/LLMSelection/FoundryOptions"; import GiteeAIOptions from "@/components/LLMSelection/GiteeAIOptions/index.jsx"; import DockerModelRunnerOptions from "@/components/LLMSelection/DockerModelRunnerOptions"; +import PrivateModeOptions from "@/components/LLMSelection/PrivateModeOptions"; import LLMItem from "@/components/LLMSelection/LLMItem"; import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react"; @@ -316,6 +318,14 @@ export const AVAILABLE_LLM_PROVIDERS = [ description: "Run Moonshot AI's powerful LLMs.", requiredConfig: ["MoonshotAiApiKey"], }, + { + name: "Privatemode", + value: "privatemode", + logo: PrivateModeLogo, + options: (settings) => , + description: "Run LLMs with end-to-end encryption.", + requiredConfig: ["PrivateModeBasePath"], + }, { name: "Novita AI", value: "novita", diff --git a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx index f56bb6bd..a9b58e4e 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx @@ -32,6 +32,7 @@ import MoonshotAiLogo from "@/media/llmprovider/moonshotai.png"; import CometApiLogo from "@/media/llmprovider/cometapi.png"; import GiteeAILogo from "@/media/llmprovider/giteeai.png"; import DockerModelRunnerLogo from "@/media/llmprovider/docker-model-runner.png"; +import PrivateModeLogo from "@/media/llmprovider/privatemode.png"; import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions"; @@ -65,6 +66,7 @@ import MoonshotAiOptions from "@/components/LLMSelection/MoonshotAiOptions"; import CometApiLLMOptions from "@/components/LLMSelection/CometApiLLMOptions"; import GiteeAiOptions from "@/components/LLMSelection/GiteeAIOptions"; import DockerModelRunnerOptions from "@/components/LLMSelection/DockerModelRunnerOptions"; +import PrivateModeOptions from "@/components/LLMSelection/PrivateModeOptions"; import LLMItem from "@/components/LLMSelection/LLMItem"; import System from "@/models/system"; @@ -273,6 +275,13 @@ const LLMS = [ options: (settings) => , description: "Run powerful foundation models privately with AWS Bedrock.", }, + { + name: "Privatemode", + value: "privatemode", + logo: PrivateModeLogo, + options: (settings) => , + description: "Run LLMs with end-to-end encryption.", + }, { name: "xAI", value: "xai", diff --git a/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx b/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx index b2a5945b..0ac04a3c 100644 --- a/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx +++ b/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx @@ -38,6 +38,7 @@ const ENABLED_PROVIDERS = [ "giteeai", "cohere", "docker-model-runner", + "privatemode", // TODO: More agent support. // "huggingface" // Can be done but already has issues with no-chat templated. Needs to be tested. ]; diff --git a/server/.env.example b/server/.env.example index b408b6fa..6523d100 100644 --- a/server/.env.example +++ b/server/.env.example @@ -166,6 +166,10 @@ SIG_SALT='salt' # Please generate random string at least 32 chars long. # DOCKER_MODEL_RUNNER_LLM_MODEL_PREF='phi-3.5-mini' # DOCKER_MODEL_RUNNER_LLM_MODEL_TOKEN_LIMIT=4096 +# LLM_PROVIDER='privatemode' +# PRIVATEMODE_LLM_BASE_PATH='http://127.0.0.1:8080' +# PRIVATEMODE_LLM_MODEL_PREF='gemma-3-27b' + ########################################### ######## Embedding API SElECTION ########## ########################################### diff --git a/server/endpoints/utils.js b/server/endpoints/utils.js index aa449709..61a61d6a 100644 --- a/server/endpoints/utils.js +++ b/server/endpoints/utils.js @@ -162,6 +162,9 @@ function getModelTag() { case "docker-model-runner": model = process.env.DOCKER_MODEL_RUNNER_LLM_MODEL_PREF; break; + case "privatemode": + model = process.env.PRIVATEMODE_LLM_MODEL_PREF; + break; default: model = "--"; break; diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 9cc89a5d..ce9df3dc 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -674,6 +674,10 @@ const SystemSettings = { process.env.DOCKER_MODEL_RUNNER_LLM_MODEL_PREF, DockerModelRunnerModelTokenLimit: process.env.DOCKER_MODEL_RUNNER_LLM_MODEL_TOKEN_LIMIT || 8192, + + // Privatemode Keys + PrivateModeBasePath: process.env.PRIVATEMODE_LLM_BASE_PATH, + PrivateModeModelPref: process.env.PRIVATEMODE_LLM_MODEL_PREF, }; }, diff --git a/server/utils/AiProviders/privatemode/index.js b/server/utils/AiProviders/privatemode/index.js new file mode 100644 index 00000000..47e8f80e --- /dev/null +++ b/server/utils/AiProviders/privatemode/index.js @@ -0,0 +1,218 @@ +const { NativeEmbedder } = require("../../EmbeddingEngines/native"); +const { + handleDefaultStreamResponseV2, + formatChatHistory, +} = require("../../helpers/chat/responses"); +const { + LLMPerformanceMonitor, +} = require("../../helpers/chat/LLMPerformanceMonitor"); + +class PrivatemodeLLM { + static contextWindows = { + "leon-se/gemma-3-27b-it-fp8-dynamic": 128000, + "gemma-3-27b": 128000, + "qwen3-coder-30b-a3b": 128000, + "gpt-oss-120b": 128000, + "openai/gpt-oss-120b": 128000, + }; + + constructor(embedder = null, modelPreference = null) { + if (!process.env.PRIVATEMODE_LLM_BASE_PATH) + throw new Error("No Privatemode Base Path was set."); + + this.className = "PrivatemodeLLM"; + const { OpenAI: OpenAIApi } = require("openai"); + this.client = new OpenAIApi({ + baseURL: PrivatemodeLLM.parseBasePath(), + apiKey: null, + }); + + this.model = modelPreference || process.env.PRIVATEMODE_LLM_MODEL_PREF; + this.limits = { + history: this.promptWindowLimit() * 0.15, + system: this.promptWindowLimit() * 0.15, + user: this.promptWindowLimit() * 0.7, + }; + + this.embedder = embedder ?? new NativeEmbedder(); + this.defaultTemp = 0.7; + this.log( + `Privatemode LLM initialized with ${this.model}. ctx: ${this.promptWindowLimit()}` + ); + } + + /** + * Parse the base path for the Privatemode API + * so we can use it for inference requests + * @param {string} providedBasePath + * @returns {string} + */ + static parseBasePath( + providedBasePath = process.env.PRIVATEMODE_LLM_BASE_PATH + ) { + try { + const baseURL = new URL(providedBasePath); + const basePath = `${baseURL.origin}/v1`; + return basePath; + } catch (e) { + return null; + } + } + + log(text, ...args) { + console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args); + } + + #appendContext(contextTexts = []) { + if (!contextTexts || !contextTexts.length) return ""; + return ( + "\nContext:\n" + + contextTexts + .map((text, i) => { + return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`; + }) + .join("") + ); + } + + streamingEnabled() { + return "streamGetChatCompletion" in this; + } + + static promptWindowLimit(_modelName) { + const limit = PrivatemodeLLM.contextWindows[_modelName] || 16384; + return Number(limit); + } + + promptWindowLimit() { + const limit = PrivatemodeLLM.contextWindows[this.model] || 16384; + return Number(limit); + } + + async isValidChatCompletionModel(_ = "") { + return true; + } + + /** + * Generates appropriate content array for a message + attachments. + * @param {{userPrompt:string, attachments: import("../../helpers").Attachment[]}} + * @returns {string|object[]} + */ + #generateContent({ userPrompt, attachments = [] }) { + if (!attachments.length) return userPrompt; + + const content = [{ type: "text", text: userPrompt }]; + for (let attachment of attachments) { + content.push({ + type: "image_url", + image_url: { + url: attachment.contentString, + detail: "auto", + }, + }); + } + return content.flat(); + } + + /** + * Construct the user prompt for this model. + * @param {{attachments: import("../../helpers").Attachment[]}} param0 + * @returns + */ + constructPrompt({ + systemPrompt = "", + contextTexts = [], + chatHistory = [], + userPrompt = "", + attachments = [], + }) { + const prompt = { + role: "system", + content: `${systemPrompt}${this.#appendContext(contextTexts)}`, + }; + return [ + prompt, + ...formatChatHistory(chatHistory, this.#generateContent), + { + role: "user", + content: this.#generateContent({ userPrompt, attachments }), + }, + ]; + } + + async getChatCompletion(messages = null, { temperature = 0.7 }) { + if (!this.model) + throw new Error( + `Privatemode chat: ${this.model} is not valid or defined model for chat completion!` + ); + + const result = await LLMPerformanceMonitor.measureAsyncFunction( + this.client.chat.completions.create({ + model: this.model, + messages, + temperature, + }) + ); + + if ( + !result.output.hasOwnProperty("choices") || + result.output.choices.length === 0 + ) + return null; + + return { + textResponse: result.output.choices[0].message.content, + metrics: { + prompt_tokens: result.output.usage?.prompt_tokens || 0, + completion_tokens: result.output.usage?.completion_tokens || 0, + total_tokens: result.output.usage?.total_tokens || 0, + outputTps: result.output.usage?.completion_tokens / result.duration, + duration: result.duration, + model: this.model, + timestamp: new Date(), + }, + }; + } + + async streamGetChatCompletion(messages = null, { temperature = 0.7 }) { + if (!this.model) + throw new Error( + `Privatemode chat: ${this.model} is not valid or defined model for chat completion!` + ); + + const measuredStreamRequest = await LLMPerformanceMonitor.measureStream({ + func: this.client.chat.completions.create({ + model: this.model, + stream: true, + messages, + temperature, + }), + messages, + runPromptTokenCalculation: true, + modelTag: this.model, + }); + return measuredStreamRequest; + } + + handleStream(response, stream, responseProps) { + return handleDefaultStreamResponseV2(response, stream, responseProps); + } + + // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations + async embedTextInput(textInput) { + return await this.embedder.embedTextInput(textInput); + } + async embedChunks(textChunks = []) { + return await this.embedder.embedChunks(textChunks); + } + + async compressMessages(promptArgs = {}, rawHistory = []) { + const { messageArrayCompressor } = require("../../helpers/chat"); + const messageArray = this.constructPrompt(promptArgs); + return await messageArrayCompressor(this, messageArray, rawHistory); + } +} + +module.exports = { + PrivatemodeLLM, +}; diff --git a/server/utils/agents/aibitat/index.js b/server/utils/agents/aibitat/index.js index 0a2f6f45..86e1a214 100644 --- a/server/utils/agents/aibitat/index.js +++ b/server/utils/agents/aibitat/index.js @@ -994,6 +994,8 @@ ${this.getHistory({ to: route.to }) return new Providers.CohereProvider({ model: config.model }); case "docker-model-runner": return new Providers.DockerModelRunnerProvider({ model: config.model }); + case "privatemode": + return new Providers.PrivatemodeProvider({ model: config.model }); default: throw new Error( `Unknown provider: ${config.provider}. Please use a valid provider.` diff --git a/server/utils/agents/aibitat/providers/ai-provider.js b/server/utils/agents/aibitat/providers/ai-provider.js index d92a5ad8..d1da6bae 100644 --- a/server/utils/agents/aibitat/providers/ai-provider.js +++ b/server/utils/agents/aibitat/providers/ai-provider.js @@ -249,6 +249,14 @@ class Provider { apiKey: process.env.COHERE_API_KEY ?? null, ...config, }); + case "privatemode": + return new ChatOpenAI({ + configuration: { + baseURL: process.env.PRIVATEMODE_LLM_BASE_PATH, + }, + apiKey: null, + ...config, + }); // OSS Model Runners // case "anythingllm_ollama": // return new ChatOllama({ diff --git a/server/utils/agents/aibitat/providers/index.js b/server/utils/agents/aibitat/providers/index.js index c53c01c3..1862dc66 100644 --- a/server/utils/agents/aibitat/providers/index.js +++ b/server/utils/agents/aibitat/providers/index.js @@ -30,6 +30,7 @@ const FoundryProvider = require("./foundry.js"); const GiteeAIProvider = require("./giteeai.js"); const CohereProvider = require("./cohere.js"); const DockerModelRunnerProvider = require("./dockerModelRunner.js"); +const PrivatemodeProvider = require("./privatemode.js"); module.exports = { OpenAIProvider, @@ -64,4 +65,5 @@ module.exports = { GiteeAIProvider, CohereProvider, DockerModelRunnerProvider, + PrivatemodeProvider, }; diff --git a/server/utils/agents/aibitat/providers/privatemode.js b/server/utils/agents/aibitat/providers/privatemode.js new file mode 100644 index 00000000..eadb1af6 --- /dev/null +++ b/server/utils/agents/aibitat/providers/privatemode.js @@ -0,0 +1,98 @@ +const OpenAI = require("openai"); +const Provider = require("./ai-provider.js"); +const InheritMultiple = require("./helpers/classes.js"); +const UnTooled = require("./helpers/untooled.js"); +const { PrivatemodeLLM } = require("../../../AiProviders/privatemode/index.js"); + +/** + * The agent provider for the Privatemodel provider. + * @extends {Provider} + * @extends {UnTooled} + */ +class PrivatemodelProvider extends InheritMultiple([Provider, UnTooled]) { + model; + + constructor(config = {}) { + const { model = process.env.PRIVATEMODE_LLM_MODEL_PREF } = config; + super(); + const client = new OpenAI({ + baseURL: PrivatemodeLLM.parseBasePath( + process.env.PRIVATEMODE_LLM_BASE_PATH + ), + apiKey: null, + maxRetries: 3, + }); + + this._client = client; + this.model = model; + this.verbose = true; + } + + get client() { + return this._client; + } + + get supportsAgentStreaming() { + return true; + } + + async #handleFunctionCallChat({ messages = [] }) { + return await this.client.chat.completions + .create({ + model: this.model, + messages, + user: this.executingUserId, + }) + .then((result) => { + if (!result.hasOwnProperty("choices")) + throw new Error("Privatemodel chat: No results!"); + if (result.choices.length === 0) + throw new Error("Privatemodel chat: No results length!"); + return result.choices[0].message.content; + }) + .catch((_) => { + return null; + }); + } + + async #handleFunctionCallStream({ messages = [] }) { + return await this.client.chat.completions.create({ + model: this.model, + stream: true, + messages, + user: this.executingUserId, + }); + } + + async stream(messages, functions = [], eventHandler = null) { + return await UnTooled.prototype.stream.call( + this, + messages, + functions, + this.#handleFunctionCallStream.bind(this), + eventHandler + ); + } + + async complete(messages, functions = []) { + return await UnTooled.prototype.complete.call( + this, + messages, + functions, + this.#handleFunctionCallChat.bind(this) + ); + } + + /** + * Get the cost of the completion. + * + * @param _usage The completion to get the cost for. + * @returns The cost of the completion. + * Stubbed since Privatemodel has no cost basis. + */ + getCost(_usage) { + return 0; + } +} + +module.exports = PrivatemodelProvider; diff --git a/server/utils/agents/index.js b/server/utils/agents/index.js index 24c496b6..582d5495 100644 --- a/server/utils/agents/index.js +++ b/server/utils/agents/index.js @@ -223,6 +223,12 @@ class AgentHandler { "Docker Model Runner base path must be provided to use agents." ); break; + case "privatemode": + if (!process.env.PRIVATEMODE_LLM_BASE_PATH) + throw new Error( + "Privatemode base path must be provided to use agents." + ); + break; default: throw new Error( "No workspace agent provider set. Please set your agent provider in the workspace's settings" @@ -305,6 +311,8 @@ class AgentHandler { return process.env.COHERE_MODEL_PREF ?? "command-r-08-2024"; case "docker-model-runner": return process.env.DOCKER_MODEL_RUNNER_LLM_MODEL_PREF ?? null; + case "privatemode": + return process.env.PRIVATEMODE_LLM_MODEL_PREF ?? null; default: return null; } diff --git a/server/utils/helpers/customModels.js b/server/utils/helpers/customModels.js index 649d2e56..974099af 100644 --- a/server/utils/helpers/customModels.js +++ b/server/utils/helpers/customModels.js @@ -45,6 +45,7 @@ const SUPPORT_CUSTOM_MODELS = [ "zai", "giteeai", "docker-model-runner", + "privatemode", // Embedding Engines "native-embedder", "cohere-embedder", @@ -120,6 +121,8 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) { return await getGiteeAIModels(apiKey); case "docker-model-runner": return await getDockerModelRunnerModels(basePath); + case "privatemode": + return await getPrivatemodeModels(basePath, "generate"); default: return { models: [], error: "Invalid provider for custom models" }; } @@ -881,6 +884,54 @@ async function getDockerModelRunnerModels(basePath = null) { } } +/** + * Get Privatemode models + * @param {string} basePath - The base path of the Privatemode endpoint. + * @param {'any' | 'generate' | 'embed' | 'transcribe'} task - The task to fetch the models for. + * @returns {Promise<{models: Array<{id: string, organization: string, name: string}>, error: string | null}>} + */ +async function getPrivatemodeModels(basePath = null, task = "any") { + try { + const { PrivatemodeLLM } = require("../AiProviders/privatemode"); + const { OpenAI: OpenAIApi } = require("openai"); + const openai = new OpenAIApi({ + baseURL: PrivatemodeLLM.parseBasePath( + basePath || process.env.PRIVATEMODE_LLM_BASE_PATH + ), + apiKey: null, + }); + const models = await openai.models + .list() + .then((results) => results.data) + .then( + (models) => + models + .filter((model) => !model.id.includes("/")) // remove legacy prefixed models + .filter((model) => + task === "any" ? true : model.tasks.includes(task) + ) // filter by task or show all if task is any + ) + .then((models) => + models.map((model) => ({ + id: model.id, + organization: "Privatemode", + name: model.id + .split("-") + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(" "), + })) + ) + .catch((e) => { + console.error(`Privatemode:listModels`, e.message); + return []; + }); + return { models, error: null }; + } catch (e) { + console.error(`Privatemode:getPrivatemodeModels`, e.message); + return { models: [], error: "Could not fetch Privatemode Models" }; + } +} + module.exports = { getCustomModels, SUPPORT_CUSTOM_MODELS, diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index d508f7ee..38f82306 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -234,6 +234,9 @@ function getLLMProvider({ provider = null, model = null } = {}) { DockerModelRunnerLLM, } = require("../AiProviders/dockerModelRunner"); return new DockerModelRunnerLLM(embedder, model); + case "privatemode": + const { PrivatemodeLLM } = require("../AiProviders/privatemode"); + return new PrivatemodeLLM(embedder, model); default: throw new Error( `ENV: No valid LLM_PROVIDER value found in environment! Using ${process.env.LLM_PROVIDER}` @@ -404,6 +407,9 @@ function getLLMProviderClass({ provider = null } = {}) { DockerModelRunnerLLM, } = require("../AiProviders/dockerModelRunner"); return DockerModelRunnerLLM; + case "privatemode": + const { PrivateModeLLM } = require("../AiProviders/privatemode"); + return PrivateModeLLM; default: return null; } @@ -482,6 +488,8 @@ function getBaseLLMProviderModel({ provider = null } = {}) { return process.env.GITEE_AI_MODEL_PREF; case "docker-model-runner": return process.env.DOCKER_MODEL_RUNNER_LLM_MODEL_PREF; + case "privatemode": + return process.env.PRIVATEMODE_LLM_MODEL_PREF; default: return null; } diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 5e4cffbe..fdac6018 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -799,6 +799,16 @@ const KEY_MAPPING = { envKey: "DOCKER_MODEL_RUNNER_LLM_MODEL_TOKEN_LIMIT", checks: [nonZero], }, + + // Privatemode Options + PrivateModeBasePath: { + envKey: "PRIVATEMODE_LLM_BASE_PATH", + checks: [isValidURL], + }, + PrivateModeModelPref: { + envKey: "PRIVATEMODE_LLM_MODEL_PREF", + checks: [isNotEmpty], + }, }; function isNotEmpty(input = "") { @@ -913,6 +923,7 @@ function supportedLLM(input = "") { "zai", "giteeai", "docker-model-runner", + "privatemode", ].includes(input); return validSelection ? null : `${input} is not a valid LLM provider.`; }