Merge branch 'master' of github.com:Mintplex-Labs/anything-llm
This commit is contained in:
timothycarambat
commit
d800f8a073
@ -1 +1,6 @@
|
||||
# Placeholder .env file for collector runtime
|
||||
|
||||
# This enables HTTP request/response logging in development. Set value to truthy string to enable, leave empty value or comment out to disable
|
||||
# ENABLE_HTTP_LOGGER=""
|
||||
# This enables timestamps for the HTTP Logger. Set value to true to enable, leave empty or comment out to disable
|
||||
# ENABLE_HTTP_LOGGER_TIMESTAMPS=""
|
||||
3
collector/.gitignore
vendored
3
collector/.gitignore
vendored
@ -4,3 +4,6 @@ yarn-error.log
|
||||
!yarn.lock
|
||||
outputs
|
||||
scripts
|
||||
.env.development
|
||||
.env.production
|
||||
.env.test
|
||||
|
||||
@ -1,16 +1,32 @@
|
||||
const { YoutubeTranscript } = require("../../../../../utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js");
|
||||
|
||||
describe("YoutubeTranscript", () => {
|
||||
it("should fetch transcript from YouTube video", async () => {
|
||||
const videoId = "BJjsfNO5JTo";
|
||||
const transcript = await YoutubeTranscript.fetchTranscript(videoId, {
|
||||
lang: "en",
|
||||
});
|
||||
if (process.env.GITHUB_ACTIONS) {
|
||||
console.log("Skipping YoutubeTranscript test in GitHub Actions as the URLs will not resolve.");
|
||||
it('is stubbed in GitHub Actions', () => expect(true).toBe(true));
|
||||
} else {
|
||||
it("should fetch transcript from YouTube video", async () => {
|
||||
const videoId = "BJjsfNO5JTo";
|
||||
const transcript = await YoutubeTranscript.fetchTranscript(videoId, {
|
||||
lang: "en",
|
||||
});
|
||||
|
||||
expect(transcript).toBeDefined();
|
||||
expect(typeof transcript).toBe("string");
|
||||
expect(transcript.length).toBeGreaterThan(0);
|
||||
// console.log("Success! Transcript length:", transcript.length);
|
||||
// console.log("First 200 characters:", transcript.substring(0, 200) + "...");
|
||||
}, 30000);
|
||||
expect(transcript).toBeDefined();
|
||||
expect(typeof transcript).toBe("string");
|
||||
expect(transcript.length).toBeGreaterThan(0);
|
||||
console.log("First 200 characters:", transcript.substring(0, 200) + "...");
|
||||
}, 30000);
|
||||
|
||||
it("should fetch non asr transcript from YouTube video", async () => {
|
||||
const videoId = "D111ao6wWH0";
|
||||
const transcript = await YoutubeTranscript.fetchTranscript(videoId, {
|
||||
lang: "zh-HK",
|
||||
});
|
||||
|
||||
expect(transcript).toBeDefined();
|
||||
expect(typeof transcript).toBe("string");
|
||||
expect(transcript.length).toBeGreaterThan(0);
|
||||
console.log("First 200 characters:", transcript.substring(0, 200) + "...");
|
||||
}, 30000);
|
||||
}
|
||||
});
|
||||
|
||||
@ -15,9 +15,21 @@ const { wipeCollectorStorage } = require("./utils/files");
|
||||
const extensions = require("./extensions");
|
||||
const { processRawText } = require("./processRawText");
|
||||
const { verifyPayloadIntegrity } = require("./middleware/verifyIntegrity");
|
||||
const { httpLogger } = require("./middleware/httpLogger");
|
||||
const app = express();
|
||||
const FILE_LIMIT = "3GB";
|
||||
|
||||
// Only log HTTP requests in development mode and if the ENABLE_HTTP_LOGGER environment variable is set to true
|
||||
if (
|
||||
process.env.NODE_ENV === "development" &&
|
||||
!!process.env.ENABLE_HTTP_LOGGER
|
||||
) {
|
||||
app.use(
|
||||
httpLogger({
|
||||
enableTimestamps: !!process.env.ENABLE_HTTP_LOGGER_TIMESTAMPS,
|
||||
})
|
||||
);
|
||||
}
|
||||
app.use(cors({ origin: true }));
|
||||
app.use(
|
||||
bodyParser.text({ limit: FILE_LIMIT }),
|
||||
|
||||
29
collector/middleware/httpLogger.js
Normal file
29
collector/middleware/httpLogger.js
Normal file
@ -0,0 +1,29 @@
|
||||
const httpLogger =
|
||||
({ enableTimestamps = false }) =>
|
||||
(req, res, next) => {
|
||||
// Capture the original res.end to log response status
|
||||
const originalEnd = res.end;
|
||||
|
||||
res.end = function (chunk, encoding) {
|
||||
// Log the request method, status code, and path
|
||||
const statusColor = res.statusCode >= 400 ? "\x1b[31m" : "\x1b[32m"; // Red for errors, green for success
|
||||
console.log(
|
||||
`\x1b[32m[HTTP]\x1b[0m ${statusColor}${res.statusCode}\x1b[0m ${
|
||||
req.method
|
||||
} -> ${req.path} ${
|
||||
enableTimestamps
|
||||
? `@ ${new Date().toLocaleTimeString("en-US", { hour12: true })}`
|
||||
: ""
|
||||
}`.trim()
|
||||
);
|
||||
|
||||
// Call the original end method
|
||||
return originalEnd.call(this, chunk, encoding);
|
||||
};
|
||||
|
||||
next();
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
httpLogger,
|
||||
};
|
||||
@ -85,6 +85,85 @@ class YoutubeTranscript {
|
||||
.replace(/\s+/g, " ");
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates a preference score for a caption track to determine the best match
|
||||
* @param {Object} track - The caption track object from YouTube
|
||||
* @param {string} track.languageCode - ISO language code (e.g., 'zh-HK', 'en', 'es')
|
||||
* @param {string} track.kind - Track type ('asr' for auto-generated, "" for human-transcribed)
|
||||
* @param {string[]} preferredLanguages - Array of language codes in preference order (e.g., ['zh-HK', 'en'])
|
||||
* @returns {number} Preference score (lower is better)
|
||||
*/
|
||||
static #calculatePreferenceScore(track, preferredLanguages) {
|
||||
// Language preference: index in preferredLanguages array (0 = most preferred)
|
||||
const languagePreference = preferredLanguages.indexOf(track.languageCode);
|
||||
const languageScore = languagePreference === -1 ? 9999 : languagePreference;
|
||||
|
||||
// Kind bonus: prefer human-transcribed (undefined) over auto-generated ('asr')
|
||||
const kindBonus = track.kind === "asr" ? 0.5 : 0;
|
||||
|
||||
return languageScore + kindBonus;
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the most suitable caption track based on preferred languages
|
||||
* @param {string} videoBody - The raw HTML response from YouTube
|
||||
* @param {string[]} preferredLanguages - Array of language codes in preference order
|
||||
* @returns {Object|null} The selected caption track or null if none found
|
||||
*/
|
||||
static #findPreferredCaptionTrack(videoBody, preferredLanguages) {
|
||||
const captionsConfigJson = videoBody.match(
|
||||
/"captions":(.*?),"videoDetails":/s
|
||||
);
|
||||
|
||||
const captionsConfig = captionsConfigJson?.[1]
|
||||
? JSON.parse(captionsConfigJson[1])
|
||||
: null;
|
||||
|
||||
const captionTracks = captionsConfig
|
||||
? captionsConfig.playerCaptionsTracklistRenderer.captionTracks
|
||||
: null;
|
||||
|
||||
if (!captionTracks || captionTracks.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sortedTracks = [...captionTracks].sort((a, b) => {
|
||||
const scoreA = this.#calculatePreferenceScore(a, preferredLanguages);
|
||||
const scoreB = this.#calculatePreferenceScore(b, preferredLanguages);
|
||||
return scoreA - scoreB;
|
||||
});
|
||||
|
||||
return sortedTracks[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches video page content and finds the preferred caption track
|
||||
* @param {string} videoId - YouTube video ID
|
||||
* @param {string[]} preferredLanguages - Array of preferred language codes
|
||||
* @returns {Promise<Object>} The preferred caption track
|
||||
* @throws {YoutubeTranscriptError} If no suitable caption track is found
|
||||
*/
|
||||
static async #getPreferredCaptionTrack(videoId, preferredLanguages) {
|
||||
const videoResponse = await fetch(
|
||||
`https://www.youtube.com/watch?v=${videoId}`,
|
||||
{ credentials: "omit" }
|
||||
);
|
||||
const videoBody = await videoResponse.text();
|
||||
|
||||
const preferredCaptionTrack = this.#findPreferredCaptionTrack(
|
||||
videoBody,
|
||||
preferredLanguages
|
||||
);
|
||||
|
||||
if (!preferredCaptionTrack) {
|
||||
throw new YoutubeTranscriptError(
|
||||
"No suitable caption track found for the video"
|
||||
);
|
||||
}
|
||||
|
||||
return preferredCaptionTrack;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch transcript from YouTube video
|
||||
* @param {string} videoId - Video URL or video identifier
|
||||
@ -93,14 +172,20 @@ class YoutubeTranscript {
|
||||
* @returns {Promise<string>} Video transcript text
|
||||
*/
|
||||
static async fetchTranscript(videoId, config = {}) {
|
||||
const preferredLanguages = config?.lang ? [config?.lang, "en"] : ["en"];
|
||||
const identifier = this.retrieveVideoId(videoId);
|
||||
const lang = config?.lang ?? "en";
|
||||
|
||||
try {
|
||||
const preferredCaptionTrack = await this.#getPreferredCaptionTrack(
|
||||
identifier,
|
||||
preferredLanguages
|
||||
);
|
||||
|
||||
const innerProto = this.#getBase64Protobuf({
|
||||
param1: "asr",
|
||||
param2: lang,
|
||||
param1: preferredCaptionTrack.kind || "",
|
||||
param2: preferredCaptionTrack.languageCode,
|
||||
});
|
||||
|
||||
const params = this.#getBase64Protobuf({
|
||||
param1: identifier,
|
||||
param2: innerProto,
|
||||
|
||||
@ -44,6 +44,7 @@ GID='1000'
|
||||
# OLLAMA_MODEL_PREF='llama2'
|
||||
# OLLAMA_MODEL_TOKEN_LIMIT=4096
|
||||
# OLLAMA_AUTH_TOKEN='your-ollama-auth-token-here (optional, only for ollama running behind auth - Bearer token)'
|
||||
# OLLAMA_RESPONSE_TIMEOUT=7200000 (optional, max timeout in milliseconds for ollama response to conclude. Default is 5min before aborting)
|
||||
|
||||
# LLM_PROVIDER='togetherai'
|
||||
# TOGETHER_AI_API_KEY='my-together-ai-key'
|
||||
@ -195,6 +196,7 @@ GID='1000'
|
||||
# EMBEDDING_BASE_PATH='http://127.0.0.1:4000'
|
||||
# GENERIC_OPEN_AI_EMBEDDING_API_KEY='sk-123abc'
|
||||
# GENERIC_OPEN_AI_EMBEDDING_MAX_CONCURRENT_CHUNKS=500
|
||||
# GENERIC_OPEN_AI_EMBEDDING_API_DELAY_MS=1000
|
||||
|
||||
# EMBEDDING_ENGINE='gemini'
|
||||
# GEMINI_EMBEDDING_API_KEY=
|
||||
|
||||
@ -46,6 +46,7 @@ export default [
|
||||
"no-undef": "warn",
|
||||
"no-empty": "warn",
|
||||
"no-extra-boolean-cast": "warn",
|
||||
"no-prototype-builtins": "off",
|
||||
"prettier/prettier": "warn"
|
||||
}
|
||||
},
|
||||
|
||||
@ -59,7 +59,7 @@ function AdvancedControls({ settings }) {
|
||||
name="CometApiLLMTimeout"
|
||||
className="border-none bg-theme-settings-input-bg text-theme-text-primary placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
|
||||
placeholder="Timeout value between token responses to auto-timeout the stream"
|
||||
defaultValue={settings?.CometApiLLMTimeout ?? 500}
|
||||
defaultValue={settings?.CometApiLLMTimeout ?? 3_000}
|
||||
autoComplete="off"
|
||||
onScroll={(e) => e.target.blur()}
|
||||
min={500}
|
||||
|
||||
@ -92,7 +92,7 @@ export default function DellProAIStudioOptions({
|
||||
type="url"
|
||||
name="DellProAiStudioBasePath"
|
||||
className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
|
||||
placeholder="http://localhost:8553/v1"
|
||||
placeholder="http://localhost:8553/v1/openai"
|
||||
value={basePathValue.value}
|
||||
required={true}
|
||||
autoComplete="off"
|
||||
|
||||
@ -59,7 +59,7 @@ function AdvancedControls({ settings }) {
|
||||
name="NovitaLLMTimeout"
|
||||
className="border-none bg-theme-settings-input-bg text-theme-text-primary placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
|
||||
placeholder="Timeout value between token responses to auto-timeout the stream"
|
||||
defaultValue={settings?.NovitaLLMTimeout ?? 500}
|
||||
defaultValue={settings?.NovitaLLMTimeout ?? 3_000}
|
||||
autoComplete="off"
|
||||
onScroll={(e) => e.target.blur()}
|
||||
min={500}
|
||||
|
||||
@ -57,7 +57,7 @@ function AdvancedControls({ settings }) {
|
||||
name="OpenRouterTimeout"
|
||||
className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
|
||||
placeholder="Timeout value between token responses to auto-timeout the stream"
|
||||
defaultValue={settings?.OpenRouterTimeout ?? 500}
|
||||
defaultValue={settings?.OpenRouterTimeout ?? 3_000}
|
||||
autoComplete="off"
|
||||
onScroll={(e) => e.target.blur()}
|
||||
min={500}
|
||||
|
||||
@ -61,7 +61,7 @@ export default function SearchBox({ user, showNewWsModal }) {
|
||||
onChange={handleSearch}
|
||||
onReset={handleReset}
|
||||
onFocus={(e) => e.target.select()}
|
||||
className="border-none w-full h-full rounded-lg bg-theme-sidebar-item-default pl-4 pr-1 placeholder:text-theme-settings-input-placeholder placeholder:pl-4 outline-none text-white search-input peer text-sm"
|
||||
className="border-none w-full h-full rounded-lg bg-theme-sidebar-item-default pl-9 focus:pl-4 pr-1 placeholder:text-theme-settings-input-placeholder outline-none text-white search-input peer text-sm"
|
||||
/>
|
||||
<MagnifyingGlass
|
||||
size={14}
|
||||
|
||||
@ -45,11 +45,16 @@ export default function VariableRow({ variable, onRefresh }) {
|
||||
bg: "bg-blue-600/20",
|
||||
text: "text-blue-400 light:text-blue-800",
|
||||
};
|
||||
case "dynamic":
|
||||
case "user":
|
||||
return {
|
||||
bg: "bg-green-600/20",
|
||||
text: "text-green-400 light:text-green-800",
|
||||
};
|
||||
case "workspace":
|
||||
return {
|
||||
bg: "bg-cyan-600/20",
|
||||
text: "text-cyan-400 light:text-cyan-800",
|
||||
};
|
||||
default:
|
||||
return {
|
||||
bg: "bg-yellow-600/20",
|
||||
@ -81,7 +86,7 @@ export default function VariableRow({ variable, onRefresh }) {
|
||||
<span
|
||||
className={`rounded-full ${colorTheme.bg} px-2 py-0.5 text-xs leading-5 font-semibold ${colorTheme.text} shadow-sm`}
|
||||
>
|
||||
{titleCase(variable.type)}
|
||||
{titleCase(variable?.type ?? "static")}
|
||||
</span>
|
||||
</td>
|
||||
<td className="px-4 py-2 flex items-center justify-end gap-x-4">
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
import truncate from "truncate";
|
||||
import { X, Trash, LinkSimple } from "@phosphor-icons/react";
|
||||
import { X } from "@phosphor-icons/react";
|
||||
import ModalWrapper from "@/components/ModalWrapper";
|
||||
import { useModal } from "@/hooks/useModal";
|
||||
import paths from "@/utils/paths";
|
||||
import Embed from "@/models/embed";
|
||||
import MarkdownRenderer from "../MarkdownRenderer";
|
||||
import { safeJsonParse } from "@/utils/request";
|
||||
|
||||
export default function ChatRow({ chat, onDelete }) {
|
||||
const {
|
||||
@ -83,7 +85,11 @@ export default function ChatRow({ chat, onDelete }) {
|
||||
</ModalWrapper>
|
||||
<ModalWrapper isOpen={isResponseOpen}>
|
||||
<TextPreview
|
||||
text={JSON.parse(chat.response)?.text}
|
||||
text={
|
||||
<MarkdownRenderer
|
||||
content={safeJsonParse(chat.response, {})?.text}
|
||||
/>
|
||||
}
|
||||
closeModal={closeResponseModal}
|
||||
/>
|
||||
</ModalWrapper>
|
||||
@ -118,9 +124,9 @@ const TextPreview = ({ text, closeModal }) => {
|
||||
</button>
|
||||
</div>
|
||||
<div className="w-full p-6">
|
||||
<pre className="w-full h-[200px] py-2 px-4 whitespace-pre-line overflow-auto rounded-lg bg-zinc-900 light:bg-theme-bg-secondary border border-gray-500 text-white text-sm">
|
||||
<div className="w-full h-[60vh] py-2 px-4 whitespace-pre-line overflow-auto rounded-lg bg-zinc-900 light:bg-theme-bg-secondary border border-gray-500 text-white text-sm">
|
||||
{text}
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -132,11 +138,7 @@ const ConnectionDetails = ({
|
||||
verbose = false,
|
||||
connection_information,
|
||||
}) => {
|
||||
let details = {};
|
||||
try {
|
||||
details = JSON.parse(connection_information);
|
||||
} catch {}
|
||||
|
||||
const details = safeJsonParse(connection_information, {});
|
||||
if (Object.keys(details).length === 0) return null;
|
||||
|
||||
if (verbose) {
|
||||
|
||||
@ -0,0 +1,87 @@
|
||||
import { useState } from "react";
|
||||
import MarkdownIt from "markdown-it";
|
||||
import { CaretDown } from "@phosphor-icons/react";
|
||||
import "highlight.js/styles/github-dark.css";
|
||||
import DOMPurify from "@/utils/chat/purify";
|
||||
|
||||
const md = new MarkdownIt({
|
||||
html: true,
|
||||
breaks: true,
|
||||
highlight: function (str, lang) {
|
||||
if (lang && hljs.getLanguage(lang)) {
|
||||
try {
|
||||
return hljs.highlight(str, { language: lang }).value;
|
||||
} catch (__) {}
|
||||
}
|
||||
return ""; // use external default escaping
|
||||
},
|
||||
});
|
||||
|
||||
const ThoughtBubble = ({ thought }) => {
|
||||
const [isExpanded, setIsExpanded] = useState(false);
|
||||
|
||||
if (!thought) return null;
|
||||
|
||||
const cleanThought = thought.replace(/<\/?think>/g, "").trim();
|
||||
if (!cleanThought) return null;
|
||||
|
||||
return (
|
||||
<div className="mb-3">
|
||||
<div
|
||||
onClick={() => setIsExpanded(!isExpanded)}
|
||||
className="cursor-pointer flex items-center gap-x-2 text-theme-text-secondary hover:text-theme-text-primary transition-colors mb-2"
|
||||
>
|
||||
<CaretDown
|
||||
size={14}
|
||||
weight="bold"
|
||||
className={`transition-transform ${isExpanded ? "rotate-180" : ""}`}
|
||||
/>
|
||||
<span className="text-xs font-medium">View thoughts</span>
|
||||
</div>
|
||||
{isExpanded && (
|
||||
<div className="bg-theme-bg-chat-input rounded-md p-3 border-l-2 border-theme-text-secondary/30">
|
||||
<div className="text-xs text-theme-text-secondary font-mono whitespace-pre-wrap">
|
||||
{cleanThought}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
function parseContent(content) {
|
||||
const parts = [];
|
||||
let lastIndex = 0;
|
||||
content.replace(/<think>([^]*?)<\/think>/g, (match, thinkContent, offset) => {
|
||||
if (offset > lastIndex) {
|
||||
parts.push({ type: "normal", text: content.slice(lastIndex, offset) });
|
||||
}
|
||||
parts.push({ type: "think", text: thinkContent });
|
||||
lastIndex = offset + match.length;
|
||||
});
|
||||
if (lastIndex < content.length) {
|
||||
parts.push({ type: "normal", text: content.slice(lastIndex) });
|
||||
}
|
||||
return parts;
|
||||
}
|
||||
|
||||
export default function MarkdownRenderer({ content }) {
|
||||
if (!content) return null;
|
||||
|
||||
const parts = parseContent(content);
|
||||
return (
|
||||
<div className="whitespace-normal">
|
||||
{parts.map((part, index) => {
|
||||
const html = md.render(part.text);
|
||||
if (part.type === "think")
|
||||
return <ThoughtBubble key={index} thought={part.text} />;
|
||||
return (
|
||||
<div
|
||||
key={index}
|
||||
dangerouslySetInnerHTML={{ __html: DOMPurify.sanitize(html) }}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@ -55,6 +55,7 @@ export default function EmbedChatsView() {
|
||||
const query = useQuery();
|
||||
const [offset, setOffset] = useState(Number(query.get("offset") || 0));
|
||||
const [canNext, setCanNext] = useState(false);
|
||||
const [showThinking, setShowThinking] = useState(true);
|
||||
|
||||
const handleDumpChats = async (exportType) => {
|
||||
const chats = await System.exportChats(exportType, "embed");
|
||||
@ -92,10 +93,15 @@ export default function EmbedChatsView() {
|
||||
|
||||
useEffect(() => {
|
||||
async function fetchChats() {
|
||||
const { chats: _chats, hasPages = false } = await Embed.chats(offset);
|
||||
setChats(_chats);
|
||||
setCanNext(hasPages);
|
||||
setLoading(false);
|
||||
setLoading(true);
|
||||
await Embed.chats(offset)
|
||||
.then(({ chats: _chats, hasPages = false }) => {
|
||||
setChats(_chats);
|
||||
setCanNext(hasPages);
|
||||
})
|
||||
.finally(() => {
|
||||
setLoading(false);
|
||||
});
|
||||
}
|
||||
fetchChats();
|
||||
}, [offset]);
|
||||
@ -211,7 +217,7 @@ export default function EmbedChatsView() {
|
||||
: "bg-theme-bg-secondary text-theme-text-primary hover:bg-theme-hover"
|
||||
}`}
|
||||
>
|
||||
{t("embed-chats.previous")}
|
||||
{t("common.previous")}
|
||||
</button>
|
||||
<button
|
||||
onClick={handleNext}
|
||||
@ -222,7 +228,7 @@ export default function EmbedChatsView() {
|
||||
: "bg-theme-bg-secondary text-theme-text-primary hover:bg-theme-hover"
|
||||
}`}
|
||||
>
|
||||
{t("embed-chats.next")}
|
||||
{t("common.next")}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@ -38,10 +38,10 @@ export const LOCALAI_COMMON_URLS = [
|
||||
];
|
||||
|
||||
export const DPAIS_COMMON_URLS = [
|
||||
"http://127.0.0.1:8553/v1",
|
||||
"http://0.0.0.0:8553/v1",
|
||||
"http://localhost:8553/v1",
|
||||
"http://host.docker.internal:8553/v1",
|
||||
"http://127.0.0.1:8553/v1/openai",
|
||||
"http://0.0.0.0:8553/v1/openai",
|
||||
"http://localhost:8553/v1/openai",
|
||||
"http://host.docker.internal:8553/v1/openai",
|
||||
];
|
||||
|
||||
export const NVIDIA_NIM_COMMON_URLS = [
|
||||
|
||||
@ -41,6 +41,7 @@ SIG_SALT='salt' # Please generate random string at least 32 chars long.
|
||||
# OLLAMA_MODEL_PREF='llama2'
|
||||
# OLLAMA_MODEL_TOKEN_LIMIT=4096
|
||||
# OLLAMA_AUTH_TOKEN='your-ollama-auth-token-here (optional, only for ollama running behind auth - Bearer token)'
|
||||
# OLLAMA_RESPONSE_TIMEOUT=7200000 (optional, max timeout in milliseconds for ollama response to conclude. Default is 5min before aborting)
|
||||
|
||||
# LLM_PROVIDER='togetherai'
|
||||
# TOGETHER_AI_API_KEY='my-together-ai-key'
|
||||
@ -194,6 +195,7 @@ SIG_SALT='salt' # Please generate random string at least 32 chars long.
|
||||
# EMBEDDING_BASE_PATH='http://127.0.0.1:4000'
|
||||
# GENERIC_OPEN_AI_EMBEDDING_API_KEY='sk-123abc'
|
||||
# GENERIC_OPEN_AI_EMBEDDING_MAX_CONCURRENT_CHUNKS=500
|
||||
# GENERIC_OPEN_AI_EMBEDDING_API_DELAY_MS=1000
|
||||
|
||||
# EMBEDDING_ENGINE='gemini'
|
||||
# GEMINI_EMBEDDING_API_KEY=
|
||||
@ -227,7 +229,7 @@ SIG_SALT='salt' # Please generate random string at least 32 chars long.
|
||||
# Enable all below if you are using vector database: LanceDB.
|
||||
VECTOR_DB="lancedb"
|
||||
|
||||
# Enable all below if you are using vector database: Weaviate.
|
||||
# Enable all below if you are using vector database: PG Vector.
|
||||
# VECTOR_DB="pgvector"
|
||||
# PGVECTOR_CONNECTION_STRING="postgresql://dbuser:dbuserpass@localhost:5432/yourdb"
|
||||
# PGVECTOR_TABLE_NAME="anythingllm_vectors" # optional, but can be defined
|
||||
@ -366,4 +368,9 @@ TTS_PROVIDER="native"
|
||||
# Runtime flags for built-in pupeeteer Chromium instance
|
||||
# This is only required on Linux machines running AnythingLLM via Docker
|
||||
# and do not want to use the --cap-add=SYS_ADMIN docker argument
|
||||
# ANYTHINGLLM_CHROMIUM_ARGS="--no-sandbox,--disable-setuid-sandbox"
|
||||
# ANYTHINGLLM_CHROMIUM_ARGS="--no-sandbox,--disable-setuid-sandbox"
|
||||
|
||||
# This enables HTTP request/response logging in development. Set value to a truthy string to enable, leave empty value or comment out to disable.
|
||||
# ENABLE_HTTP_LOGGER=""
|
||||
# This enables timestamps for the HTTP Logger. Set value to a truthy string to enable, leave empty value or comment out to disable.
|
||||
# ENABLE_HTTP_LOGGER_TIMESTAMPS=""
|
||||
61
server/__tests__/models/systemPromptVariables.test.js
Normal file
61
server/__tests__/models/systemPromptVariables.test.js
Normal file
@ -0,0 +1,61 @@
|
||||
const { SystemPromptVariables } = require("../../models/systemPromptVariables");
|
||||
const prisma = require("../../utils/prisma");
|
||||
|
||||
const mockUser = {
|
||||
id: 1,
|
||||
username: "john.doe",
|
||||
bio: "I am a test user",
|
||||
};
|
||||
|
||||
const mockWorkspace = {
|
||||
id: 1,
|
||||
name: "Test Workspace",
|
||||
slug: 'test-workspace',
|
||||
};
|
||||
|
||||
const mockSystemPromptVariables = [
|
||||
{
|
||||
id: 1,
|
||||
key: "mystaticvariable",
|
||||
value: "AnythingLLM testing runtime",
|
||||
description: "A test variable",
|
||||
type: "static",
|
||||
userId: null,
|
||||
},
|
||||
];
|
||||
|
||||
describe("SystemPromptVariables.expandSystemPromptVariables", () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
// Mock just the Prisma actions since that is what is used by default values
|
||||
prisma.system_prompt_variables.findMany = jest.fn().mockResolvedValue(mockSystemPromptVariables);
|
||||
prisma.workspaces.findUnique = jest.fn().mockResolvedValue(mockWorkspace);
|
||||
prisma.users.findUnique = jest.fn().mockResolvedValue(mockUser);
|
||||
});
|
||||
|
||||
it("should expand user-defined system prompt variables", async () => {
|
||||
const variables = await SystemPromptVariables.expandSystemPromptVariables("Hello {mystaticvariable}");
|
||||
expect(variables).toBe(`Hello ${mockSystemPromptVariables[0].value}`);
|
||||
});
|
||||
|
||||
it("should expand workspace-defined system prompt variables", async () => {
|
||||
const variables = await SystemPromptVariables.expandSystemPromptVariables("Hello {workspace.name}", null, mockWorkspace.id);
|
||||
expect(variables).toBe(`Hello ${mockWorkspace.name}`);
|
||||
});
|
||||
|
||||
it("should expand user-defined system prompt variables", async () => {
|
||||
const variables = await SystemPromptVariables.expandSystemPromptVariables("Hello {user.name}", mockUser.id);
|
||||
expect(variables).toBe(`Hello ${mockUser.username}`);
|
||||
});
|
||||
|
||||
it("should work with any combination of variables", async () => {
|
||||
const variables = await SystemPromptVariables.expandSystemPromptVariables("Hello {mystaticvariable} {workspace.name} {user.name}", mockUser.id, mockWorkspace.id);
|
||||
expect(variables).toBe(`Hello ${mockSystemPromptVariables[0].value} ${mockWorkspace.name} ${mockUser.username}`);
|
||||
});
|
||||
|
||||
it('should fail gracefully with invalid variables that are undefined for any reason', async () => {
|
||||
// Undefined sub-fields on valid classes are push to a placeholder [Class prop]. This is expected behavior.
|
||||
const variables = await SystemPromptVariables.expandSystemPromptVariables("Hello {invalid.variable} {user.password} the current user is {user.name} on workspace id #{workspace.id}", null, null);
|
||||
expect(variables).toBe("Hello {invalid.variable} [User password] the current user is [User name] on workspace id #[Workspace ID]");
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,76 @@
|
||||
const { PGVector } = require("../../../../utils/vectorDbProviders/pgvector");
|
||||
|
||||
describe("PGVector.sanitizeForJsonb", () => {
|
||||
it("returns null/undefined as-is", () => {
|
||||
expect(PGVector.sanitizeForJsonb(null)).toBeNull();
|
||||
expect(PGVector.sanitizeForJsonb(undefined)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps safe whitespace (tab, LF, CR) and removes disallowed C0 controls", () => {
|
||||
const input = "a\u0000\u0001\u0002\tline\ncarriage\rreturn\u001Fend";
|
||||
const result = PGVector.sanitizeForJsonb(input);
|
||||
// Expect all < 0x20 except 9,10,13 removed; keep letters and allowed whitespace
|
||||
expect(result).toBe("a\tline\ncarriage\rreturnend");
|
||||
});
|
||||
|
||||
it("removes only disallowed control chars; keeps normal printable chars", () => {
|
||||
const input = "Hello\u0000, World! \u0007\u0008\u000B\u000C\u001F";
|
||||
const result = PGVector.sanitizeForJsonb(input);
|
||||
expect(result).toBe("Hello, World! ");
|
||||
});
|
||||
|
||||
it("deeply sanitizes objects", () => {
|
||||
const input = {
|
||||
plain: "ok",
|
||||
bad: "has\u0000nul",
|
||||
nested: {
|
||||
arr: ["fine", "bad\u0001", { deep: "\u0002oops" }],
|
||||
},
|
||||
};
|
||||
const result = PGVector.sanitizeForJsonb(input);
|
||||
expect(result).toEqual({
|
||||
plain: "ok",
|
||||
bad: "hasnul",
|
||||
nested: { arr: ["fine", "bad", { deep: "oops" }] },
|
||||
});
|
||||
});
|
||||
|
||||
it("deeply sanitizes arrays", () => {
|
||||
const input = ["\u0000", 1, true, { s: "bad\u0003" }, ["ok", "\u0004bad"]];
|
||||
const result = PGVector.sanitizeForJsonb(input);
|
||||
expect(result).toEqual(["", 1, true, { s: "bad" }, ["ok", "bad"]]);
|
||||
});
|
||||
|
||||
it("converts Date to ISO string", () => {
|
||||
const d = new Date("2020-01-02T03:04:05.000Z");
|
||||
expect(PGVector.sanitizeForJsonb(d)).toBe(d.toISOString());
|
||||
});
|
||||
|
||||
it("returns primitives unchanged (number, boolean, bigint)", () => {
|
||||
expect(PGVector.sanitizeForJsonb(42)).toBe(42);
|
||||
expect(PGVector.sanitizeForJsonb(3.14)).toBe(3.14);
|
||||
expect(PGVector.sanitizeForJsonb(true)).toBe(true);
|
||||
expect(PGVector.sanitizeForJsonb(false)).toBe(false);
|
||||
expect(PGVector.sanitizeForJsonb(BigInt(1))).toBe(BigInt(1));
|
||||
});
|
||||
|
||||
it("returns symbol unchanged", () => {
|
||||
const sym = Symbol("x");
|
||||
expect(PGVector.sanitizeForJsonb(sym)).toBe(sym);
|
||||
});
|
||||
|
||||
it("does not mutate original objects/arrays", () => {
|
||||
const obj = { a: "bad\u0000", nested: { b: "ok" } };
|
||||
const arr = ["\u0001", { c: "bad\u0002" }];
|
||||
const objCopy = JSON.parse(JSON.stringify(obj));
|
||||
const arrCopy = JSON.parse(JSON.stringify(arr));
|
||||
const resultObj = PGVector.sanitizeForJsonb(obj);
|
||||
const resultArr = PGVector.sanitizeForJsonb(arr);
|
||||
// Original inputs remain unchanged
|
||||
expect(obj).toEqual(objCopy);
|
||||
expect(arr).toEqual(arrCopy);
|
||||
// Results are sanitized copies
|
||||
expect(resultObj).toEqual({ a: "bad", nested: { b: "ok" } });
|
||||
expect(resultArr).toEqual(["", { c: "bad" }]);
|
||||
});
|
||||
});
|
||||
@ -29,10 +29,22 @@ const { communityHubEndpoints } = require("./endpoints/communityHub");
|
||||
const { agentFlowEndpoints } = require("./endpoints/agentFlows");
|
||||
const { mcpServersEndpoints } = require("./endpoints/mcpServers");
|
||||
const { mobileEndpoints } = require("./endpoints/mobile");
|
||||
const { httpLogger } = require("./middleware/httpLogger");
|
||||
const app = express();
|
||||
const apiRouter = express.Router();
|
||||
const FILE_LIMIT = "3GB";
|
||||
|
||||
// Only log HTTP requests in development mode and if the ENABLE_HTTP_LOGGER environment variable is set to true
|
||||
if (
|
||||
process.env.NODE_ENV === "development" &&
|
||||
!!process.env.ENABLE_HTTP_LOGGER
|
||||
) {
|
||||
app.use(
|
||||
httpLogger({
|
||||
enableTimestamps: !!process.env.ENABLE_HTTP_LOGGER_TIMESTAMPS,
|
||||
})
|
||||
);
|
||||
}
|
||||
app.use(cors({ origin: true }));
|
||||
app.use(bodyParser.text({ limit: FILE_LIMIT }));
|
||||
app.use(bodyParser.json({ limit: FILE_LIMIT }));
|
||||
|
||||
23
server/middleware/httpLogger.js
Normal file
23
server/middleware/httpLogger.js
Normal file
@ -0,0 +1,23 @@
|
||||
const httpLogger =
|
||||
({ enableTimestamps = false }) =>
|
||||
(req, res, next) => {
|
||||
// Capture the original res.end to log response status
|
||||
const originalEnd = res.end;
|
||||
|
||||
res.end = function (chunk, encoding) {
|
||||
// Log the request method, status code, and path
|
||||
const statusColor = res.statusCode >= 400 ? "\x1b[31m" : "\x1b[32m"; // Red for errors, green for success
|
||||
console.log(
|
||||
`\x1b[32m[HTTP]\x1b[0m ${statusColor}${res.statusCode}\x1b[0m ${req.method} -> ${req.path} ${enableTimestamps ? `@ ${new Date().toLocaleTimeString("en-US", { hour12: true })}` : ""}`.trim()
|
||||
);
|
||||
|
||||
// Call the original end method
|
||||
return originalEnd.call(this, chunk, encoding);
|
||||
};
|
||||
|
||||
next();
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
httpLogger,
|
||||
};
|
||||
@ -7,13 +7,13 @@ const moment = require("moment");
|
||||
* @property {string} key
|
||||
* @property {string|function} value
|
||||
* @property {string} description
|
||||
* @property {'system'|'user'|'static'} type
|
||||
* @property {'system'|'user'|'workspace'|'static'} type
|
||||
* @property {number} userId
|
||||
* @property {boolean} multiUserRequired
|
||||
*/
|
||||
|
||||
const SystemPromptVariables = {
|
||||
VALID_TYPES: ["user", "system", "static"],
|
||||
VALID_TYPES: ["user", "workspace", "system", "static"],
|
||||
DEFAULT_VARIABLES: [
|
||||
{
|
||||
key: "time",
|
||||
@ -36,6 +36,16 @@ const SystemPromptVariables = {
|
||||
type: "system",
|
||||
multiUserRequired: false,
|
||||
},
|
||||
{
|
||||
key: "user.id",
|
||||
value: (userId = null) => {
|
||||
if (!userId) return "[User ID]";
|
||||
return userId;
|
||||
},
|
||||
description: "Current user's ID",
|
||||
type: "user",
|
||||
multiUserRequired: true,
|
||||
},
|
||||
{
|
||||
key: "user.name",
|
||||
value: async (userId = null) => {
|
||||
@ -74,6 +84,30 @@ const SystemPromptVariables = {
|
||||
type: "user",
|
||||
multiUserRequired: true,
|
||||
},
|
||||
{
|
||||
key: "workspace.id",
|
||||
value: (workspaceId = null) => {
|
||||
if (!workspaceId) return "[Workspace ID]";
|
||||
return workspaceId;
|
||||
},
|
||||
description: "Current workspace's ID",
|
||||
type: "workspace",
|
||||
multiUserRequired: false,
|
||||
},
|
||||
{
|
||||
key: "workspace.name",
|
||||
value: async (workspaceId = null) => {
|
||||
if (!workspaceId) return "[Workspace name]";
|
||||
const workspace = await prisma.workspaces.findUnique({
|
||||
where: { id: Number(workspaceId) },
|
||||
select: { name: true },
|
||||
});
|
||||
return workspace?.name || "[Workspace name is empty or unknown]";
|
||||
},
|
||||
description: "Current workspace's name",
|
||||
type: "workspace",
|
||||
multiUserRequired: false,
|
||||
},
|
||||
],
|
||||
|
||||
/**
|
||||
@ -183,12 +217,17 @@ const SystemPromptVariables = {
|
||||
},
|
||||
|
||||
/**
|
||||
* Injects variables into a string based on the user ID (if provided) and the variables available
|
||||
* Injects variables into a string based on the user ID and workspace ID (if provided) and the variables available
|
||||
* @param {string} str - the input string to expand variables into
|
||||
* @param {number|null} userId - the user ID to use for dynamic variables
|
||||
* @param {number|null} workspaceId - the workspace ID to use for workspace variables
|
||||
* @returns {Promise<string>}
|
||||
*/
|
||||
expandSystemPromptVariables: async function (str, userId = null) {
|
||||
expandSystemPromptVariables: async function (
|
||||
str,
|
||||
userId = null,
|
||||
workspaceId = null
|
||||
) {
|
||||
if (!str) return str;
|
||||
|
||||
try {
|
||||
@ -202,26 +241,62 @@ const SystemPromptVariables = {
|
||||
for (const match of matches) {
|
||||
const key = match.substring(1, match.length - 1); // Remove { and }
|
||||
|
||||
// Handle `user.X` variables with current user's data
|
||||
if (key.startsWith("user.")) {
|
||||
const userProp = key.split(".")[1];
|
||||
// Determine if the variable is a class-based variable (workspace.X or user.X)
|
||||
const isWorkspaceOrUserVariable = ["workspace.", "user."].some(
|
||||
(prefix) => key.startsWith(prefix)
|
||||
);
|
||||
|
||||
// Handle class-based variables with current workspace's or user's data
|
||||
if (isWorkspaceOrUserVariable) {
|
||||
let variableTypeDisplay;
|
||||
if (key.startsWith("workspace.")) variableTypeDisplay = "Workspace";
|
||||
else if (key.startsWith("user.")) variableTypeDisplay = "User";
|
||||
else throw new Error(`Invalid class-based variable: ${key}`);
|
||||
|
||||
// Get the property name after the prefix
|
||||
const prop = key.split(".")[1];
|
||||
const variable = allVariables.find((v) => v.key === key);
|
||||
|
||||
// If the variable is a function, call it to get the current value
|
||||
if (variable && typeof variable.value === "function") {
|
||||
// If the variable is an async function, call it to get the current value
|
||||
if (variable.value.constructor.name === "AsyncFunction") {
|
||||
let value;
|
||||
try {
|
||||
const value = await variable.value(userId);
|
||||
result = result.replace(match, value);
|
||||
if (variableTypeDisplay === "Workspace")
|
||||
value = await variable.value(workspaceId);
|
||||
else if (variableTypeDisplay === "User")
|
||||
value = await variable.value(userId);
|
||||
else throw new Error(`Invalid class-based variable: ${key}`);
|
||||
} catch (error) {
|
||||
console.error(`Error processing user variable ${key}:`, error);
|
||||
result = result.replace(match, `[User ${userProp}]`);
|
||||
console.error(
|
||||
`Error processing ${variableTypeDisplay} variable ${key}:`,
|
||||
error
|
||||
);
|
||||
value = `[${variableTypeDisplay} ${prop}]`;
|
||||
}
|
||||
result = result.replace(match, value);
|
||||
} else {
|
||||
const value = variable.value();
|
||||
let value;
|
||||
try {
|
||||
// Call the variable function with the appropriate workspace or user ID
|
||||
if (variableTypeDisplay === "Workspace")
|
||||
value = variable.value(workspaceId);
|
||||
else if (variableTypeDisplay === "User")
|
||||
value = variable.value(userId);
|
||||
else throw new Error(`Invalid class-based variable: ${key}`);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`Error processing ${variableTypeDisplay} variable ${key}:`,
|
||||
error
|
||||
);
|
||||
value = `[${variableTypeDisplay} ${prop}]`;
|
||||
}
|
||||
result = result.replace(match, value);
|
||||
}
|
||||
} else {
|
||||
result = result.replace(match, `[User ${userProp}]`);
|
||||
// If the variable is not a function, replace the match with the variable value
|
||||
result = result.replace(match, `[${variableTypeDisplay} ${prop}]`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -15,6 +15,7 @@ class AnthropicLLM {
|
||||
if (!process.env.ANTHROPIC_API_KEY)
|
||||
throw new Error("No Anthropic API key was set.");
|
||||
|
||||
this.className = "AnthropicLLM";
|
||||
// Docs: https://www.npmjs.com/package/@anthropic-ai/sdk
|
||||
const AnthropicAI = require("@anthropic-ai/sdk");
|
||||
const anthropic = new AnthropicAI({
|
||||
@ -37,7 +38,7 @@ class AnthropicLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
streamingEnabled() {
|
||||
|
||||
@ -23,6 +23,7 @@ class ApiPieLLM {
|
||||
if (!process.env.APIPIE_LLM_API_KEY)
|
||||
throw new Error("No ApiPie LLM API key was set.");
|
||||
|
||||
this.className = "ApiPieLLM";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
this.basePath = "https://apipie.ai/v1";
|
||||
this.openai = new OpenAIApi({
|
||||
@ -49,7 +50,7 @@ class ApiPieLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
// This checks if the .cached_at file has a timestamp that is more than 1Week (in millis)
|
||||
|
||||
@ -19,10 +19,12 @@ const cacheFolder = path.resolve(
|
||||
);
|
||||
|
||||
class CometApiLLM {
|
||||
defaultTimeout = 3_000;
|
||||
constructor(embedder = null, modelPreference = null) {
|
||||
if (!process.env.COMETAPI_LLM_API_KEY)
|
||||
throw new Error("No CometAPI API key was set.");
|
||||
|
||||
this.className = "CometApiLLM";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
this.basePath = "https://api.cometapi.com/v1";
|
||||
this.openai = new OpenAIApi({
|
||||
@ -54,17 +56,21 @@ class CometApiLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
/**
|
||||
* CometAPI has various models that never return `finish_reasons` and thus leave the stream open
|
||||
* which causes issues in subsequent messages. This timeout value forces us to close the stream after
|
||||
* x milliseconds. This is a configurable value via the COMETAPI_LLM_TIMEOUT_MS value
|
||||
* @returns {number} The timeout value in milliseconds (default: 500)
|
||||
* @returns {number} The timeout value in milliseconds (default: 3_000)
|
||||
*/
|
||||
#parseTimeout() {
|
||||
if (isNaN(Number(process.env.COMETAPI_LLM_TIMEOUT_MS))) return 500;
|
||||
this.log(
|
||||
`CometAPI timeout is set to ${process.env.COMETAPI_LLM_TIMEOUT_MS ?? this.defaultTimeout}ms`
|
||||
);
|
||||
if (isNaN(Number(process.env.COMETAPI_LLM_TIMEOUT_MS)))
|
||||
return this.defaultTimeout;
|
||||
const setValue = Number(process.env.COMETAPI_LLM_TIMEOUT_MS);
|
||||
if (setValue < 500) return 500;
|
||||
return setValue;
|
||||
|
||||
@ -13,6 +13,7 @@ class DeepSeekLLM {
|
||||
constructor(embedder = null, modelPreference = null) {
|
||||
if (!process.env.DEEPSEEK_API_KEY)
|
||||
throw new Error("No DeepSeek API key was set.");
|
||||
this.className = "DeepSeekLLM";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
|
||||
this.openai = new OpenAIApi({
|
||||
@ -35,7 +36,7 @@ class DeepSeekLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
#appendContext(contextTexts = []) {
|
||||
|
||||
@ -13,6 +13,7 @@ class DellProAiStudioLLM {
|
||||
if (!process.env.DPAIS_LLM_BASE_PATH)
|
||||
throw new Error("No Dell Pro AI Studio Base Path was set.");
|
||||
|
||||
this.className = "DellProAiStudioLLM";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
this.dpais = new OpenAIApi({
|
||||
baseURL: DellProAiStudioLLM.parseBasePath(),
|
||||
@ -50,7 +51,7 @@ class DellProAiStudioLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
#appendContext(contextTexts = []) {
|
||||
|
||||
@ -29,6 +29,7 @@ class GeminiLLM {
|
||||
if (!process.env.GEMINI_API_KEY)
|
||||
throw new Error("No Gemini API key was set.");
|
||||
|
||||
this.className = "GeminiLLM";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
this.model =
|
||||
modelPreference ||
|
||||
@ -71,7 +72,7 @@ class GeminiLLM {
|
||||
}
|
||||
|
||||
#log(text, ...args) {
|
||||
console.log(`\x1b[32m[GeminiLLM]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[32m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
// This checks if the .cached_at file has a timestamp that is more than 1Week (in millis)
|
||||
|
||||
@ -18,6 +18,7 @@ class GenericOpenAiLLM {
|
||||
"GenericOpenAI must have a valid base path to use for the api."
|
||||
);
|
||||
|
||||
this.className = "GenericOpenAiLLM";
|
||||
this.basePath = process.env.GENERIC_OPEN_AI_BASE_PATH;
|
||||
this.openai = new OpenAIApi({
|
||||
baseURL: this.basePath,
|
||||
@ -45,7 +46,7 @@ class GenericOpenAiLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
#appendContext(contextTexts = []) {
|
||||
|
||||
@ -17,6 +17,7 @@ class KoboldCPPLLM {
|
||||
"KoboldCPP must have a valid base path to use for the api."
|
||||
);
|
||||
|
||||
this.className = "KoboldCPPLLM";
|
||||
this.basePath = process.env.KOBOLD_CPP_BASE_PATH;
|
||||
this.openai = new OpenAIApi({
|
||||
baseURL: this.basePath,
|
||||
@ -37,7 +38,7 @@ class KoboldCPPLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
#appendContext(contextTexts = []) {
|
||||
|
||||
@ -15,6 +15,7 @@ class LiteLLM {
|
||||
"LiteLLM must have a valid base path to use for the api."
|
||||
);
|
||||
|
||||
this.className = "LiteLLM";
|
||||
this.basePath = process.env.LITE_LLM_BASE_PATH;
|
||||
this.openai = new OpenAIApi({
|
||||
baseURL: this.basePath,
|
||||
@ -35,7 +36,7 @@ class LiteLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
#appendContext(contextTexts = []) {
|
||||
|
||||
@ -12,6 +12,7 @@ class MistralLLM {
|
||||
if (!process.env.MISTRAL_API_KEY)
|
||||
throw new Error("No Mistral API key was set.");
|
||||
|
||||
this.className = "MistralLLM";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
this.openai = new OpenAIApi({
|
||||
baseURL: "https://api.mistral.ai/v1",
|
||||
@ -31,7 +32,7 @@ class MistralLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
#appendContext(contextTexts = []) {
|
||||
|
||||
@ -12,6 +12,7 @@ class MoonshotAiLLM {
|
||||
constructor(embedder = null, modelPreference = null) {
|
||||
if (!process.env.MOONSHOT_AI_API_KEY)
|
||||
throw new Error("No Moonshot AI API key was set.");
|
||||
this.className = "MoonshotAiLLM";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
|
||||
this.openai = new OpenAIApi({
|
||||
@ -36,7 +37,7 @@ class MoonshotAiLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
#appendContext(contextTexts = []) {
|
||||
|
||||
@ -18,10 +18,13 @@ const cacheFolder = path.resolve(
|
||||
);
|
||||
|
||||
class NovitaLLM {
|
||||
defaultTimeout = 3_000;
|
||||
|
||||
constructor(embedder = null, modelPreference = null) {
|
||||
if (!process.env.NOVITA_LLM_API_KEY)
|
||||
throw new Error("No Novita API key was set.");
|
||||
|
||||
this.className = "NovitaLLM";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
this.basePath = "https://api.novita.ai/v3/openai";
|
||||
this.openai = new OpenAIApi({
|
||||
@ -55,19 +58,23 @@ class NovitaLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
/**
|
||||
* Novita has various models that never return `finish_reasons` and thus leave the stream open
|
||||
* which causes issues in subsequent messages. This timeout value forces us to close the stream after
|
||||
* x milliseconds. This is a configurable value via the NOVITA_LLM_TIMEOUT_MS value
|
||||
* @returns {number} The timeout value in milliseconds (default: 500)
|
||||
* @returns {number} The timeout value in milliseconds (default: 3_000)
|
||||
*/
|
||||
#parseTimeout() {
|
||||
if (isNaN(Number(process.env.NOVITA_LLM_TIMEOUT_MS))) return 500;
|
||||
this.log(
|
||||
`Novita timeout is set to ${process.env.NOVITA_LLM_TIMEOUT_MS ?? this.defaultTimeout}ms`
|
||||
);
|
||||
if (isNaN(Number(process.env.NOVITA_LLM_TIMEOUT_MS)))
|
||||
return this.defaultTimeout;
|
||||
const setValue = Number(process.env.NOVITA_LLM_TIMEOUT_MS);
|
||||
if (setValue < 500) return 500;
|
||||
if (setValue < 500) return 500; // 500ms is the minimum timeout
|
||||
return setValue;
|
||||
}
|
||||
|
||||
@ -318,7 +325,7 @@ class NovitaLLM {
|
||||
});
|
||||
}
|
||||
|
||||
if (message.finish_reason !== null) {
|
||||
if (message?.finish_reason !== null) {
|
||||
writeResponseChunk(response, {
|
||||
uuid,
|
||||
sources,
|
||||
|
||||
@ -12,6 +12,7 @@ class NvidiaNimLLM {
|
||||
if (!process.env.NVIDIA_NIM_LLM_BASE_PATH)
|
||||
throw new Error("No NVIDIA NIM API Base Path was set.");
|
||||
|
||||
this.className = "NvidiaNimLLM";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
this.nvidiaNim = new OpenAIApi({
|
||||
baseURL: parseNvidiaNimBasePath(process.env.NVIDIA_NIM_LLM_BASE_PATH),
|
||||
@ -33,7 +34,7 @@ class NvidiaNimLLM {
|
||||
}
|
||||
|
||||
#log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
#appendContext(contextTexts = []) {
|
||||
|
||||
@ -31,7 +31,11 @@ class OllamaAILLM {
|
||||
const headers = this.authToken
|
||||
? { Authorization: `Bearer ${this.authToken}` }
|
||||
: {};
|
||||
this.client = new Ollama({ host: this.basePath, headers: headers });
|
||||
this.client = new Ollama({
|
||||
host: this.basePath,
|
||||
headers: headers,
|
||||
fetch: this.#applyFetch(),
|
||||
});
|
||||
this.embedder = embedder ?? new NativeEmbedder();
|
||||
this.defaultTemp = 0.7;
|
||||
this.#log(
|
||||
@ -55,6 +59,43 @@ class OllamaAILLM {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a custom fetch function to the Ollama client.
|
||||
* This is useful when we want to bypass the default 5m timeout for global fetch
|
||||
* for machines which run responses very slowly.
|
||||
* @returns {Function} The custom fetch function.
|
||||
*/
|
||||
#applyFetch() {
|
||||
try {
|
||||
if (!("OLLAMA_RESPONSE_TIMEOUT" in process.env)) return fetch;
|
||||
const { Agent } = require("undici");
|
||||
const moment = require("moment");
|
||||
let timeout = process.env.OLLAMA_RESPONSE_TIMEOUT;
|
||||
|
||||
if (!timeout || isNaN(Number(timeout)) || Number(timeout) <= 5 * 60_000) {
|
||||
this.#log(
|
||||
"Timeout option was not set, is not a number, or is less than 5 minutes in ms - falling back to default",
|
||||
{ timeout }
|
||||
);
|
||||
return fetch;
|
||||
} else timeout = Number(timeout);
|
||||
|
||||
const noTimeoutFetch = (input, init = {}) => {
|
||||
return fetch(input, {
|
||||
...init,
|
||||
dispatcher: new Agent({ headersTimeout: timeout }),
|
||||
});
|
||||
};
|
||||
|
||||
const humanDiff = moment.duration(timeout).humanize();
|
||||
this.#log(`Applying custom fetch w/timeout of ${humanDiff}.`);
|
||||
return noTimeoutFetch;
|
||||
} catch (error) {
|
||||
this.#log("Error applying custom fetch - using default fetch", error);
|
||||
return fetch;
|
||||
}
|
||||
}
|
||||
|
||||
streamingEnabled() {
|
||||
return "streamGetChatCompletion" in this;
|
||||
}
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
const { v4: uuidv4 } = require("uuid");
|
||||
const { NativeEmbedder } = require("../../EmbeddingEngines/native");
|
||||
const {
|
||||
handleDefaultStreamResponseV2,
|
||||
formatChatHistory,
|
||||
writeResponseChunk,
|
||||
clientAbortedHandler,
|
||||
} = require("../../helpers/chat/responses");
|
||||
const { MODEL_MAP } = require("../modelMap");
|
||||
const {
|
||||
@ -11,6 +13,7 @@ const {
|
||||
class OpenAiLLM {
|
||||
constructor(embedder = null, modelPreference = null) {
|
||||
if (!process.env.OPEN_AI_KEY) throw new Error("No OpenAI API key was set.");
|
||||
this.className = "OpenAiLLM";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
|
||||
this.openai = new OpenAIApi({
|
||||
@ -31,15 +34,7 @@ class OpenAiLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the model is an o1 model.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
get isOTypeModel() {
|
||||
return this.model.startsWith("o");
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
#appendContext(contextTexts = []) {
|
||||
@ -55,8 +50,6 @@ class OpenAiLLM {
|
||||
}
|
||||
|
||||
streamingEnabled() {
|
||||
// o3-mini is the only o-type model that supports streaming
|
||||
if (this.isOTypeModel && this.model !== "o3-mini") return false;
|
||||
return "streamGetChatCompletion" in this;
|
||||
}
|
||||
|
||||
@ -96,14 +89,11 @@ class OpenAiLLM {
|
||||
return userPrompt;
|
||||
}
|
||||
|
||||
const content = [{ type: "text", text: userPrompt }];
|
||||
const content = [{ type: "input_text", text: userPrompt }];
|
||||
for (let attachment of attachments) {
|
||||
content.push({
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: attachment.contentString,
|
||||
detail: "auto",
|
||||
},
|
||||
type: "input_image",
|
||||
image_url: attachment.contentString,
|
||||
});
|
||||
}
|
||||
return content.flat();
|
||||
@ -121,11 +111,8 @@ class OpenAiLLM {
|
||||
userPrompt = "",
|
||||
attachments = [], // This is the specific attachment for only this prompt
|
||||
}) {
|
||||
// o1 Models do not support the "system" role
|
||||
// in order to combat this, we can use the "user" role as a replacement for now
|
||||
// https://community.openai.com/t/o1-models-do-not-support-system-role-in-chat-completion/953880
|
||||
const prompt = {
|
||||
role: this.isOTypeModel ? "user" : "system",
|
||||
role: "system",
|
||||
content: `${systemPrompt}${this.#appendContext(contextTexts)}`,
|
||||
};
|
||||
return [
|
||||
@ -138,6 +125,24 @@ class OpenAiLLM {
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the appropriate temperature for the model.
|
||||
* @param {string} modelName
|
||||
* @param {number} temperature
|
||||
* @returns {number}
|
||||
*/
|
||||
#temperature(modelName, temperature) {
|
||||
// For models that don't support temperature
|
||||
// OpenAI accepts temperature 1
|
||||
const NO_TEMP_MODELS = ["o", "gpt-5"];
|
||||
|
||||
if (NO_TEMP_MODELS.some((prefix) => modelName.startsWith(prefix))) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return temperature;
|
||||
}
|
||||
|
||||
async getChatCompletion(messages = null, { temperature = 0.7 }) {
|
||||
if (!(await this.isValidChatCompletionModel(this.model)))
|
||||
throw new Error(
|
||||
@ -145,30 +150,30 @@ class OpenAiLLM {
|
||||
);
|
||||
|
||||
const result = await LLMPerformanceMonitor.measureAsyncFunction(
|
||||
this.openai.chat.completions
|
||||
this.openai.responses
|
||||
.create({
|
||||
model: this.model,
|
||||
messages,
|
||||
temperature: this.isOTypeModel ? 1 : temperature, // o1 models only accept temperature 1
|
||||
input: messages,
|
||||
store: false,
|
||||
temperature: this.#temperature(this.model, temperature),
|
||||
})
|
||||
.catch((e) => {
|
||||
throw new Error(e.message);
|
||||
})
|
||||
);
|
||||
|
||||
if (
|
||||
!result.output.hasOwnProperty("choices") ||
|
||||
result.output.choices.length === 0
|
||||
)
|
||||
return null;
|
||||
if (!result.output.hasOwnProperty("output_text")) return null;
|
||||
|
||||
const usage = result.output.usage || {};
|
||||
return {
|
||||
textResponse: result.output.choices[0].message.content,
|
||||
textResponse: result.output.output_text,
|
||||
metrics: {
|
||||
prompt_tokens: result.output.usage.prompt_tokens || 0,
|
||||
completion_tokens: result.output.usage.completion_tokens || 0,
|
||||
total_tokens: result.output.usage.total_tokens || 0,
|
||||
outputTps: result.output.usage.completion_tokens / result.duration,
|
||||
prompt_tokens: usage.prompt_tokens || 0,
|
||||
completion_tokens: usage.completion_tokens || 0,
|
||||
total_tokens: usage.total_tokens || 0,
|
||||
outputTps: usage.completion_tokens
|
||||
? usage.completion_tokens / result.duration
|
||||
: 0,
|
||||
duration: result.duration,
|
||||
},
|
||||
};
|
||||
@ -181,23 +186,88 @@ class OpenAiLLM {
|
||||
);
|
||||
|
||||
const measuredStreamRequest = await LLMPerformanceMonitor.measureStream(
|
||||
this.openai.chat.completions.create({
|
||||
this.openai.responses.create({
|
||||
model: this.model,
|
||||
stream: true,
|
||||
messages,
|
||||
temperature: this.isOTypeModel ? 1 : temperature, // o1 models only accept temperature 1
|
||||
input: messages,
|
||||
store: false,
|
||||
temperature: this.#temperature(this.model, temperature),
|
||||
}),
|
||||
messages
|
||||
// runPromptTokenCalculation: true - We manually count the tokens because OpenAI does not provide them in the stream
|
||||
// since we are not using the OpenAI API version that supports this `stream_options` param.
|
||||
// TODO: implement this once we upgrade to the OpenAI API version that supports this param.
|
||||
messages,
|
||||
false
|
||||
);
|
||||
|
||||
return measuredStreamRequest;
|
||||
}
|
||||
|
||||
handleStream(response, stream, responseProps) {
|
||||
return handleDefaultStreamResponseV2(response, stream, responseProps);
|
||||
const { uuid = uuidv4(), sources = [] } = responseProps;
|
||||
|
||||
let hasUsageMetrics = false;
|
||||
let usage = {
|
||||
completion_tokens: 0,
|
||||
};
|
||||
|
||||
return new Promise(async (resolve) => {
|
||||
let fullText = "";
|
||||
|
||||
const handleAbort = () => {
|
||||
stream?.endMeasurement(usage);
|
||||
clientAbortedHandler(resolve, fullText);
|
||||
};
|
||||
response.on("close", handleAbort);
|
||||
|
||||
try {
|
||||
for await (const chunk of stream) {
|
||||
if (chunk.type === "response.output_text.delta") {
|
||||
const token = chunk.delta;
|
||||
if (token) {
|
||||
fullText += token;
|
||||
if (!hasUsageMetrics) usage.completion_tokens++;
|
||||
writeResponseChunk(response, {
|
||||
uuid,
|
||||
sources: [],
|
||||
type: "textResponseChunk",
|
||||
textResponse: token,
|
||||
close: false,
|
||||
error: false,
|
||||
});
|
||||
}
|
||||
} else if (chunk.type === "response.completed") {
|
||||
const { response: res } = chunk;
|
||||
if (res.hasOwnProperty("usage") && !!res.usage) {
|
||||
hasUsageMetrics = true;
|
||||
usage = { ...usage, ...res.usage };
|
||||
}
|
||||
|
||||
writeResponseChunk(response, {
|
||||
uuid,
|
||||
sources,
|
||||
type: "textResponseChunk",
|
||||
textResponse: "",
|
||||
close: true,
|
||||
error: false,
|
||||
});
|
||||
response.removeListener("close", handleAbort);
|
||||
stream?.endMeasurement(usage);
|
||||
resolve(fullText);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.log(`\x1b[43m\x1b[34m[STREAMING ERROR]\x1b[0m ${e.message}`);
|
||||
writeResponseChunk(response, {
|
||||
uuid,
|
||||
type: "abort",
|
||||
textResponse: null,
|
||||
sources: [],
|
||||
close: true,
|
||||
error: e.message,
|
||||
});
|
||||
stream?.endMeasurement(usage);
|
||||
resolve(fullText);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Simple wrapper for dynamic embedder & normalize interface for all LLM implementations
|
||||
|
||||
@ -18,10 +18,21 @@ const cacheFolder = path.resolve(
|
||||
);
|
||||
|
||||
class OpenRouterLLM {
|
||||
/**
|
||||
* Some openrouter models never send a finish_reason and thus leave the stream open in the UI.
|
||||
* However, because OR is a middleware it can also wait an inordinately long time between chunks so we need
|
||||
* to ensure that we dont accidentally close the stream too early. If the time between chunks is greater than this timeout
|
||||
* we will close the stream and assume it to be complete. This is common for free models or slow providers they can
|
||||
* possibly delegate to during invocation.
|
||||
* @type {number}
|
||||
*/
|
||||
defaultTimeout = 3_000;
|
||||
|
||||
constructor(embedder = null, modelPreference = null) {
|
||||
if (!process.env.OPENROUTER_API_KEY)
|
||||
throw new Error("No OpenRouter API key was set.");
|
||||
|
||||
this.className = "OpenRouterLLM";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
this.basePath = "https://openrouter.ai/api/v1";
|
||||
this.openai = new OpenAIApi({
|
||||
@ -78,22 +89,23 @@ class OpenRouterLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenRouter has various models that never return `finish_reasons` and thus leave the stream open
|
||||
* which causes issues in subsequent messages. This timeout value forces us to close the stream after
|
||||
* x milliseconds. This is a configurable value via the OPENROUTER_TIMEOUT_MS value
|
||||
* @returns {number} The timeout value in milliseconds (default: 500)
|
||||
* @returns {number} The timeout value in milliseconds (default: 3_000)
|
||||
*/
|
||||
#parseTimeout() {
|
||||
this.log(
|
||||
`OpenRouter timeout is set to ${process.env.OPENROUTER_TIMEOUT_MS ?? 500}ms`
|
||||
`OpenRouter timeout is set to ${process.env.OPENROUTER_TIMEOUT_MS ?? this.defaultTimeout}ms`
|
||||
);
|
||||
if (isNaN(Number(process.env.OPENROUTER_TIMEOUT_MS))) return 500;
|
||||
if (isNaN(Number(process.env.OPENROUTER_TIMEOUT_MS)))
|
||||
return this.defaultTimeout;
|
||||
const setValue = Number(process.env.OPENROUTER_TIMEOUT_MS);
|
||||
if (setValue < 500) return 500;
|
||||
if (setValue < 500) return 500; // 500ms is the minimum timeout
|
||||
return setValue;
|
||||
}
|
||||
|
||||
|
||||
@ -18,6 +18,7 @@ class PPIOLLM {
|
||||
constructor(embedder = null, modelPreference = null) {
|
||||
if (!process.env.PPIO_API_KEY) throw new Error("No PPIO API key was set.");
|
||||
|
||||
this.className = "PPIOLLM";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
this.basePath = "https://api.ppinfra.com/v3/openai/";
|
||||
this.openai = new OpenAIApi({
|
||||
@ -50,7 +51,7 @@ class PPIOLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
async #syncModels() {
|
||||
|
||||
@ -15,6 +15,7 @@ class TextGenWebUILLM {
|
||||
"TextGenWebUI must have a valid base path to use for the api."
|
||||
);
|
||||
|
||||
this.className = "TextGenWebUILLM";
|
||||
this.basePath = process.env.TEXT_GEN_WEB_UI_BASE_PATH;
|
||||
this.openai = new OpenAIApi({
|
||||
baseURL: this.basePath,
|
||||
@ -33,7 +34,7 @@ class TextGenWebUILLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
#appendContext(contextTexts = []) {
|
||||
|
||||
@ -12,6 +12,7 @@ class XAiLLM {
|
||||
constructor(embedder = null, modelPreference = null) {
|
||||
if (!process.env.XAI_LLM_API_KEY)
|
||||
throw new Error("No xAI API key was set.");
|
||||
this.className = "XAiLLM";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
|
||||
this.openai = new OpenAIApi({
|
||||
@ -34,7 +35,7 @@ class XAiLLM {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
#appendContext(contextTexts = []) {
|
||||
|
||||
@ -8,6 +8,7 @@ class AzureOpenAiEmbedder {
|
||||
if (!process.env.AZURE_OPENAI_KEY)
|
||||
throw new Error("No Azure API key was set.");
|
||||
|
||||
this.className = "AzureOpenAiEmbedder";
|
||||
this.apiVersion = "2024-12-01-preview";
|
||||
const openai = new AzureOpenAI({
|
||||
apiKey: process.env.AZURE_OPENAI_KEY,
|
||||
@ -29,7 +30,7 @@ class AzureOpenAiEmbedder {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[AzureOpenAiEmbedder]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
async embedTextInput(textInput) {
|
||||
|
||||
@ -11,6 +11,7 @@ class GeminiEmbedder {
|
||||
if (!process.env.GEMINI_EMBEDDING_API_KEY)
|
||||
throw new Error("No Gemini API key was set.");
|
||||
|
||||
this.className = "GeminiEmbedder";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
this.model = process.env.EMBEDDING_MODEL_PREF || "text-embedding-004";
|
||||
this.openai = new OpenAIApi({
|
||||
@ -29,7 +30,7 @@ class GeminiEmbedder {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[GeminiEmbedder]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -6,6 +6,7 @@ class GenericOpenAiEmbedder {
|
||||
throw new Error(
|
||||
"GenericOpenAI must have a valid base path to use for the api."
|
||||
);
|
||||
this.className = "GenericOpenAiEmbedder";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
this.basePath = process.env.EMBEDDING_BASE_PATH;
|
||||
this.openai = new OpenAIApi({
|
||||
@ -25,7 +26,36 @@ class GenericOpenAiEmbedder {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[GenericOpenAiEmbedder]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
/**
|
||||
* returns the `GENERIC_OPEN_AI_EMBEDDING_API_DELAY_MS` env variable as a number or null if the env variable is not set or is not a number.
|
||||
* The minimum delay is 500ms.
|
||||
*
|
||||
* For some implementation this is necessary to avoid 429 errors due to rate limiting or
|
||||
* hardware limitations where a single-threaded process is not able to handle the requests fast enough.
|
||||
* @returns {number}
|
||||
*/
|
||||
get apiRequestDelay() {
|
||||
if (!("GENERIC_OPEN_AI_EMBEDDING_API_DELAY_MS" in process.env)) return null;
|
||||
if (isNaN(Number(process.env.GENERIC_OPEN_AI_EMBEDDING_API_DELAY_MS)))
|
||||
return null;
|
||||
const delayTimeout = Number(
|
||||
process.env.GENERIC_OPEN_AI_EMBEDDING_API_DELAY_MS
|
||||
);
|
||||
if (delayTimeout < 500) return 500; // minimum delay of 500ms
|
||||
return delayTimeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* runs the delay if it is set and valid.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async runDelay() {
|
||||
if (!this.apiRequestDelay) return;
|
||||
this.log(`Delaying new batch request for ${this.apiRequestDelay}ms`);
|
||||
await new Promise((resolve) => setTimeout(resolve, this.apiRequestDelay));
|
||||
}
|
||||
|
||||
/**
|
||||
@ -52,62 +82,38 @@ class GenericOpenAiEmbedder {
|
||||
|
||||
async embedChunks(textChunks = []) {
|
||||
// Because there is a hard POST limit on how many chunks can be sent at once to OpenAI (~8mb)
|
||||
// we concurrently execute each max batch of text chunks possible.
|
||||
// we sequentially execute each max batch of text chunks possible.
|
||||
// Refer to constructor maxConcurrentChunks for more info.
|
||||
const embeddingRequests = [];
|
||||
const allResults = [];
|
||||
for (const chunk of toChunks(textChunks, this.maxConcurrentChunks)) {
|
||||
embeddingRequests.push(
|
||||
new Promise((resolve) => {
|
||||
this.openai.embeddings
|
||||
.create({
|
||||
model: this.model,
|
||||
input: chunk,
|
||||
})
|
||||
.then((result) => {
|
||||
resolve({ data: result?.data, error: null });
|
||||
})
|
||||
.catch((e) => {
|
||||
e.type =
|
||||
e?.response?.data?.error?.code ||
|
||||
e?.response?.status ||
|
||||
"failed_to_embed";
|
||||
e.message = e?.response?.data?.error?.message || e.message;
|
||||
resolve({ data: [], error: e });
|
||||
});
|
||||
})
|
||||
);
|
||||
}
|
||||
const { data = [], error = null } = await new Promise((resolve) => {
|
||||
this.openai.embeddings
|
||||
.create({
|
||||
model: this.model,
|
||||
input: chunk,
|
||||
})
|
||||
.then((result) => resolve({ data: result?.data, error: null }))
|
||||
.catch((e) => {
|
||||
e.type =
|
||||
e?.response?.data?.error?.code ||
|
||||
e?.response?.status ||
|
||||
"failed_to_embed";
|
||||
e.message = e?.response?.data?.error?.message || e.message;
|
||||
resolve({ data: [], error: e });
|
||||
});
|
||||
});
|
||||
|
||||
const { data = [], error = null } = await Promise.all(
|
||||
embeddingRequests
|
||||
).then((results) => {
|
||||
// If any errors were returned from OpenAI abort the entire sequence because the embeddings
|
||||
// will be incomplete.
|
||||
const errors = results
|
||||
.filter((res) => !!res.error)
|
||||
.map((res) => res.error)
|
||||
.flat();
|
||||
if (errors.length > 0) {
|
||||
let uniqueErrors = new Set();
|
||||
errors.map((error) =>
|
||||
uniqueErrors.add(`[${error.type}]: ${error.message}`)
|
||||
);
|
||||
if (error)
|
||||
throw new Error(`GenericOpenAI Failed to embed: ${error.message}`);
|
||||
allResults.push(...(data || []));
|
||||
if (this.apiRequestDelay) await this.runDelay();
|
||||
}
|
||||
|
||||
return {
|
||||
data: [],
|
||||
error: Array.from(uniqueErrors).join(", "),
|
||||
};
|
||||
}
|
||||
return {
|
||||
data: results.map((res) => res?.data || []).flat(),
|
||||
error: null,
|
||||
};
|
||||
});
|
||||
|
||||
if (!!error) throw new Error(`GenericOpenAI Failed to embed: ${error}`);
|
||||
return data.length > 0 &&
|
||||
data.every((embd) => embd.hasOwnProperty("embedding"))
|
||||
? data.map((embd) => embd.embedding)
|
||||
return allResults.length > 0 &&
|
||||
allResults.every((embd) => embd.hasOwnProperty("embedding"))
|
||||
? allResults.map((embd) => embd.embedding)
|
||||
: null;
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,6 +8,7 @@ class LMStudioEmbedder {
|
||||
if (!process.env.EMBEDDING_MODEL_PREF)
|
||||
throw new Error("No embedding model was set.");
|
||||
|
||||
this.className = "LMStudioEmbedder";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
this.lmstudio = new OpenAIApi({
|
||||
baseURL: parseLMStudioBasePath(process.env.EMBEDDING_BASE_PATH),
|
||||
@ -21,7 +22,7 @@ class LMStudioEmbedder {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
async #isAlive() {
|
||||
|
||||
@ -30,6 +30,7 @@ class NativeEmbedder {
|
||||
#fallbackHost = "https://cdn.anythingllm.com/support/models/";
|
||||
|
||||
constructor() {
|
||||
this.className = "NativeEmbedder";
|
||||
this.model = this.getEmbeddingModel();
|
||||
this.modelInfo = this.getEmbedderInfo();
|
||||
this.cacheDir = path.resolve(
|
||||
@ -50,7 +51,7 @@ class NativeEmbedder {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[NativeEmbedder]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -8,6 +8,7 @@ class OllamaEmbedder {
|
||||
if (!process.env.EMBEDDING_MODEL_PREF)
|
||||
throw new Error("No embedding model was set.");
|
||||
|
||||
this.className = "OllamaEmbedder";
|
||||
this.basePath = process.env.EMBEDDING_BASE_PATH;
|
||||
this.model = process.env.EMBEDDING_MODEL_PREF;
|
||||
// Limit of how many strings we can process in a single pass to stay with resource or network limits
|
||||
@ -20,7 +21,7 @@ class OllamaEmbedder {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -3,6 +3,7 @@ const { toChunks } = require("../../helpers");
|
||||
class OpenAiEmbedder {
|
||||
constructor() {
|
||||
if (!process.env.OPEN_AI_KEY) throw new Error("No OpenAI API key was set.");
|
||||
this.className = "OpenAiEmbedder";
|
||||
const { OpenAI: OpenAIApi } = require("openai");
|
||||
this.openai = new OpenAIApi({
|
||||
apiKey: process.env.OPEN_AI_KEY,
|
||||
@ -17,7 +18,7 @@ class OpenAiEmbedder {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[OpenAiEmbedder]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
async embedTextInput(textInput) {
|
||||
|
||||
@ -53,6 +53,7 @@ class MCPHypervisor {
|
||||
constructor() {
|
||||
if (MCPHypervisor._instance) return MCPHypervisor._instance;
|
||||
MCPHypervisor._instance = this;
|
||||
this.className = "MCPHypervisor";
|
||||
this.log("Initializing MCP Hypervisor - subsequent calls will boot faster");
|
||||
this.#setupConfigFile();
|
||||
return this;
|
||||
@ -88,7 +89,7 @@ class MCPHypervisor {
|
||||
}
|
||||
|
||||
log(text, ...args) {
|
||||
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
|
||||
console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -28,6 +28,7 @@ class MSSQLConnector {
|
||||
connectionString: null, // we will force into RFC-3986
|
||||
}
|
||||
) {
|
||||
this.className = "MSSQLConnector";
|
||||
this.connectionString = config.connectionString;
|
||||
this._client = null;
|
||||
this.#parseDatabase();
|
||||
@ -72,7 +73,7 @@ class MSSQLConnector {
|
||||
result.rows = query.recordset;
|
||||
result.count = query.rowsAffected.reduce((sum, a) => sum + a, 0);
|
||||
} catch (err) {
|
||||
console.log(this.constructor.name, err);
|
||||
console.log(this.className, err);
|
||||
result.error = err.message;
|
||||
} finally {
|
||||
// Check client is connected before closing since we use this for validation
|
||||
|
||||
@ -9,6 +9,7 @@ class MySQLConnector {
|
||||
connectionString: null,
|
||||
}
|
||||
) {
|
||||
this.className = "MySQLConnector";
|
||||
this.connectionString = config.connectionString;
|
||||
this._client = null;
|
||||
this.database_id = this.#parseDatabase();
|
||||
@ -39,7 +40,7 @@ class MySQLConnector {
|
||||
result.rows = query;
|
||||
result.count = query?.length;
|
||||
} catch (err) {
|
||||
console.log(this.constructor.name, err);
|
||||
console.log(this.className, err);
|
||||
result.error = err.message;
|
||||
} finally {
|
||||
// Check client is connected before closing since we use this for validation
|
||||
|
||||
@ -8,6 +8,7 @@ class PostgresSQLConnector {
|
||||
schema: null,
|
||||
}
|
||||
) {
|
||||
this.className = "PostgresSQLConnector";
|
||||
this.connectionString = config.connectionString;
|
||||
this.schema = config.schema || "public";
|
||||
this._client = new pgSql.Client({
|
||||
@ -34,7 +35,7 @@ class PostgresSQLConnector {
|
||||
result.rows = query.rows;
|
||||
result.count = query.rowCount;
|
||||
} catch (err) {
|
||||
console.log(this.constructor.name, err);
|
||||
console.log(this.className, err);
|
||||
result.error = err.message;
|
||||
} finally {
|
||||
// Check client is connected before closing since we use this for validation
|
||||
|
||||
@ -17,6 +17,7 @@ class GeminiProvider extends InheritMultiple([Provider, UnTooled]) {
|
||||
constructor(config = {}) {
|
||||
const { model = "gemini-2.0-flash-lite" } = config;
|
||||
super();
|
||||
this.className = "GeminiProvider";
|
||||
const client = new OpenAI({
|
||||
baseURL: "https://generativelanguage.googleapis.com/v1beta/openai/",
|
||||
apiKey: process.env.GEMINI_API_KEY,
|
||||
@ -134,7 +135,7 @@ class GeminiProvider extends InheritMultiple([Provider, UnTooled]) {
|
||||
} catch (error) {
|
||||
throw new APIError(
|
||||
error?.message
|
||||
? `${this.constructor.name} encountered an error while executing the request: ${error.message}`
|
||||
? `${this.className} encountered an error while executing the request: ${error.message}`
|
||||
: "There was an error with the Gemini provider executing the request"
|
||||
);
|
||||
}
|
||||
|
||||
@ -94,7 +94,8 @@ async function chatPrompt(workspace, user = null) {
|
||||
"Given the following conversation, relevant context, and a follow up question, reply with an answer to the current question the user is asking. Return only your response to the question given the above information following the users instructions as needed.";
|
||||
return await SystemPromptVariables.expandSystemPromptVariables(
|
||||
basePrompt,
|
||||
user?.id
|
||||
user?.id,
|
||||
workspace?.id
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@ -1170,6 +1170,9 @@ function dumpENV() {
|
||||
|
||||
// Specify Chromium args for collector
|
||||
"ANYTHINGLLM_CHROMIUM_ARGS",
|
||||
|
||||
// Allow setting a custom response timeout for Ollama
|
||||
"OLLAMA_RESPONSE_TIMEOUT",
|
||||
];
|
||||
|
||||
// Simple sanitization of each value to prevent ENV injection via newline or quote escaping.
|
||||
|
||||
@ -44,6 +44,7 @@ const PGVector = {
|
||||
"SELECT * FROM pg_catalog.pg_tables WHERE schemaname = 'public'",
|
||||
getEmbeddingTableSchemaSql:
|
||||
"SELECT column_name,data_type FROM information_schema.columns WHERE table_name = $1",
|
||||
createExtensionSql: "CREATE EXTENSION IF NOT EXISTS vector;",
|
||||
createTableSql: (dimensions) =>
|
||||
`CREATE TABLE IF NOT EXISTS "${PGVector.tableName()}" (id UUID PRIMARY KEY, namespace TEXT, embedding vector(${Number(dimensions)}), metadata JSONB, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)`,
|
||||
|
||||
@ -51,6 +52,55 @@ const PGVector = {
|
||||
console.log(`\x1b[35m[PGVectorDb]\x1b[0m ${message}`, ...args);
|
||||
},
|
||||
|
||||
/**
|
||||
* Recursively sanitize values intended for JSONB to prevent Postgres errors
|
||||
* like "unsupported Unicode escape sequence". This primarily removes the
|
||||
* NUL character (\u0000) and other disallowed control characters from
|
||||
* strings. Arrays and objects are traversed and sanitized deeply.
|
||||
* @param {any} value
|
||||
* @returns {any}
|
||||
*/
|
||||
sanitizeForJsonb: function (value) {
|
||||
// Fast path for null/undefined and primitives that do not need changes
|
||||
if (value === null || value === undefined) return value;
|
||||
|
||||
// Strings: strip NUL and unsafe C0 control characters except common whitespace
|
||||
if (typeof value === "string") {
|
||||
// Build a sanitized string by excluding C0 control characters except
|
||||
// horizontal tab (9), line feed (10), and carriage return (13).
|
||||
let sanitized = "";
|
||||
for (let i = 0; i < value.length; i++) {
|
||||
const code = value.charCodeAt(i);
|
||||
if (code === 9 || code === 10 || code === 13 || code >= 0x20) {
|
||||
sanitized += value[i];
|
||||
}
|
||||
}
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
// Arrays: sanitize each element
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((item) => this.sanitizeForJsonb(item));
|
||||
}
|
||||
|
||||
// Dates: keep as ISO string
|
||||
if (value instanceof Date) {
|
||||
return value.toISOString();
|
||||
}
|
||||
|
||||
// Objects: sanitize each property value
|
||||
if (typeof value === "object") {
|
||||
const result = {};
|
||||
for (const [k, v] of Object.entries(value)) {
|
||||
result[k] = this.sanitizeForJsonb(v);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Numbers, booleans, etc.
|
||||
return value;
|
||||
},
|
||||
|
||||
client: function (connectionString = null) {
|
||||
return new pgsql.Client({
|
||||
connectionString: connectionString || PGVector.connectionString(),
|
||||
@ -361,9 +411,11 @@ const PGVector = {
|
||||
|
||||
/**
|
||||
* Update or create a collection in the database
|
||||
* @param {pgsql.Connection} connection
|
||||
* @param {{id: number, vector: number[], metadata: Object}[]} submissions
|
||||
* @param {string} namespace
|
||||
* @param {Object} params
|
||||
* @param {pgsql.Connection} params.connection
|
||||
* @param {{id: number, vector: number[], metadata: Object}[]} params.submissions
|
||||
* @param {string} params.namespace
|
||||
* @param {number} params.dimensions
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
updateOrCreateCollection: async function ({
|
||||
@ -380,9 +432,10 @@ const PGVector = {
|
||||
await connection.query(`BEGIN`);
|
||||
for (const submission of submissions) {
|
||||
const embedding = `[${submission.vector.map(Number).join(",")}]`; // stringify the vector for pgvector
|
||||
const sanitizedMetadata = this.sanitizeForJsonb(submission.metadata);
|
||||
await connection.query(
|
||||
`INSERT INTO "${PGVector.tableName()}" (id, namespace, embedding, metadata) VALUES ($1, $2, $3, $4)`,
|
||||
[submission.id, namespace, embedding, submission.metadata]
|
||||
[submission.id, namespace, embedding, sanitizedMetadata]
|
||||
);
|
||||
}
|
||||
this.log(`Committing ${submissions.length} vectors to ${namespace}`);
|
||||
@ -405,6 +458,7 @@ const PGVector = {
|
||||
*/
|
||||
createTableIfNotExists: async function (connection, dimensions = 384) {
|
||||
this.log(`Creating embedding table with ${dimensions} dimensions`);
|
||||
await connection.query(this.createExtensionSql);
|
||||
await connection.query(this.createTableSql(dimensions));
|
||||
return true;
|
||||
},
|
||||
|
||||
Loading…
Reference in New Issue
Block a user