From 192ca411f261296cc503d1a8353450839a458fc1 Mon Sep 17 00:00:00 2001 From: Sean Hatfield Date: Mon, 23 Mar 2026 15:10:21 -0700 Subject: [PATCH] Telegram bot connector (#5190) * wip telegram bot connector * encrypt bot token, reorg telegram bot modules, secure pairing codes * offload telegram chat to background worker, add @agent support with chart png rendering, reconnect ui * refactor telegram bot settings page into subcomponents * response.locals for mum, telemetry for connecting to telegram * simplify telegram command registration * improve telegram bot ux: rework switch/history/resume commands * add voice, photo, and TTS support to telegram bot with long message handling * lint * rename external_connectors to external_communication_connectors, add voice response mode, persist chat workspace/thread selection * lint * fix telegram bot connect/disconnect bugs, kill telegram bot on multiuser mode enable * add english translations * fix qr code in light mode * repatch migration * WIP checkpoint * pipeline overhaul for using response obj * format functions * fix comment block * remove conditional dumpENV + lint * remove .end() from sendStatus calls * patch broken streaming where streaming only first chunk * refactor * use Ephemeral handler now * show metrics and citations in real GUI * bugfixes * prevent MuM persistence, UI cleanup, styling for status * add new workspace flow in UI Add thread chat count fix 69 byte payload callback limit bug * handle pagination for workspaces, threads, and models * modularize commands and navigation * add /proof support for citation recall * handle backlog message spam * support abort of response streams * code cleanup * spam prevention * fix translations, update voice typing indicator, fix token bug * frontend refactor, update tips on /status and voice response improvements * collapse agent though blocks * support images * Fix mime issues with audio from other devices * fix config issue post server stop * persist image on agentic chats * 5189 i18n (#5245) * i18n translations connect #5189 * prune translations * fix errors * fix translation gaps --------- Co-authored-by: Timothy Carambat --- collector/utils/constants.js | 10 + .../src/components/SettingsSidebar/index.jsx | 14 + .../src/hooks/useScrollActiveItemIntoView.js | 2 +- frontend/src/locales/ar/common.js | 89 +- frontend/src/locales/cs/common.js | 94 +- frontend/src/locales/da/common.js | 93 +- frontend/src/locales/de/common.js | 100 +- frontend/src/locales/en/common.js | 88 +- frontend/src/locales/es/common.js | 95 +- frontend/src/locales/et/common.js | 91 +- frontend/src/locales/fa/common.js | 91 +- .../src/locales/findUnusedTranslations.mjs | 12 +- frontend/src/locales/fr/common.js | 95 +- frontend/src/locales/he/common.js | 88 +- frontend/src/locales/it/common.js | 95 +- frontend/src/locales/ja/common.js | 90 +- frontend/src/locales/ko/common.js | 89 +- frontend/src/locales/lt/common.js | 93 +- frontend/src/locales/lv/common.js | 94 +- frontend/src/locales/nl/common.js | 95 +- frontend/src/locales/pl/common.js | 94 +- frontend/src/locales/pt_BR/common.js | 93 +- frontend/src/locales/ro/common.js | 96 +- frontend/src/locales/ru/common.js | 94 +- frontend/src/locales/tr/common.js | 92 +- frontend/src/locales/vn/common.js | 92 +- frontend/src/locales/zh/common.js | 86 +- frontend/src/locales/zh_TW/common.js | 86 +- frontend/src/main.jsx | 9 + frontend/src/models/telegram.js | 176 ++++ .../ConnectedView/UsersTable/index.jsx | 153 +++ .../TelegramBot/ConnectedView/index.jsx | 313 ++++++ .../SetupView/CreateBotSection/index.jsx | 118 +++ .../TelegramBot/SetupView/index.jsx | 159 ++++ .../Connections/TelegramBot/index.jsx | 97 ++ frontend/src/utils/paths.js | 3 + server/endpoints/telegram.js | 307 ++++++ server/index.js | 2 + server/jobs/handle-telegram-chat.js | 64 ++ .../models/externalCommunicationConnector.js | 110 +++ server/package.json | 3 + .../20260319202916_init/migration.sql | 12 + server/prisma/schema.prisma | 9 + server/utils/BackgroundWorkers/index.js | 44 + server/utils/EncryptionManager/index.js | 2 +- server/utils/agents/index.js | 36 + server/utils/boot/index.js | 3 + server/utils/middleware/multiUserProtected.js | 13 + server/utils/telegramBot/chat/agent.js | 377 ++++++++ server/utils/telegramBot/chat/stream.js | 469 +++++++++ server/utils/telegramBot/constants.js | 22 + server/utils/telegramBot/index.js | 632 +++++++++++++ .../utils/commands/handlers/handleAbort.js | 15 + .../utils/commands/handlers/handleHelp.js | 15 + .../utils/commands/handlers/handleHistory.js | 80 ++ .../commands/handlers/handleNewThread.js | 35 + .../utils/commands/handlers/handleProof.js | 202 ++++ .../utils/commands/handlers/handleReset.js | 32 + .../utils/commands/handlers/handleStart.js | 19 + .../utils/commands/handlers/handleStatus.js | 80 ++ .../utils/commands/handlers/showModelMenu.js | 98 ++ .../utils/commands/handlers/showThreadMenu.js | 120 +++ .../commands/handlers/showWorkspaceMenu.js | 84 ++ .../utils/telegramBot/utils/commands/index.js | 125 +++ server/utils/telegramBot/utils/format.js | 180 ++++ server/utils/telegramBot/utils/index.js | 203 ++++ server/utils/telegramBot/utils/media.js | 163 ++++ .../utils/telegramBot/utils/messageQueue.js | 53 ++ .../navigation/callbacks/handleBackSources.js | 16 + .../callbacks/handleBackWorkspaces.js | 18 + .../navigation/callbacks/handleModelCancel.js | 14 + .../callbacks/handleModelPagination.js | 24 + .../navigation/callbacks/handleModelSelect.js | 52 + .../callbacks/handleSourcePagination.js | 24 + .../callbacks/handleSourceSelect.js | 106 +++ .../callbacks/handleThreadPagination.js | 26 + .../callbacks/handleThreadSelect.js | 43 + .../callbacks/handleWorkspaceCreate.js | 33 + .../callbacks/handleWorkspacePagination.js | 26 + .../callbacks/handleWorkspaceSelect.js | 24 + .../utils/navigation/callbacks/index.js | 57 ++ .../telegramBot/utils/navigation/index.js | 35 + .../utils/telegramBot/utils/verification.js | 171 ++++ server/yarn.lock | 895 +++++++++++++++++- 84 files changed, 8209 insertions(+), 243 deletions(-) create mode 100644 frontend/src/models/telegram.js create mode 100644 frontend/src/pages/GeneralSettings/Connections/TelegramBot/ConnectedView/UsersTable/index.jsx create mode 100644 frontend/src/pages/GeneralSettings/Connections/TelegramBot/ConnectedView/index.jsx create mode 100644 frontend/src/pages/GeneralSettings/Connections/TelegramBot/SetupView/CreateBotSection/index.jsx create mode 100644 frontend/src/pages/GeneralSettings/Connections/TelegramBot/SetupView/index.jsx create mode 100644 frontend/src/pages/GeneralSettings/Connections/TelegramBot/index.jsx create mode 100644 server/endpoints/telegram.js create mode 100644 server/jobs/handle-telegram-chat.js create mode 100644 server/models/externalCommunicationConnector.js create mode 100644 server/prisma/migrations/20260319202916_init/migration.sql create mode 100644 server/utils/telegramBot/chat/agent.js create mode 100644 server/utils/telegramBot/chat/stream.js create mode 100644 server/utils/telegramBot/constants.js create mode 100644 server/utils/telegramBot/index.js create mode 100644 server/utils/telegramBot/utils/commands/handlers/handleAbort.js create mode 100644 server/utils/telegramBot/utils/commands/handlers/handleHelp.js create mode 100644 server/utils/telegramBot/utils/commands/handlers/handleHistory.js create mode 100644 server/utils/telegramBot/utils/commands/handlers/handleNewThread.js create mode 100644 server/utils/telegramBot/utils/commands/handlers/handleProof.js create mode 100644 server/utils/telegramBot/utils/commands/handlers/handleReset.js create mode 100644 server/utils/telegramBot/utils/commands/handlers/handleStart.js create mode 100644 server/utils/telegramBot/utils/commands/handlers/handleStatus.js create mode 100644 server/utils/telegramBot/utils/commands/handlers/showModelMenu.js create mode 100644 server/utils/telegramBot/utils/commands/handlers/showThreadMenu.js create mode 100644 server/utils/telegramBot/utils/commands/handlers/showWorkspaceMenu.js create mode 100644 server/utils/telegramBot/utils/commands/index.js create mode 100644 server/utils/telegramBot/utils/format.js create mode 100644 server/utils/telegramBot/utils/index.js create mode 100644 server/utils/telegramBot/utils/media.js create mode 100644 server/utils/telegramBot/utils/messageQueue.js create mode 100644 server/utils/telegramBot/utils/navigation/callbacks/handleBackSources.js create mode 100644 server/utils/telegramBot/utils/navigation/callbacks/handleBackWorkspaces.js create mode 100644 server/utils/telegramBot/utils/navigation/callbacks/handleModelCancel.js create mode 100644 server/utils/telegramBot/utils/navigation/callbacks/handleModelPagination.js create mode 100644 server/utils/telegramBot/utils/navigation/callbacks/handleModelSelect.js create mode 100644 server/utils/telegramBot/utils/navigation/callbacks/handleSourcePagination.js create mode 100644 server/utils/telegramBot/utils/navigation/callbacks/handleSourceSelect.js create mode 100644 server/utils/telegramBot/utils/navigation/callbacks/handleThreadPagination.js create mode 100644 server/utils/telegramBot/utils/navigation/callbacks/handleThreadSelect.js create mode 100644 server/utils/telegramBot/utils/navigation/callbacks/handleWorkspaceCreate.js create mode 100644 server/utils/telegramBot/utils/navigation/callbacks/handleWorkspacePagination.js create mode 100644 server/utils/telegramBot/utils/navigation/callbacks/handleWorkspaceSelect.js create mode 100644 server/utils/telegramBot/utils/navigation/callbacks/index.js create mode 100644 server/utils/telegramBot/utils/navigation/index.js create mode 100644 server/utils/telegramBot/utils/verification.js diff --git a/collector/utils/constants.js b/collector/utils/constants.js index 08ab2f37..4813e6e5 100644 --- a/collector/utils/constants.js +++ b/collector/utils/constants.js @@ -27,6 +27,11 @@ const ACCEPTED_MIMES = { "audio/wav": [".wav"], "audio/mpeg": [".mp3"], + "audio/ogg": [".ogg", ".oga"], + "audio/opus": [".opus"], + "audio/mp4": [".m4a"], + "audio/x-m4a": [".m4a"], + "audio/webm": [".webm"], "video/mp4": [".mp4"], "video/mpeg": [".mpeg"], @@ -68,6 +73,11 @@ const SUPPORTED_FILETYPE_CONVERTERS = { ".wav": "./convert/asAudio.js", ".mp4": "./convert/asAudio.js", ".mpeg": "./convert/asAudio.js", + ".ogg": "./convert/asAudio.js", + ".oga": "./convert/asAudio.js", + ".opus": "./convert/asAudio.js", + ".m4a": "./convert/asAudio.js", + ".webm": "./convert/asAudio.js", ".png": "./convert/asImage.js", ".jpg": "./convert/asImage.js", diff --git a/frontend/src/components/SettingsSidebar/index.jsx b/frontend/src/components/SettingsSidebar/index.jsx index d0e72a5c..e61de158 100644 --- a/frontend/src/components/SettingsSidebar/index.jsx +++ b/frontend/src/components/SettingsSidebar/index.jsx @@ -10,6 +10,7 @@ import { PencilSimpleLine, Nut, Toolbox, + Plugs, } from "@phosphor-icons/react"; import AgentIcon from "@/media/animations/agent-static.png"; import CommunityHubIcon from "@/media/illustrations/community-hub.png"; @@ -363,6 +364,19 @@ const SidebarOptions = ({ user = null, t }) => ( }, ]} /> +