Clear cached docs on text splitter changes (#4163)

* clear cached docs on text splitter changes

* update warning message and title

* remove unsused translation keys

* wrap trycatch

---------

Co-authored-by: timothycarambat <rambat1010@gmail.com>

frontend/src/locales/ar/common.js

@@ -416,9 +416,6 @@ const TRANSLATIONS = {
       "في بعض الأحيان، قد ترغب في تغيير الطريقة الافتراضية التي يتم بها تقسيم المستندات الجديدة وتقطيعها قبل إدراجها في قاعدة بيانات المتجهة الخاصة بك.",
     "desc-end":
       "يجب عليك فقط تعديل هذا الإعداد إذا كنت تفهم كيفية عمل تقسيم النص وتأثيراته الجانبية.",
-    "warn-start": "التغييرات هنا سوف تنطبق فقط على",
-    "warn-center": "المستندات المضمنة حديثًا",
-    "warn-end": "، وليس على المستندات الموجودة.",
     size: {
       title: "حجم قطعة النص",
       description:

frontend/src/locales/da/common.js

@@ -418,9 +418,6 @@ const TRANSLATIONS = {
       "Nogle gange vil du måske ændre den standardmåde, som nye dokumenter deles og opdeles i bidder, inden de indsættes i din vektordatabase.",
     "desc-end":
       "Du bør kun ændre denne indstilling, hvis du forstår, hvordan tekstopdeling fungerer og dens bivirkninger.",
-    "warn-start": "Ændringer her vil kun gælde for",
-    "warn-center": "nyligt indlejrede dokumenter",
-    "warn-end": ", ikke eksisterende dokumenter.",
     size: {
       title: "Størrelse på tekstbidder",
       description:

frontend/src/locales/de/common.js

@@ -607,9 +607,6 @@ const TRANSLATIONS = {
       "Manchmal möchten Sie vielleicht die Standardmethode ändern, wie neue Dokumente gesplittet und gechunkt werden, bevor sie in Ihre Vektordatenbank eingefügt werden.",
     "desc-end":
       "Sie sollten diese Einstellung nur ändern, wenn Sie verstehen, wie Textsplitting funktioniert und welche Nebenwirkungen es hat.",
-    "warn-start": "Änderungen hier gelten nur für",
-    "warn-center": "neu eingebettete Dokumente",
-    "warn-end": ", nicht für bestehende Dokumente.",
     size: {
       title: "Textchunk-Größe",
       description:

frontend/src/locales/en/common.js

@@ -631,9 +631,6 @@ const TRANSLATIONS = {
       "Sometimes, you may want to change the default way that new documents are split and chunked before being inserted into your vector database.",
     "desc-end":
       "You should only modify this setting if you understand how text splitting works and it's side effects.",
-    "warn-start": "Changes here will only apply to",
-    "warn-center": "newly embedded documents",
-    "warn-end": ", not existing documents.",
     size: {
       title: "Text Chunk Size",
       description:

frontend/src/locales/es/common.js

@@ -415,9 +415,6 @@ const TRANSLATIONS = {
       "A veces, es posible que desees cambiar la forma predeterminada en que los nuevos documentos se dividen y fragmentan antes de ser insertados en tu base de datos de vectores.",
     "desc-end":
       "Solo debes modificar esta configuración si entiendes cómo funciona la división de texto y sus efectos secundarios.",
-    "warn-start": "Los cambios aquí solo se aplicarán a",
-    "warn-center": "documentos recién incrustados",
-    "warn-end": ", no a los documentos existentes.",
     size: {
       title: "Tamaño del fragmento de texto",
       description:

frontend/src/locales/et/common.js

@@ -582,9 +582,6 @@ const TRANSLATIONS = {
     "desc-start":
       "Vahel soovid muuta, kuidas uued dokumendid enne vektoriandmebaasi lisamist tükeldatakse.",
     "desc-end": "Muuda seda ainult siis, kui mõistad tekstilõike mõju.",
-    "warn-start": "Muudatused kehtivad ainult",
-    "warn-center": "uutele dokumentidele",
-    "warn-end": ", mitte olemasolevatele.",
     size: {
       title: "Tekstitüki suurus",
       description: "Maksimaalne märgipikkus ühes vektoris.",

frontend/src/locales/fa/common.js

@@ -408,9 +408,6 @@ const TRANSLATIONS = {
       "تقسیم متن به شما امکان می‌دهد اسناد بزرگ را به بخش‌های کوچک‌تر تقسیم کنید که برای جاسازی و پردازش مناسب‌تر هستند.",
     "desc-end":
       "سعی کنید تعادلی بین اندازه بخش و همپوشانی ایجاد کنید تا از دست رفتن اطلاعات را به حداقل برسانید.",
-    "warn-start": "تغییر این مقادیر نیاز به",
-    "warn-center": "پردازش مجدد تمام اسناد",
-    "warn-end": "خواهد داشت.",
     size: {
       title: "حداکثر اندازه بخش",
       description:

frontend/src/locales/fr/common.js

@@ -416,9 +416,6 @@ const TRANSLATIONS = {
       "Parfois, vous voudrez peut-être changer la façon dont les nouveaux documents sont divisés et découpés avant d'être insérés dans votre base de données vectorielle.",
     "desc-end":
       "Vous ne devez modifier ce paramètre que si vous comprenez comment fonctionne la division du texte et ses effets secondaires.",
-    "warn-start": "Les changements ici s'appliqueront uniquement aux",
-    "warn-center": "nouveaux documents intégrés",
-    "warn-end": ", pas aux documents existants.",
     size: {
       title: "Taille des segments de texte",
       description:

frontend/src/locales/he/common.js

@@ -404,9 +404,6 @@ const TRANSLATIONS = {
       "לפעמים, ייתכן שתרצה לשנות את הדרך ברירת המחדל שבה מסמכים חדשים מחולקים ומופרדים לפני שהם מוכנסים לבסיס הנתונים הווקטורי שלך.",
     "desc-end":
       "עליך לשנות הגדרה זו רק אם אתה מבין כיצד חלוקת טקסט פועלת והשפעותיה.",
-    "warn-start": "שינויים כאן יחולו רק על",
-    "warn-center": "מסמכים שהוטמעו לאחרונה",
-    "warn-end": ", לא על מסמכים קיימים.",
     size: {
       title: "גודל קטע טקסט",
       description: "זהו אורך הדמויות המרבי שיכול להיות נוכח בקטור יחיד.",

frontend/src/locales/it/common.js

@@ -414,9 +414,6 @@ const TRANSLATIONS = {
       "A volte, potresti voler cambiare il modo predefinito in cui i nuovi documenti vengono suddivisi e spezzettati in blocchi prima di essere inseriti nel tuo database vettoriale.",
     "desc-end":
       "Dovresti modificare questa impostazione solo se capisci come funziona la suddivisione del testo e i suoi effetti collaterali.",
-    "warn-start": "Le modifiche qui si applicheranno solo a",
-    "warn-center": "nuovi documenti incorporati",
-    "warn-end": ", non documenti esistenti.",
     size: {
       title: "Dimensioni blocco di testo",
       description:

frontend/src/locales/ja/common.js

@@ -416,9 +416,6 @@ const TRANSLATIONS = {
       "新しいドキュメントがベクトルデータベースに挿入される前に、どのように分割およびチャンク化されるかのデフォルトの方法を変更する場合があります。",
     "desc-end":
       "テキスト分割の仕組みとその副作用を理解している場合にのみ、この設定を変更するべきです。",
-    "warn-start": "ここでの変更は、",
-    "warn-center": "新しく埋め込まれるドキュメントにのみ適用されます",
-    "warn-end": "、既存のドキュメントには適用されません。",
     size: {
       title: "テキストチャンクサイズ",
       description: "1つのベクトルに含まれる最大の文字数です。",

frontend/src/locales/ko/common.js

@@ -591,9 +591,6 @@ const TRANSLATIONS = {
       "새 문서를 벡터 데이터베이스에 삽입하기 전에 기본 텍스트 분할 방식을 변경할 수 있습니다.",
     "desc-end":
       "텍스트 분할 방식과 그 영향을 이해하고 있는 경우에만 이 설정을 변경해야 합니다.",
-    "warn-start": "여기의 변경 사항은",
-    "warn-center": "새로 임베딩되는 문서",
-    "warn-end": "에만 적용됩니다. 기존 문서에는 적용되지 않습니다.",
     size: {
       title: "텍스트 청크 크기",
       description: "단일 벡터에 들어갈 수 있는 최대 문자 길이입니다.",

frontend/src/locales/lv/common.js

@@ -602,9 +602,6 @@ const TRANSLATIONS = {
       "Dažreiz jūs, iespējams, vēlēsieties mainīt noklusējuma veidu, kā jauni dokumenti tiek sadalīti un sagatavoti pirms ievietošanas vektoru datubāzē.",
     "desc-end":
       "Jums vajadzētu mainīt šo iestatījumu tikai tad, ja saprotat, kā darbojas teksta sadalīšana un tās blakusefekti.",
-    "warn-start": "Šeit veiktās izmaiņas attieksies tikai uz",
-    "warn-center": "jauniem iegultajiem dokumentiem",
-    "warn-end": ", nevis esošajiem dokumentiem.",
     size: {
       title: "Teksta gabala izmērs",
       description:

frontend/src/locales/nl/common.js

@@ -411,9 +411,6 @@ const TRANSLATIONS = {
       "Soms wil je misschien de standaard manier wijzigen waarop nieuwe documenten worden gesplitst en gechunkt voordat ze in je vector database worden ingevoerd.",
     "desc-end":
       "Je moet deze instelling alleen wijzigen als je begrijpt hoe tekstsplitsing werkt en de bijbehorende effecten.",
-    "warn-start": "Wijzigingen hier zijn alleen van toepassing op",
-    "warn-center": "nieuw ingebedde documenten",
-    "warn-end": ", niet op bestaande documenten.",
     size: {
       title: "Tekst Chunk Grootte",
       description:

frontend/src/locales/pl/common.js

@@ -607,9 +607,6 @@ const TRANSLATIONS = {
       "Czasami może zaistnieć potrzeba zmiany domyślnego sposobu, w jaki nowe dokumenty są dzielone i fragmentowane przed wstawieniem ich do wektorowej bazy danych.",
     "desc-end":
       "Powinieneś modyfikować to ustawienie tylko wtedy, gdy rozumiesz, jak działa dzielenie tekstu i jakie są jego skutki uboczne.",
-    "warn-start": "Zmiany tutaj będą miały zastosowanie tylko do",
-    "warn-center": "nowo osadzone dokumenty",
-    "warn-end": "a nie istniejące dokumenty.",
     size: {
       title: "Rozmiar fragmentu tekstu",
       description:

frontend/src/locales/pt_BR/common.js

@@ -588,9 +588,6 @@ const TRANSLATIONS = {
     "desc-start":
       "Você pode alterar a forma como novos documentos são divididos antes de serem inseridos no banco de dados vetorial.",
     "desc-end": "Modifique apenas se entender os efeitos da divisão de texto.",
-    "warn-start": "Alterações afetarão apenas",
-    "warn-center": "documentos novos",
-    "warn-end": ", não os existentes.",
     size: {
       title: "Tamanho dos Trechos",
       description: "Comprimento máximo de caracteres em um único vetor.",

frontend/src/locales/ru/common.js

@@ -420,9 +420,6 @@ const TRANSLATIONS = {
       "Иногда может понадобиться изменить стандартный способ разделения и сегментации новых документов перед их вставкой в векторную базу данных.",
     "desc-end":
       "Следует изменять этот параметр только при полном понимании работы разделения текста и его побочных эффектов.",
-    "warn-start": "Изменения здесь будут применяться только к",
-    "warn-center": "новым встроенным документам",
-    "warn-end": ", а не к существующим документам.",
     size: {
       title: "Размер сегмента текста",
       description:

frontend/src/locales/tr/common.js

@@ -411,9 +411,6 @@ const TRANSLATIONS = {
       "Bazı durumlarda, yeni belgelerin vektör veritabanınıza eklenmeden önce hangi varsayılan yöntemle bölünüp parçalanacağını değiştirmek isteyebilirsiniz.",
     "desc-end":
       "Metin bölmenin nasıl çalıştığını ve olası yan etkilerini tam olarak bilmiyorsanız bu ayarı değiştirmemelisiniz.",
-    "warn-start": "Buradaki değişiklikler yalnızca",
-    "warn-center": "yeni eklenen belgeler",
-    "warn-end": "için geçerli olacak, mevcut belgelere uygulanmaz.",
     size: {
       title: "Metin Parça Boyutu",
       description:

frontend/src/locales/vn/common.js

@@ -410,9 +410,6 @@ const TRANSLATIONS = {
       "Sometimes, you may want to change the default way that new documents are split and chunked before being inserted into your vector database.",
     "desc-end":
       "You should only modify this setting if you understand how text splitting works and it's side effects.",
-    "warn-start": "Changes here will only apply to",
-    "warn-center": "newly embedded documents",
-    "warn-end": ", not existing documents.",
     size: {
       title: "Text Chunk Size",
       description:

frontend/src/locales/zh/common.js

@@ -565,9 +565,6 @@ const TRANSLATIONS = {
     "desc-start":
       "有时，你可能希望更改新文档在插入到矢量数据库之前拆分和分块的默认方式。",
     "desc-end": "只有在了解文本拆分的工作原理及其副作用时，才应修改此设置。",
-    "warn-start": "此处的更改仅适用于",
-    "warn-center": "新嵌入的文档",
-    "warn-end": "，而不是现有文档。",
     size: {
       title: "文本块大小",
       description: "这是单个向量中可以存在的字符的最大长度。",

frontend/src/locales/zh_TW/common.js

@@ -398,9 +398,6 @@ const TRANSLATIONS = {
       "有時您可能需要修改新文件在插入向量資料庫之前的預設分割和區塊化方式。",
     "desc-end":
       "只有在了解文字分割的運作方式及其副作用的情況下，才應該修改此設定。",
-    "warn-start": "此處的修改只會套用至",
-    "warn-center": "新嵌入的文件",
-    "warn-end": "，而不會套用至現有文件。",
     size: {
       title: "文字區塊大小",
       description: "這是單一向量中可包含的最大字元長度。",

frontend/src/pages/GeneralSettings/EmbeddingTextSplitterPreference/index.jsx

@@ -5,8 +5,11 @@ import PreLoader from "@/components/Preloader";
 import CTAButton from "@/components/lib/CTAButton";
 import Admin from "@/models/admin";
 import showToast from "@/utils/toast";
-import { nFormatter, numberWithCommas } from "@/utils/numbers";
+import { numberWithCommas } from "@/utils/numbers";
 import { useTranslation } from "react-i18next";
+import { useModal } from "@/hooks/useModal";
+import ModalWrapper from "@/components/ModalWrapper";
+import ChangeWarningModal from "@/components/ChangeWarning";
 
 function isNullOrNaN(value) {
   if (value === null) return true;
@@ -18,6 +21,7 @@ export default function EmbeddingTextSplitterPreference() {
   const [loading, setLoading] = useState(true);
   const [saving, setSaving] = useState(false);
   const [hasChanges, setHasChanges] = useState(false);
+  const { isOpen, openModal, closeModal } = useModal();
   const { t } = useTranslation();
 
   const handleSubmit = async (e) => {
@@ -35,22 +39,35 @@ export default function EmbeddingTextSplitterPreference() {
       return;
     }
 
+    openModal();
+  };
+
+  const handleSaveSettings = async () => {
     setSaving(true);
-    await Admin.updateSystemPreferences({
+    try {
-      text_splitter_chunk_size: isNullOrNaN(
+      const form = new FormData(
-        form.get("text_splitter_chunk_size")
+        document.getElementById("text-splitter-chunking-form")
-      )
+      );
-        ? 1000
+      await Admin.updateSystemPreferences({
-        : Number(form.get("text_splitter_chunk_size")),
+        text_splitter_chunk_size: isNullOrNaN(
-      text_splitter_chunk_overlap: isNullOrNaN(
+          form.get("text_splitter_chunk_size")
-        form.get("text_splitter_chunk_overlap")
+        )
-      )
+          ? 1000
-        ? 1000
+          : Number(form.get("text_splitter_chunk_size")),
-        : Number(form.get("text_splitter_chunk_overlap")),
+        text_splitter_chunk_overlap: isNullOrNaN(
-    });
+          form.get("text_splitter_chunk_overlap")
-    setSaving(false);
+        )
-    setHasChanges(false);
+          ? 1000
-    showToast("Text chunking strategy settings saved.", "success");
+          : Number(form.get("text_splitter_chunk_overlap")),
+      });
+      setHasChanges(false);
+      closeModal();
+      showToast("Text chunking strategy settings saved.", "success");
+    } catch (error) {
+      showToast("Failed to save text chunking strategy settings.", "error");
+    } finally {
+      setSaving(false);
+    }
   };
 
   useEffect(() => {
@@ -83,6 +100,7 @@ export default function EmbeddingTextSplitterPreference() {
             onSubmit={handleSubmit}
             onChange={() => setHasChanges(true)}
             className="flex w-full"
+            id="text-splitter-chunking-form"
           >
             <div className="flex flex-col w-full px-1 md:pl-6 md:pr-[50px] md:py-6 py-16">
               <div className="w-full flex flex-col gap-y-1 pb-4 border-white light:border-theme-sidebar-border border-b-2 border-opacity-10">
@@ -95,10 +113,6 @@ export default function EmbeddingTextSplitterPreference() {
                   {t("text.desc-start")} <br />
                   {t("text.desc-end")}
                 </p>
-                <p className="text-xs leading-[18px] font-semibold text-white/80">
-                  {t("text.warn-start")} <i>{t("text.warn-center")}</i>
-                  {t("text.warn-end")}
-                </p>
               </div>
               <div className="w-full justify-end flex">
                 {hasChanges && (
@@ -172,6 +186,14 @@ export default function EmbeddingTextSplitterPreference() {
           </form>
         </div>
       )}
+
+      <ModalWrapper isOpen={isOpen}>
+        <ChangeWarningModal
+          warningText="Changing text splitter settings will clear any previously cached documents.\n\nThese new settings will be applied to all documents when embedding them into a workspace."
+          onClose={closeModal}
+          onConfirm={handleSaveSettings}
+        />
+      </ModalWrapper>
     </div>
   );
 }

server/models/systemSettings.js

@@ -73,6 +73,8 @@ const SystemSettings = {
       try {
         if (isNullOrNaN(update)) throw new Error("Value is not a number.");
         if (Number(update) <= 0) throw new Error("Value must be non-zero.");
+        const { purgeEntireVectorCache } = require("../utils/files");
+        purgeEntireVectorCache();
         return Number(update);
       } catch (e) {
         console.error(
@@ -86,6 +88,8 @@ const SystemSettings = {
       try {
         if (isNullOrNaN(update)) throw new Error("Value is not a number");
         if (Number(update) < 0) throw new Error("Value cannot be less than 0.");
+        const { purgeEntireVectorCache } = require("../utils/files");
+        purgeEntireVectorCache();
         return Number(update);
       } catch (e) {
         console.error(