michael/merlyn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:Mintplex-Labs/anything-llm

Browse Source
This commit is contained in:
timothycarambat 2025-04-28 09:14:17 -07:00
commit 0b8e89f6a7
54 changed files with 2648 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/dev-build.yaml vendored
View File

@ -6,7 +6,7 @@ concurrency:
on:
push:
branches: ['na'] # put your current branch to create a build. Core team only.
branches: ['3698-main-screen-localization'] # put your current branch to create a build. Core team only.
paths-ignore:
- '**.md'
- 'cloud-deployments/*'

26
collector/extensions/index.js
View File

@ -154,6 +154,32 @@ function extensions(app) {
return;
}
);
app.post(
"/ext/drupalwiki",
[verifyPayloadIntegrity, setDataSigner],
async function (request, response) {
try {
const { loadAndStoreSpaces } = require("../utils/extensions/DrupalWiki");
const { success, reason, data } = await loadAndStoreSpaces(
reqBody(request),
response
);
response.status(200).json({ success, reason, data });
} catch (e) {
console.error(e);
response.status(400).json({
success: false,
reason: e.message,
data: {
title: null,
author: null,
},
});
}
return;
}
);
}
module.exports = extensions;

53
collector/extensions/resync/index.js
View File

@ -2,7 +2,7 @@ const { getLinkText } = require("../../processLink");
/**
* Fetches the content of a raw link. Returns the content as a text string of the link in question.
* @param {object} data - metadata from document (eg: link)
* @param {object} data - metadata from document (eg: link)
* @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
*/
async function resyncLink({ link }, response) {
@ -24,7 +24,7 @@ async function resyncLink({ link }, response) {
* Fetches the content of a YouTube link. Returns the content as a text string of the video in question.
* We offer this as there may be some videos where a transcription could be manually edited after initial scraping
* but in general - transcriptions often never change.
* @param {object} data - metadata from document (eg: link)
* @param {object} data - metadata from document (eg: link)
* @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
*/
async function resyncYouTube({ link }, response) {
@ -44,9 +44,9 @@ async function resyncYouTube({ link }, response) {
}
/**
* Fetches the content of a specific confluence page via its chunkSource.
* Fetches the content of a specific confluence page via its chunkSource.
* Returns the content as a text string of the page in question and only that page.
* @param {object} data - metadata from document (eg: chunkSource)
* @param {object} data - metadata from document (eg: chunkSource)
* @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
*/
async function resyncConfluence({ chunkSource }, response) {
@ -76,9 +76,9 @@ async function resyncConfluence({ chunkSource }, response) {
}
/**
* Fetches the content of a specific confluence page via its chunkSource.
* Fetches the content of a specific confluence page via its chunkSource.
* Returns the content as a text string of the page in question and only that page.
* @param {object} data - metadata from document (eg: chunkSource)
* @param {object} data - metadata from document (eg: chunkSource)
* @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
*/
async function resyncGithub({ chunkSource }, response) {
@ -106,9 +106,48 @@ async function resyncGithub({ chunkSource }, response) {
}
}
/**
* Fetches the content of a specific DrupalWiki page via its chunkSource.
* Returns the content as a text string of the page in question and only that page.
* @param {object} data - metadata from document (eg: chunkSource)
* @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
*/
async function resyncDrupalWiki({ chunkSource }, response) {
if (!chunkSource) throw new Error('Invalid source property provided');
try {
// DrupalWiki data is `payload` encrypted. So we need to expand its
// encrypted payload back into query params so we can reFetch the page with same access token/params.
const source = response.locals.encryptionWorker.expandPayload(chunkSource);
const { loadPage } = require("../../utils/extensions/DrupalWiki");
const { success, reason, content } = await loadPage({
baseUrl: source.searchParams.get('baseUrl'),
pageId: source.searchParams.get('pageId'),
accessToken: source.searchParams.get('accessToken'),
});
if (!success) {
console.error(`Failed to sync DrupalWiki page content. ${reason}`);
response.status(200).json({
success: false,
content: null,
});
} else {
response.status(200).json({ success, content });
}
} catch (e) {
console.error(e);
response.status(200).json({
success: false,
content: null,
});
}
}
module.exports = {
link: resyncLink,
youtube: resyncYouTube,
confluence: resyncConfluence,
github: resyncGithub,
}
drupalwiki: resyncDrupalWiki,
}

8
collector/index.js
View File

@ -62,9 +62,13 @@ app.post(
"/process-link",
[verifyPayloadIntegrity],
async function (request, response) {
const { link } = reqBody(request);
const { link, scraperHeaders = {} } = reqBody(request);
try {
const { success, reason, documents = [] } = await processLink(link);
const {
success,
reason,
documents = [],
} = await processLink(link, scraperHeaders);
response.status(200).json({ url: link, success, reason, documents });
} catch (e) {
console.error(e);

9
collector/middleware/verifyIntegrity.js
View File

@ -1,9 +1,12 @@
const { CommunicationKey } = require("../utils/comKey");
const RuntimeSettings = require("../utils/runtimeSettings");
const runtimeSettings = new RuntimeSettings();
function verifyPayloadIntegrity(request, response, next) {
const comKey = new CommunicationKey();
if (process.env.NODE_ENV === "development") {
comKey.log('verifyPayloadIntegrity is skipped in development.')
comKey.log('verifyPayloadIntegrity is skipped in development.');
runtimeSettings.parseOptionsFromRequest(request);
next();
return;
}
@ -12,7 +15,9 @@ function verifyPayloadIntegrity(request, response, next) {
if (!signature) return response.status(400).json({ msg: 'Failed integrity signature check.' })
const validSignedPayload = comKey.verify(signature, request.body);
if (!validSignedPayload) return response.status(400).json({ msg: 'Failed integrity signature check.' })
if (!validSignedPayload) return response.status(400).json({ msg: 'Failed integrity signature check.' });
runtimeSettings.parseOptionsFromRequest(request);
next();
}

84
collector/processLink/convert/generic.js
View File

@ -8,18 +8,25 @@ const { default: slugify } = require("slugify");
/**
* Scrape a generic URL and return the content in the specified format
* @param {string} link - The URL to scrape
* @param {('html' | 'text')} captureAs - The format to capture the page content as
* @param {boolean} processAsDocument - Whether to process the content as a document or return the content directly
* @param {Object} config - The configuration object
* @param {string} config.link - The URL to scrape
* @param {('html' | 'text')} config.captureAs - The format to capture the page content as. Default is 'text'
* @param {boolean} config.processAsDocument - Whether to process the content as a document or return the content directly. Default is true
* @param {{[key: string]: string}} config.scraperHeaders - Custom headers to use when making the request
* @returns {Promise<Object>} - The content of the page
*/
async function scrapeGenericUrl(
async function scrapeGenericUrl({
link,
captureAs = "text",
processAsDocument = true
) {
processAsDocument = true,
scraperHeaders = {},
}) {
console.log(`-- Working URL ${link} => (${captureAs}) --`);
const content = await getPageContent(link, captureAs);
const content = await getPageContent({
link,
captureAs,
headers: scraperHeaders,
});
if (!content.length) {
console.error(`Resulting URL content was empty at ${link}.`);
@ -63,13 +70,38 @@ async function scrapeGenericUrl(
return { success: true, reason: null, documents: [document] };
}
/**
* Validate the headers object
* - Keys & Values must be strings and not empty
* - Assemble a new object with only the valid keys and values
* @param {{[key: string]: string}} headers - The headers object to validate
* @returns {{[key: string]: string}} - The validated headers object
*/
function validatedHeaders(headers = {}) {
try {
if (Object.keys(headers).length === 0) return {};
let validHeaders = {};
for (const key of Object.keys(headers)) {
if (!key?.trim()) continue;
if (typeof headers[key] !== "string" || !headers[key]?.trim()) continue;
validHeaders[key] = headers[key].trim();
}
return validHeaders;
} catch (error) {
console.error("Error validating headers", error);
return {};
}
}
/**
* Get the content of a page
* @param {string} link - The URL to get the content of
* @param {('html' | 'text')} captureAs - The format to capture the page content as
* @param {Object} config - The configuration object
* @param {string} config.link - The URL to get the content of
* @param {('html' | 'text')} config.captureAs - The format to capture the page content as. Default is 'text'
* @param {{[key: string]: string}} config.headers - Custom headers to use when making the request
* @returns {Promise<string>} - The content of the page
*/
async function getPageContent(link, captureAs = "text") {
async function getPageContent({ link, captureAs = "text", headers = {} }) {
try {
let pageContents = [];
const loader = new PuppeteerWebBaseLoader(link, {
@ -91,12 +123,37 @@ async function getPageContent(link, captureAs = "text") {
},
});
const docs = await loader.load();
// Override scrape method if headers are available
let overrideHeaders = validatedHeaders(headers);
if (Object.keys(overrideHeaders).length > 0) {
loader.scrape = async function () {
const { launch } = await PuppeteerWebBaseLoader.imports();
const browser = await launch({
headless: "new",
defaultViewport: null,
ignoreDefaultArgs: ["--disable-extensions"],
...this.options?.launchOptions,
});
const page = await browser.newPage();
await page.setExtraHTTPHeaders(overrideHeaders);
for (const doc of docs) {
pageContents.push(doc.pageContent);
await page.goto(this.webPath, {
timeout: 180000,
waitUntil: "networkidle2",
...this.options?.gotoOptions,
});
const bodyHTML = this.options?.evaluate
? await this.options.evaluate(page, browser)
: await page.evaluate(() => document.body.innerHTML);
await browser.close();
return bodyHTML;
};
}
const docs = await loader.load();
for (const doc of docs) pageContents.push(doc.pageContent);
return pageContents.join(" ");
} catch (error) {
console.error(
@ -112,6 +169,7 @@ async function getPageContent(link, captureAs = "text") {
"Content-Type": "text/plain",
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36,gzip(gfe)",
...validatedHeaders(headers),
},
}).then((res) => res.text());
return pageText;

25
collector/processLink/index.js
View File

@ -1,20 +1,37 @@
const { validURL } = require("../utils/url");
const { scrapeGenericUrl } = require("./convert/generic");
async function processLink(link) {
/**
* Process a link and return the text content. This util will save the link as a document
* so it can be used for embedding later.
* @param {string} link - The link to process
* @param {{[key: string]: string}} scraperHeaders - Custom headers to apply when scraping the link
* @returns {Promise<{success: boolean, content: string}>} - Response from collector
*/
async function processLink(link, scraperHeaders = {}) {
if (!validURL(link)) return { success: false, reason: "Not a valid URL." };
return await scrapeGenericUrl(link);
return await scrapeGenericUrl({
link,
captureAs: "text",
processAsDocument: true,
scraperHeaders,
});
}
/**
* Get the text content of a link
* Get the text content of a link - does not save the link as a document
* Mostly used in agentic flows/tools calls to get the text content of a link
* @param {string} link - The link to get the text content of
* @param {('html' | 'text' | 'json')} captureAs - The format to capture the page content as
* @returns {Promise<{success: boolean, content: string}>} - Response from collector
*/
async function getLinkText(link, captureAs = "text") {
if (!validURL(link)) return { success: false, reason: "Not a valid URL." };
return await scrapeGenericUrl(link, captureAs, false);
return await scrapeGenericUrl({
link,
captureAs,
processAsDocument: false,
});
}
module.exports = {

320
collector/utils/extensions/DrupalWiki/DrupalWiki/index.js Normal file
View File

@ -0,0 +1,320 @@
/**
* Copyright 2024
*
* Authors:
* - Eugen Mayer (KontextWork)
*/
const { htmlToText } = require("html-to-text");
const { tokenizeString } = require("../../../tokenizer");
const { sanitizeFileName, writeToServerDocuments } = require("../../../files");
const { default: slugify } = require("slugify");
const path = require("path");
const fs = require("fs");
const { processSingleFile } = require("../../../../processSingleFile");
const {
WATCH_DIRECTORY,
SUPPORTED_FILETYPE_CONVERTERS,
} = require("../../../constants");
class Page {
/**
*
* @param {number }id
* @param {string }title
* @param {string} created
* @param {string} type
* @param {string} processedBody
* @param {string} url
* @param {number} spaceId
*/
constructor({ id, title, created, type, processedBody, url, spaceId }) {
this.id = id;
this.title = title;
this.url = url;
this.created = created;
this.type = type;
this.processedBody = processedBody;
this.spaceId = spaceId;
}
}
class DrupalWiki {
/**
*
* @param baseUrl
* @param spaceId
* @param accessToken
*/
constructor({ baseUrl, accessToken }) {
this.baseUrl = baseUrl;
this.accessToken = accessToken;
this.storagePath = this.#prepareStoragePath(baseUrl);
}
/**
* Load all pages for the given space, fetching storing each page one by one
* to minimize the memory usage
*
* @param {number} spaceId
* @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker
* @returns {Promise<void>}
*/
async loadAndStoreAllPagesForSpace(spaceId, encryptionWorker) {
const pageIndex = await this.#getPageIndexForSpace(spaceId);
for (const pageId of pageIndex) {
try {
const page = await this.loadPage(pageId);
// Pages with an empty body will lead to embedding issues / exceptions
if (page.processedBody.trim() !== "") {
this.#storePage(page, encryptionWorker);
await this.#downloadAndProcessAttachments(page.id);
} else {
console.log(`Skipping page (${page.id}) since it has no content`);
}
} catch (e) {
console.error(
`Could not process DrupalWiki page ${pageId} (skipping and continuing): `
);
console.error(e);
}
}
}
/**
* @param {number} pageId
* @returns {Promise<Page>}
*/
async loadPage(pageId) {
return this.#fetchPage(pageId);
}
/**
* Fetches the page ids for the configured space
* @param {number} spaceId
* @returns{Promise<number[]>} array of pageIds
*/
async #getPageIndexForSpace(spaceId) {
// errors on fetching the pageIndex is fatal, no error handling
let hasNext = true;
let pageIds = [];
let pageNr = 0;
do {
let { isLast, pageIdsForPage } = await this.#getPagesForSpacePaginated(
spaceId,
pageNr
);
hasNext = !isLast;
pageNr++;
if (pageIdsForPage.length) {
pageIds = pageIds.concat(pageIdsForPage);
}
} while (hasNext);
return pageIds;
}
/**
*
* @param {number} pageNr
* @param {number} spaceId
* @returns {Promise<{isLast,pageIds}>}
*/
async #getPagesForSpacePaginated(spaceId, pageNr) {
/*
* {
* content: Page[],
* last: boolean,
* pageable: {
* pageNumber: number
* }
* }
*/
const data = await this._doFetch(
`${this.baseUrl}/api/rest/scope/api/page?size=100&space=${spaceId}&page=${pageNr}`
);
const pageIds = data.content.map((page) => {
return Number(page.id);
});
return {
isLast: data.last,
pageIdsForPage: pageIds,
};
}
/**
* @param pageId
* @returns {Promise<Page>}
*/
async #fetchPage(pageId) {
const data = await this._doFetch(
`${this.baseUrl}/api/rest/scope/api/page/${pageId}`
);
const url = `${this.baseUrl}/node/${data.id}`;
return new Page({
id: data.id,
title: data.title,
created: data.lastModified,
type: data.type,
processedBody: this.#processPageBody({
body: data.body,
title: data.title,
lastModified: data.lastModified,
url: url,
}),
url: url,
});
}
/**
* @param {Page} page
* @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker
*/
#storePage(page, encryptionWorker) {
const { hostname } = new URL(this.baseUrl);
// This UUID will ensure that re-importing the same page without any changes will not
// show up (deduplication).
const targetUUID = `${hostname}.${page.spaceId}.${page.id}.${page.created}`;
const wordCount = page.processedBody.split(" ").length;
const tokenCount =
page.processedBody.length > 0
? tokenizeString(page.processedBody).length
: 0;
const data = {
id: targetUUID,
url: `drupalwiki://${page.url}`,
title: page.title,
docAuthor: this.baseUrl,
description: page.title,
docSource: `${this.baseUrl} DrupalWiki`,
chunkSource: this.#generateChunkSource(page.id, encryptionWorker),
published: new Date().toLocaleString(),
wordCount: wordCount,
pageContent: page.processedBody,
token_count_estimate: tokenCount,
};
const fileName = sanitizeFileName(`${slugify(page.title)}-${data.id}`);
console.log(
`[DrupalWiki Loader]: Saving page '${page.title}' (${page.id}) to '${this.storagePath}/${fileName}'`
);
writeToServerDocuments(data, fileName, this.storagePath);
}
/**
* Generate the full chunkSource for a specific Confluence page so that we can resync it later.
* This data is encrypted into a single `payload` query param so we can replay credentials later
* since this was encrypted with the systems persistent password and salt.
* @param {number} pageId
* @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker
* @returns {string}
*/
#generateChunkSource(pageId, encryptionWorker) {
const payload = {
baseUrl: this.baseUrl,
pageId: pageId,
accessToken: this.accessToken,
};
return `drupalwiki://${this.baseUrl}?payload=${encryptionWorker.encrypt(
JSON.stringify(payload)
)}`;
}
async _doFetch(url) {
const response = await fetch(url, {
headers: this.#getHeaders(),
});
if (!response.ok) {
throw new Error(`Failed to fetch ${url}: ${response.status}`);
}
return response.json();
}
#getHeaders() {
return {
"Content-Type": "application/json",
Accept: "application/json",
Authorization: `Bearer ${this.accessToken}`,
};
}
#prepareStoragePath(baseUrl) {
const { hostname } = new URL(baseUrl);
const subFolder = slugify(`drupalwiki-${hostname}`).toLowerCase();
const outFolder =
process.env.NODE_ENV === "development"
? path.resolve(
__dirname,
`../../../../server/storage/documents/${subFolder}`
)
: path.resolve(process.env.STORAGE_DIR, `documents/${subFolder}`);
if (!fs.existsSync(outFolder)) {
fs.mkdirSync(outFolder, { recursive: true });
}
return outFolder;
}
/**
* @param {string} body
* @param {string} url
* @param {string} title
* @param {string} lastModified
* @returns {string}
* @private
*/
#processPageBody({ body, url, title, lastModified }) {
// use the title as content if there is none
const textContent = body.trim() !== "" ? body : title;
const plainTextContent = htmlToText(textContent, {
wordwrap: false,
preserveNewlines: true,
});
// preserve structure
const plainBody = plainTextContent.replace(/\n{3,}/g, "\n\n");
// add the link to the document
return `Link/URL: ${url}\n\n${plainBody}`;
}
async #downloadAndProcessAttachments(pageId) {
try {
const data = await this._doFetch(
`${this.baseUrl}/api/rest/scope/api/attachment?pageId=${pageId}&size=2000`
);
const extensionsList = Object.keys(SUPPORTED_FILETYPE_CONVERTERS);
for (const attachment of data.content || data) {
const { fileName, id: attachId } = attachment;
const lowerName = fileName.toLowerCase();
if (!extensionsList.some((ext) => lowerName.endsWith(ext))) {
continue;
}
const downloadUrl = `${this.baseUrl}/api/rest/scope/api/attachment/${attachId}/download`;
const attachmentResponse = await fetch(downloadUrl, {
headers: this.#getHeaders(),
});
if (!attachmentResponse.ok) {
console.log(`Skipping attachment: ${fileName} - Download failed`);
continue;
}
const buffer = await attachmentResponse.arrayBuffer();
const localFilePath = `${WATCH_DIRECTORY}/${fileName}`;
require("fs").writeFileSync(localFilePath, Buffer.from(buffer));
await processSingleFile(fileName);
}
} catch (err) {
console.error(`Fetching/processing attachments failed:`, err);
}
}
}
module.exports = { DrupalWiki };

102
collector/utils/extensions/DrupalWiki/index.js Normal file
View File

@ -0,0 +1,102 @@
/**
* Copyright 2024
*
* Authors:
* - Eugen Mayer (KontextWork)
*/
const { DrupalWiki } = require("./DrupalWiki");
const { validBaseUrl } = require("../../../utils/http");
async function loadAndStoreSpaces(
{ baseUrl = null, spaceIds = null, accessToken = null },
response
) {
if (!baseUrl) {
return {
success: false,
reason:
"Please provide your baseUrl like https://mywiki.drupal-wiki.net.",
};
} else if (!validBaseUrl(baseUrl)) {
return {
success: false,
reason: "Provided base URL is not a valid URL.",
};
}
if (!spaceIds) {
return {
success: false,
reason:
"Please provide a list of spaceIds like 21,56,67 you want to extract",
};
}
if (!accessToken) {
return {
success: false,
reason: "Please provide a REST API-Token.",
};
}
console.log(`-- Working Drupal Wiki ${baseUrl} for spaceIds: ${spaceIds} --`);
const drupalWiki = new DrupalWiki({ baseUrl, accessToken });
const encryptionWorker = response.locals.encryptionWorker;
const spaceIdsArr = spaceIds.split(",").map((idStr) => {
return Number(idStr.trim());
});
for (const spaceId of spaceIdsArr) {
try {
await drupalWiki.loadAndStoreAllPagesForSpace(spaceId, encryptionWorker);
console.log(`--- Finished space ${spaceId} ---`);
} catch (e) {
console.error(e);
return {
success: false,
reason: e.message,
data: {},
};
}
}
console.log(`-- Finished all spaces--`);
return {
success: true,
reason: null,
data: {
spaceIds,
destination: drupalWiki.storagePath,
},
};
}
/**
* Gets the page content from a specific Confluence page, not all pages in a workspace.
* @returns
*/
async function loadPage({ baseUrl, pageId, accessToken }) {
console.log(`-- Working Drupal Wiki Page ${pageId} of ${baseUrl} --`);
const drupalWiki = new DrupalWiki({ baseUrl, accessToken });
try {
const page = await drupalWiki.loadPage(pageId);
return {
success: true,
reason: null,
content: page.processedBody,
};
} catch (e) {
return {
success: false,
reason: `Failed (re)-fetching DrupalWiki page ${pageId} form ${baseUrl}}`,
content: null,
};
}
}
module.exports = {
loadAndStoreSpaces,
loadPage,
};

17
collector/utils/http/index.js
View File

@ -12,7 +12,24 @@ function queryParams(request) {
return request.query;
}
/**
* Validates if the provided baseUrl is a valid URL at all.
* - Does not validate if the URL is reachable or accessible.
* - Does not do any further validation of the URL like `validURL` in `utils/url/index.js`
* @param {string} baseUrl
* @returns {boolean}
*/
function validBaseUrl(baseUrl) {
try {
new URL(baseUrl);
return true;
} catch (e) {
return false;
}
}
module.exports = {
reqBody,
queryParams,
validBaseUrl,
};

83
collector/utils/runtimeSettings/index.js Normal file
View File

@ -0,0 +1,83 @@
const { reqBody } = require("../http");
/**
* Runtime settings are used to configure the collector per-request.
* These settings are persisted across requests, but can be overridden per-request.
*
* The settings are passed in the request body via `options.runtimeSettings`
* which is set in the backend #attachOptions function in CollectorApi.
*
* We do this so that the collector and backend can share the same ENV variables
* but only pass the relevant settings to the collector per-request and be able to
* access them across the collector via a single instance of RuntimeSettings.
*
* TODO: We may want to set all options passed from backend to collector here,
* but for now - we are only setting the runtime settings specifically for backwards
* compatibility with existing CollectorApi usage.
*/
class RuntimeSettings {
static _instance = null;
settings = {};
// Any settings here will be persisted across requests
// and must be explicitly defined here.
settingConfigs = {
allowAnyIp: {
default: false,
// Value must be explicitly "true" or "false" as a string
validate: (value) => String(value) === "true",
},
};
constructor() {
if (RuntimeSettings._instance) return RuntimeSettings._instance;
RuntimeSettings._instance = this;
return this;
}
/**
* Parse the runtime settings from the request body options body
* see #attachOptions https://github.com/Mintplex-Labs/anything-llm/blob/ebf112007e0d579af3d2b43569db95bdfc59074b/server/utils/collectorApi/index.js#L18
* @param {import('express').Request} request
* @returns {void}
*/
parseOptionsFromRequest(request = {}) {
const options = reqBody(request)?.options?.runtimeSettings || {};
for (const [key, value] of Object.entries(options)) {
if (!this.settingConfigs.hasOwnProperty(key)) continue;
this.set(key, value);
}
return;
}
/**
* Get a runtime setting
* - Will throw an error if the setting requested is not a supported runtime setting key
* - Will return the default value if the setting requested is not set at all
* @param {string} key
* @returns {any}
*/
get(key) {
if (!this.settingConfigs[key])
throw new Error(`Invalid runtime setting: ${key}`);
return this.settings.hasOwnProperty(key)
? this.settings[key]
: this.settingConfigs[key].default;
}
/**
* Set a runtime setting
* - Will throw an error if the setting requested is not a supported runtime setting key
* - Will validate the value against the setting's validate function
* @param {string} key
* @param {any} value
* @returns {void}
*/
set(key, value = null) {
if (!this.settingConfigs[key])
throw new Error(`Invalid runtime setting: ${key}`);
this.settings[key] = this.settingConfigs[key].validate(value);
}
}
module.exports = RuntimeSettings;

18
collector/utils/url/index.js
View File

@ -1,3 +1,4 @@
const RuntimeSettings = require("../runtimeSettings");
/** ATTN: SECURITY RESEARCHERS
* To Security researchers about to submit an SSRF report CVE - please don't.
* We are aware that the code below is does not defend against any of the thousands of ways
@ -13,15 +14,24 @@
const VALID_PROTOCOLS = ["https:", "http:"];
const INVALID_OCTETS = [192, 172, 10, 127];
const runtimeSettings = new RuntimeSettings();
/**
* If an ip address is passed in the user is attempting to collector some internal service running on internal/private IP.
* This is not a security feature and simply just prevents the user from accidentally entering invalid IP addresses.
* Can be bypassed via COLLECTOR_ALLOW_ANY_IP environment variable.
* @param {URL} param0
* @param {URL['hostname']} param0.hostname
* @returns {boolean}
*/
function isInvalidIp({ hostname }) {
if (runtimeSettings.get("allowAnyIp")) {
console.log(
"\x1b[33mURL IP local address restrictions have been disabled by administrator!\x1b[0m"
);
return false;
}
const IPRegex = new RegExp(
/^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/gi
);
@ -40,6 +50,14 @@ function isInvalidIp({ hostname }) {
return INVALID_OCTETS.includes(Number(octetOne));
}
/**
* Validates a URL
* - Checks the URL forms a valid URL
* - Checks the URL is at least HTTP(S)
* - Checks the URL is not an internal IP - can be bypassed via COLLECTOR_ALLOW_ANY_IP
* @param {string} url
* @returns {boolean}
*/
function validURL(url) {
try {
const destination = new URL(url);

4
docker/.env.example
View File

@ -322,6 +322,10 @@ GID='1000'
# See https://docs.anythingllm.com/configuration#simple-sso-passthrough for more information.
# SIMPLE_SSO_ENABLED=1
# Allow scraping of any IP address in collector - must be string "true" to be enabled
# See https://docs.anythingllm.com/configuration#local-ip-address-scraping for more information.
# COLLECTOR_ALLOW_ANY_IP="true"
# Specify the target languages for when using OCR to parse images and PDFs.
# This is a comma separated list of language codes as a string. Unsupported languages will be ignored.
# Default is English. See https://tesseract-ocr.github.io/tessdoc/Data-Files-in-different-versions.html for a list of valid language codes.

2
frontend/public/embed/anythingllm-chat-widget.min.css vendored
View File

File diff suppressed because one or more lines are too long

10
frontend/public/embed/anythingllm-chat-widget.min.js vendored
View File

File diff suppressed because one or more lines are too long

BIN
frontend/src/components/DataConnectorOption/media/drupalwiki.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.1 KiB

2
frontend/src/components/DataConnectorOption/media/index.js
View File

@ -3,6 +3,7 @@ import GitLab from "./gitlab.svg";
import YouTube from "./youtube.svg";
import Link from "./link.svg";
import Confluence from "./confluence.jpeg";
import DrupalWiki from "./drupalwiki.jpg";
const ConnectorImages = {
github: GitHub,
@ -10,6 +11,7 @@ const ConnectorImages = {
youtube: YouTube,
websiteDepth: Link,
confluence: Confluence,
drupalwiki: DrupalWiki,
};
export default ConnectorImages;

190
frontend/src/components/Modals/ManageWorkspace/DataConnectors/Connectors/DrupalWiki/index.jsx Normal file
View File

@ -0,0 +1,190 @@
/**
* Copyright 2024
*
* Authors:
* - Eugen Mayer (KontextWork)
*/
import { useState } from "react";
import System from "@/models/system";
import showToast from "@/utils/toast";
import { Warning } from "@phosphor-icons/react";
import { Tooltip } from "react-tooltip";
export default function DrupalWikiOptions() {
const [loading, setLoading] = useState(false);
const handleSubmit = async (e) => {
e.preventDefault();
const form = new FormData(e.target);
try {
setLoading(true);
showToast(
"Fetching all pages for the given Drupal Wiki spaces - this may take a while.",
"info",
{
clear: true,
autoClose: false,
}
);
const { data, error } = await System.dataConnectors.drupalwiki.collect({
baseUrl: form.get("baseUrl"),
spaceIds: form.get("spaceIds"),
accessToken: form.get("accessToken"),
});
if (!!error) {
showToast(error, "error", { clear: true });
setLoading(false);
return;
}
showToast(
`Pages collected from Drupal Wiki spaces ${data.spaceIds}. Output folder is ${data.destination}.`,
"success",
{ clear: true }
);
e.target.reset();
setLoading(false);
} catch (e) {
console.error(e);
showToast(e.message, "error", { clear: true });
setLoading(false);
}
};
return (
<div className="flex w-full">
<div className="flex flex-col w-full px-1 md:pb-6 pb-16">
<form className="w-full" onSubmit={handleSubmit}>
<div className="w-full flex flex-col py-2">
<div className="w-full flex flex-col gap-4">
<div className="flex flex-col pr-10">
<div className="flex flex-col gap-y-1 mb-4">
<label className="text-white text-sm font-bold flex gap-x-2 items-center">
<p className="font-bold text-white">Drupal Wiki base URL</p>
</label>
<p className="text-xs font-normal text-theme-text-secondary">
This is the base URL of your&nbsp;
<a
href="https://drupal-wiki.com"
target="_blank"
rel="noopener noreferrer"
className="underline"
>
Drupal Wiki
</a>
.
</p>
</div>
<input
type="url"
name="baseUrl"
className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="eg: https://mywiki.drupal-wiki.net, https://drupalwiki.mycompany.tld, etc..."
required={true}
autoComplete="off"
spellCheck={false}
/>
</div>
<div className="flex flex-col pr-10">
<div className="flex flex-col gap-y-1 mb-4">
<label className="text-white text-sm font-bold">
Drupal Wiki Space IDs
</label>
<p className="text-xs font-normal text-theme-text-secondary">
Comma seperated Space IDs you want to extract. See the&nbsp;
<a
href="https://help.drupal-wiki.com/node/606"
target="_blank"
rel="noopener noreferrer"
className="underline"
onClick={(e) => e.stopPropagation()}
>
manual
</a>
&nbsp; on how to retrieve the Space IDs. Be sure that your
'API-Token User' has access to those spaces.
</p>
</div>
<input
type="text"
name="spaceIds"
className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="eg: 12,34,69"
required={true}
autoComplete="off"
spellCheck={false}
/>
</div>
<div className="flex flex-col pr-10">
<div className="flex flex-col gap-y-1 mb-4">
<label className="text-white text-sm font-bold flex gap-x-2 items-center">
<p className="font-bold text-white">
Drupal Wiki API Token
</p>
<Warning
size={14}
className="ml-1 text-orange-500 cursor-pointer"
data-tooltip-id="access-token-tooltip"
data-tooltip-place="right"
/>
<Tooltip
delayHide={300}
id="access-token-tooltip"
className="max-w-xs z-99"
clickable={true}
>
<p className="text-sm font-light text-theme-text-primary">
You need to provide an API token for authentication. See
the Drupal Wiki&nbsp;
<a
href="https://help.drupal-wiki.com/node/605#2-Zugriffs-Token-generieren"
target="_blank"
rel="noopener noreferrer"
className="underline"
>
manual
</a>
&nbsp;on how to generate an API-Token for your user.
</p>
</Tooltip>
</label>
<p className="text-xs font-normal text-theme-text-secondary">
Access token for authentication.
</p>
</div>
<input
type="password"
name="accessToken"
className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="pat:123"
required={true}
autoComplete="off"
spellCheck={false}
/>
</div>
</div>
</div>
<div className="flex flex-col gap-y-2 w-full pr-10">
<button
type="submit"
disabled={loading}
className="mt-2 w-full justify-center border border-slate-200 px-4 py-2 rounded-lg text-dark-text text-sm font-bold items-center flex gap-x-2 bg-slate-200 hover:bg-slate-300 hover:text-slate-800 disabled:bg-slate-300 disabled:cursor-not-allowed"
>
{loading ? "Collecting pages..." : "Submit"}
</button>
{loading && (
<p className="text-xs text-theme-text-secondary">
Once complete, all pages will be available for embedding into
workspaces.
</p>
)}
</div>
</form>
</div>
</div>
);
}

7
frontend/src/components/Modals/ManageWorkspace/DataConnectors/index.jsx
View File

@ -5,6 +5,7 @@ import GithubOptions from "./Connectors/Github";
import GitlabOptions from "./Connectors/Gitlab";
import YoutubeOptions from "./Connectors/Youtube";
import ConfluenceOptions from "./Connectors/Confluence";
import DrupalWikiOptions from "./Connectors/DrupalWiki";
import { useState } from "react";
import ConnectorOption from "./ConnectorOption";
import WebsiteDepthOptions from "./Connectors/WebsiteDepth";
@ -40,6 +41,12 @@ export const getDataConnectors = (t) => ({
description: t("connectors.confluence.description"),
options: <ConfluenceOptions />,
},
drupalwiki: {
name: "Drupal Wiki",
image: ConnectorImages.drupalwiki,
description: "Import Drupal Wiki spaces in a single click.",
options: <DrupalWikiOptions />,
},
});
export default function DataConnectors() {

19
frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx
View File

@ -15,6 +15,7 @@ import {
YoutubeLogo,
} from "@phosphor-icons/react";
import ConfluenceLogo from "@/media/dataConnectors/confluence.png";
import DrupalWikiLogo from "@/media/dataConnectors/drupalwiki.png";
import { toPercentString } from "@/utils/numbers";
function combineLikeSources(sources) {
@ -197,14 +198,17 @@ function parseChunkSource({ title = "", chunks = [] }) {
!chunks.length ||
(!chunks[0].chunkSource?.startsWith("link://") &&
!chunks[0].chunkSource?.startsWith("confluence://") &&
!chunks[0].chunkSource?.startsWith("github://"))
!chunks[0].chunkSource?.startsWith("github://") &&
!chunks[0].chunkSource?.startsWith("drupalwiki://"))
)
return nullResponse;
try {
const url = new URL(
chunks[0].chunkSource.split("link://")[1] ||
chunks[0].chunkSource.split("confluence://")[1] ||
chunks[0].chunkSource.split("github://")[1]
chunks[0].chunkSource.split("github://")[1] ||
chunks[0].chunkSource.split("drupalwiki://")[1]
);
let text = url.host + url.pathname;
let icon = "link";
@ -224,6 +228,11 @@ function parseChunkSource({ title = "", chunks = [] }) {
icon = "confluence";
}
if (url.host.includes("drupal-wiki.net")) {
text = title;
icon = "drupalwiki";
}
return {
isUrl: true,
href: url.toString(),
@ -239,10 +248,16 @@ const ConfluenceIcon = ({ ...props }) => (
<img src={ConfluenceLogo} {...props} />
);
// Patch to render DrupalWiki icon as a element like we do with Phosphor
const DrupalWikiIcon = ({ ...props }) => (
<img src={DrupalWikiLogo} {...props} />
);
const ICONS = {
file: FileText,
link: Link,
youtube: YoutubeLogo,
github: GithubLogo,
confluence: ConfluenceIcon,
drupalwiki: DrupalWikiIcon,
};

80
frontend/src/locales/ar/common.js
View File

@ -732,6 +732,86 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError: null,
checklist: {
title: null,
tasksLeft: null,
completed: null,
dismiss: null,
tasks: {
create_workspace: {
title: null,
description: null,
action: null,
},
send_chat: {
title: null,
description: null,
action: null,
},
embed_document: {
title: null,
description: null,
action: null,
},
setup_system_prompt: {
title: null,
description: null,
action: null,
},
define_slash_command: {
title: null,
description: null,
action: null,
},
visit_community: {
title: null,
description: null,
action: null,
},
},
},
quickLinks: {
title: null,
sendChat: null,
embedDocument: null,
createWorkspace: null,
},
exploreMore: {
title: null,
features: {
customAgents: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
slashCommands: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
systemPrompts: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
},
},
announcements: {
title: null,
},
resources: {
title: null,
links: {
docs: null,
star: null,
},
},
},
};
export default TRANSLATIONS;

80
frontend/src/locales/da/common.js
View File

@ -771,6 +771,86 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError: null,
checklist: {
title: null,
tasksLeft: null,
completed: null,
dismiss: null,
tasks: {
create_workspace: {
title: null,
description: null,
action: null,
},
send_chat: {
title: null,
description: null,
action: null,
},
embed_document: {
title: null,
description: null,
action: null,
},
setup_system_prompt: {
title: null,
description: null,
action: null,
},
define_slash_command: {
title: null,
description: null,
action: null,
},
visit_community: {
title: null,
description: null,
action: null,
},
},
},
quickLinks: {
title: null,
sendChat: null,
embedDocument: null,
createWorkspace: null,
},
exploreMore: {
title: null,
features: {
customAgents: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
slashCommands: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
systemPrompts: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
},
},
announcements: {
title: null,
},
resources: {
title: null,
links: {
docs: null,
star: null,
},
},
},
};
export default TRANSLATIONS;

87
frontend/src/locales/de/common.js
View File

@ -769,6 +769,93 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError:
"Bitte erstellen Sie einen Arbeitsbereich, bevor Sie einen Chat beginnen.",
checklist: {
title: "Erste Schritte",
tasksLeft: "Aufgaben übrig",
completed: "Sie sind auf dem Weg, ein AnythingLLM-Experte zu werden!",
dismiss: "schließen",
tasks: {
create_workspace: {
title: "Einen Arbeitsbereich erstellen",
description:
"Erstellen Sie Ihren ersten Arbeitsbereich, um zu beginnen",
action: "Erstellen",
},
send_chat: {
title: "Einen Chat senden",
description: "Starten Sie ein Gespräch mit Ihrem KI-Assistenten",
action: "Chat",
},
embed_document: {
title: "Ein Dokument einbetten",
description:
"Fügen Sie Ihr erstes Dokument zu Ihrem Arbeitsbereich hinzu",
action: "Einbetten",
},
setup_system_prompt: {
title: "Ein System-Prompt einrichten",
description: "Konfigurieren Sie das Verhalten Ihres KI-Assistenten",
action: "Einrichten",
},
define_slash_command: {
title: "Einen Slash-Befehl definieren",
description:
"Erstellen Sie benutzerdefinierte Befehle für Ihren Assistenten",
action: "Definieren",
},
visit_community: {
title: "Community Hub besuchen",
description: "Entdecken Sie Community-Ressourcen und Vorlagen",
action: "Stöbern",
},
},
},
quickLinks: {
title: "Schnellzugriffe",
sendChat: "Chat senden",
embedDocument: "Dokument einbetten",
createWorkspace: "Arbeitsbereich erstellen",
},
exploreMore: {
title: "Weitere Funktionen erkunden",
features: {
customAgents: {
title: "Benutzerdefinierte KI-Agenten",
description:
"Erstellen Sie leistungsstarke KI-Agenten und Automatisierungen ohne Code.",
primaryAction: "Chatten mit @agent",
secondaryAction: "Einen Agenten-Flow erstellen",
},
slashCommands: {
title: "Slash-Befehle",
description:
"Sparen Sie Zeit und fügen Sie Eingabeaufforderungen mit benutzerdefinierten Slash-Befehlen ein.",
primaryAction: "Einen Slash-Befehl erstellen",
secondaryAction: "Im Hub erkunden",
},
systemPrompts: {
title: "System-Eingabeaufforderungen",
description:
"Ändern Sie die System-Eingabeaufforderung, um die KI-Antworten eines Arbeitsbereichs anzupassen.",
primaryAction: "Eine System-Eingabeaufforderung ändern",
secondaryAction: "Eingabevariablen verwalten",
},
},
},
announcements: {
title: "Updates & Ankündigungen",
},
resources: {
title: "Ressourcen",
links: {
docs: "Dokumentation",
star: "Auf Github mit Stern versehen",
},
},
},
};
export default TRANSLATIONS;

83
frontend/src/locales/en/common.js
View File

@ -152,6 +152,89 @@ const TRANSLATIONS = {
contact: "Contact Mintplex Labs",
},
"main-page": {
noWorkspaceError: "Please create a workspace before starting a chat.",
checklist: {
title: "Getting Started",
tasksLeft: "tasks left",
completed: "You're on your way to becoming an AnythingLLM expert!",
dismiss: "close",
tasks: {
create_workspace: {
title: "Create a workspace",
description: "Create your first workspace to get started",
action: "Create",
},
send_chat: {
title: "Send a chat",
description: "Start a conversation with your AI assistant",
action: "Chat",
},
embed_document: {
title: "Embed a document",
description: "Add your first document to your workspace",
action: "Embed",
},
setup_system_prompt: {
title: "Set up a system prompt",
description: "Configure your AI assistant's behavior",
action: "Set Up",
},
define_slash_command: {
title: "Define a slash command",
description: "Create custom commands for your assistant",
action: "Define",
},
visit_community: {
title: "Visit Community Hub",
description: "Explore community resources and templates",
action: "Browse",
},
},
},
quickLinks: {
title: "Quick Links",
sendChat: "Send Chat",
embedDocument: "Embed a Document",
createWorkspace: "Create Workspace",
},
exploreMore: {
title: "Explore more features",
features: {
customAgents: {
title: "Custom AI Agents",
description: "Build powerful AI Agents and automations with no code.",
primaryAction: "Chat using @agent",
secondaryAction: "Build an agent flow",
},
slashCommands: {
title: "Slash Commands",
description:
"Save time and inject prompts using custom slash commands.",
primaryAction: "Create a Slash Command",
secondaryAction: "Explore on Hub",
},
systemPrompts: {
title: "System Prompts",
description:
"Modify the system prompt to customize the AI replies of a workspace.",
primaryAction: "Modify a System Prompt",
secondaryAction: "Manage prompt variables",
},
},
},
announcements: {
title: "Updates & Announcements",
},
resources: {
title: "Resources",
links: {
docs: "Docs",
star: "Star on Github",
},
},
},
"new-workspace": {
title: "New Workspace",
placeholder: "My Workspace",

85
frontend/src/locales/es/common.js
View File

@ -731,6 +731,91 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError:
"Por favor, crea un espacio de trabajo antes de iniciar un chat.",
checklist: {
title: "Comenzando",
tasksLeft: "tareas restantes",
completed:
"¡Estás en camino de convertirte en un experto en AnythingLLM!",
dismiss: "cerrar",
tasks: {
create_workspace: {
title: "Crear un espacio de trabajo",
description: "Crea tu primer espacio de trabajo para comenzar",
action: "Crear",
},
send_chat: {
title: "Enviar un chat",
description: "Inicia una conversación con tu asistente de IA",
action: "Chatear",
},
embed_document: {
title: "Incrustar un documento",
description: "Añade tu primer documento a tu espacio de trabajo",
action: "Incrustar",
},
setup_system_prompt: {
title: "Configurar un prompt del sistema",
description: "Configura el comportamiento de tu asistente de IA",
action: "Configurar",
},
define_slash_command: {
title: "Definir un comando de barra",
description: "Crea comandos personalizados para tu asistente",
action: "Definir",
},
visit_community: {
title: "Visitar el Centro de la Comunidad",
description: "Explora recursos y plantillas de la comunidad",
action: "Explorar",
},
},
},
quickLinks: {
title: "Enlaces Rápidos",
sendChat: "Enviar Chat",
embedDocument: "Incrustar un Documento",
createWorkspace: "Crear Espacio de Trabajo",
},
exploreMore: {
title: "Explora más características",
features: {
customAgents: {
title: "Agentes de IA Personalizados",
description:
"Crea poderosos agentes de IA y automatizaciones sin código.",
primaryAction: "Chatear usando @agente",
secondaryAction: "Crear un flujo de agente",
},
slashCommands: {
title: "Comandos de Barra",
description:
"Ahorra tiempo e inyecta prompts utilizando comandos de barra personalizados.",
primaryAction: "Crear un Comando de Barra",
secondaryAction: "Explorar en el Hub",
},
systemPrompts: {
title: "Prompts del Sistema",
description:
"Modifica el prompt del sistema para personalizar las respuestas de IA de un espacio de trabajo.",
primaryAction: "Modificar un Prompt del Sistema",
secondaryAction: "Gestionar variables de prompt",
},
},
},
announcements: {
title: "Actualizaciones y Anuncios",
},
resources: {
title: "Recursos",
links: {
docs: "Documentación",
star: "Destacar en Github",
},
},
},
};
export default TRANSLATIONS;

80
frontend/src/locales/fa/common.js
View File

@ -724,6 +724,86 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError: null,
checklist: {
title: null,
tasksLeft: null,
completed: null,
dismiss: null,
tasks: {
create_workspace: {
title: null,
description: null,
action: null,
},
send_chat: {
title: null,
description: null,
action: null,
},
embed_document: {
title: null,
description: null,
action: null,
},
setup_system_prompt: {
title: null,
description: null,
action: null,
},
define_slash_command: {
title: null,
description: null,
action: null,
},
visit_community: {
title: null,
description: null,
action: null,
},
},
},
quickLinks: {
title: null,
sendChat: null,
embedDocument: null,
createWorkspace: null,
},
exploreMore: {
title: null,
features: {
customAgents: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
slashCommands: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
systemPrompts: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
},
},
announcements: {
title: null,
},
resources: {
title: null,
links: {
docs: null,
star: null,
},
},
},
};
export default TRANSLATIONS;

80
frontend/src/locales/fr/common.js
View File

@ -732,6 +732,86 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError: null,
checklist: {
title: null,
tasksLeft: null,
completed: null,
dismiss: null,
tasks: {
create_workspace: {
title: null,
description: null,
action: null,
},
send_chat: {
title: null,
description: null,
action: null,
},
embed_document: {
title: null,
description: null,
action: null,
},
setup_system_prompt: {
title: null,
description: null,
action: null,
},
define_slash_command: {
title: null,
description: null,
action: null,
},
visit_community: {
title: null,
description: null,
action: null,
},
},
},
quickLinks: {
title: null,
sendChat: null,
embedDocument: null,
createWorkspace: null,
},
exploreMore: {
title: null,
features: {
customAgents: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
slashCommands: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
systemPrompts: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
},
},
announcements: {
title: null,
},
resources: {
title: null,
links: {
docs: null,
star: null,
},
},
},
};
export default TRANSLATIONS;

80
frontend/src/locales/he/common.js
View File

@ -717,6 +717,86 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError: null,
checklist: {
title: null,
tasksLeft: null,
completed: null,
dismiss: null,
tasks: {
create_workspace: {
title: null,
description: null,
action: null,
},
send_chat: {
title: null,
description: null,
action: null,
},
embed_document: {
title: null,
description: null,
action: null,
},
setup_system_prompt: {
title: null,
description: null,
action: null,
},
define_slash_command: {
title: null,
description: null,
action: null,
},
visit_community: {
title: null,
description: null,
action: null,
},
},
},
quickLinks: {
title: null,
sendChat: null,
embedDocument: null,
createWorkspace: null,
},
exploreMore: {
title: null,
features: {
customAgents: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
slashCommands: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
systemPrompts: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
},
},
announcements: {
title: null,
},
resources: {
title: null,
links: {
docs: null,
star: null,
},
},
},
};
export default TRANSLATIONS;

80
frontend/src/locales/it/common.js
View File

@ -730,6 +730,86 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError: null,
checklist: {
title: null,
tasksLeft: null,
completed: null,
dismiss: null,
tasks: {
create_workspace: {
title: null,
description: null,
action: null,
},
send_chat: {
title: null,
description: null,
action: null,
},
embed_document: {
title: null,
description: null,
action: null,
},
setup_system_prompt: {
title: null,
description: null,
action: null,
},
define_slash_command: {
title: null,
description: null,
action: null,
},
visit_community: {
title: null,
description: null,
action: null,
},
},
},
quickLinks: {
title: null,
sendChat: null,
embedDocument: null,
createWorkspace: null,
},
exploreMore: {
title: null,
features: {
customAgents: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
slashCommands: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
systemPrompts: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
},
},
announcements: {
title: null,
},
resources: {
title: null,
links: {
docs: null,
star: null,
},
},
},
};
export default TRANSLATIONS;

89
frontend/src/locales/ja/common.js
View File

@ -278,17 +278,17 @@ const TRANSLATIONS = {
provider: {
title: "ワークスペースエージェントのLLMプロバイダー",
description:
"このワークスペースの@agentエージェントで使用するLLMプロバイダーとモデルを指定します。",
"このワークスペースの@agentで使用するLLMプロバイダーとモデルを指定します。",
},
mode: {
chat: {
title: "ワークスペースエージェントのチャットモデル",
description:
"このワークスペースの@agentエージェントで使用するチャットモデルを指定します。",
"このワークスペースの@agentで使用するチャットモデルを指定します。",
},
title: "ワークスペースエージェントのモデル",
description:
"このワークスペースの@agentエージェントで使用するLLMモデルを指定します。",
"このワークスペースの@agentで使用するLLMモデルを指定します。",
wait: "-- モデルを読み込み中 --",
},
skill: {
@ -763,6 +763,89 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError:
"チャットを開始する前にワークスペースを作成してください。",
checklist: {
title: "はじめに",
tasksLeft: "残りのタスク",
completed: "AnythingLLMの達人への道を進んでいます！",
dismiss: "閉じる",
tasks: {
create_workspace: {
title: "ワークスペースを作成する",
description: "始めるには最初のワークスペースを作成してください",
action: "作成",
},
send_chat: {
title: "チャットを送信する",
description: "AIアシスタントとの会話を開始する",
action: "チャット",
},
embed_document: {
title: "ドキュメントを埋め込む",
description: "ワークスペースに最初のドキュメントを追加する",
action: "埋め込む",
},
setup_system_prompt: {
title: "システムプロンプトを設定する",
description: "AIアシスタントの動作を設定する",
action: "設定",
},
define_slash_command: {
title: "スラッシュコマンドを定義する",
description: "アシスタント用のカスタムコマンドを作成する",
action: "定義",
},
visit_community: {
title: "コミュニティハブを訪問する",
description: "コミュニティリソースとテンプレートを探索する",
action: "閲覧",
},
},
},
quickLinks: {
title: "クイックリンク",
sendChat: "チャットを送信",
embedDocument: "ドキュメントを埋め込む",
createWorkspace: "ワークスペースを作成",
},
exploreMore: {
title: "その他の機能を探索",
features: {
customAgents: {
title: "カスタムAIエージェント",
description: "コードなしで強力なAIエージェントと自動化を構築。",
primaryAction: "@agentを使用してチャット",
secondaryAction: "エージェントフローを構築",
},
slashCommands: {
title: "スラッシュコマンド",
description:
"カスタムスラッシュコマンドで時間を節約しプロンプトを挿入。",
primaryAction: "スラッシュコマンドを作成",
secondaryAction: "ハブで探索",
},
systemPrompts: {
title: "システムプロンプト",
description:
"システムプロンプトを変更してワークスペースのAI返答をカスタマイズ。",
primaryAction: "システムプロンプトを変更",
secondaryAction: "プロンプト変数を管理",
},
},
},
announcements: {
title: "更新とお知らせ",
},
resources: {
title: "リソース",
links: {
docs: "ドキュメント",
star: "Githubでスター",
},
},
},
};
export default TRANSLATIONS;

80
frontend/src/locales/ko/common.js
View File

@ -717,6 +717,86 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError: null,
checklist: {
title: null,
tasksLeft: null,
completed: null,
dismiss: null,
tasks: {
create_workspace: {
title: null,
description: null,
action: null,
},
send_chat: {
title: null,
description: null,
action: null,
},
embed_document: {
title: null,
description: null,
action: null,
},
setup_system_prompt: {
title: null,
description: null,
action: null,
},
define_slash_command: {
title: null,
description: null,
action: null,
},
visit_community: {
title: null,
description: null,
action: null,
},
},
},
quickLinks: {
title: null,
sendChat: null,
embedDocument: null,
createWorkspace: null,
},
exploreMore: {
title: null,
features: {
customAgents: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
slashCommands: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
systemPrompts: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
},
},
announcements: {
title: null,
},
resources: {
title: null,
links: {
docs: null,
star: null,
},
},
},
};
export default TRANSLATIONS;

80
frontend/src/locales/nl/common.js
View File

@ -727,6 +727,86 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError: null,
checklist: {
title: null,
tasksLeft: null,
completed: null,
dismiss: null,
tasks: {
create_workspace: {
title: null,
description: null,
action: null,
},
send_chat: {
title: null,
description: null,
action: null,
},
embed_document: {
title: null,
description: null,
action: null,
},
setup_system_prompt: {
title: null,
description: null,
action: null,
},
define_slash_command: {
title: null,
description: null,
action: null,
},
visit_community: {
title: null,
description: null,
action: null,
},
},
},
quickLinks: {
title: null,
sendChat: null,
embedDocument: null,
createWorkspace: null,
},
exploreMore: {
title: null,
features: {
customAgents: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
slashCommands: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
systemPrompts: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
},
},
announcements: {
title: null,
},
resources: {
title: null,
links: {
docs: null,
star: null,
},
},
},
};
export default TRANSLATIONS;

80
frontend/src/locales/pt_BR/common.js
View File

@ -728,6 +728,86 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError: null,
checklist: {
title: null,
tasksLeft: null,
completed: null,
dismiss: null,
tasks: {
create_workspace: {
title: null,
description: null,
action: null,
},
send_chat: {
title: null,
description: null,
action: null,
},
embed_document: {
title: null,
description: null,
action: null,
},
setup_system_prompt: {
title: null,
description: null,
action: null,
},
define_slash_command: {
title: null,
description: null,
action: null,
},
visit_community: {
title: null,
description: null,
action: null,
},
},
},
quickLinks: {
title: null,
sendChat: null,
embedDocument: null,
createWorkspace: null,
},
exploreMore: {
title: null,
features: {
customAgents: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
slashCommands: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
systemPrompts: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
},
},
announcements: {
title: null,
},
resources: {
title: null,
links: {
docs: null,
star: null,
},
},
},
};
export default TRANSLATIONS;

80
frontend/src/locales/ru/common.js
View File

@ -772,6 +772,86 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError: null,
checklist: {
title: null,
tasksLeft: null,
completed: null,
dismiss: null,
tasks: {
create_workspace: {
title: null,
description: null,
action: null,
},
send_chat: {
title: null,
description: null,
action: null,
},
embed_document: {
title: null,
description: null,
action: null,
},
setup_system_prompt: {
title: null,
description: null,
action: null,
},
define_slash_command: {
title: null,
description: null,
action: null,
},
visit_community: {
title: null,
description: null,
action: null,
},
},
},
quickLinks: {
title: null,
sendChat: null,
embedDocument: null,
createWorkspace: null,
},
exploreMore: {
title: null,
features: {
customAgents: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
slashCommands: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
systemPrompts: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
},
},
announcements: {
title: null,
},
resources: {
title: null,
links: {
docs: null,
star: null,
},
},
},
};
export default TRANSLATIONS;

80
frontend/src/locales/tr/common.js
View File

@ -727,6 +727,86 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError: null,
checklist: {
title: null,
tasksLeft: null,
completed: null,
dismiss: null,
tasks: {
create_workspace: {
title: null,
description: null,
action: null,
},
send_chat: {
title: null,
description: null,
action: null,
},
embed_document: {
title: null,
description: null,
action: null,
},
setup_system_prompt: {
title: null,
description: null,
action: null,
},
define_slash_command: {
title: null,
description: null,
action: null,
},
visit_community: {
title: null,
description: null,
action: null,
},
},
},
quickLinks: {
title: null,
sendChat: null,
embedDocument: null,
createWorkspace: null,
},
exploreMore: {
title: null,
features: {
customAgents: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
slashCommands: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
systemPrompts: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
},
},
announcements: {
title: null,
},
resources: {
title: null,
links: {
docs: null,
star: null,
},
},
},
};
export default TRANSLATIONS;

80
frontend/src/locales/vn/common.js
View File

@ -726,6 +726,86 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError: null,
checklist: {
title: null,
tasksLeft: null,
completed: null,
dismiss: null,
tasks: {
create_workspace: {
title: null,
description: null,
action: null,
},
send_chat: {
title: null,
description: null,
action: null,
},
embed_document: {
title: null,
description: null,
action: null,
},
setup_system_prompt: {
title: null,
description: null,
action: null,
},
define_slash_command: {
title: null,
description: null,
action: null,
},
visit_community: {
title: null,
description: null,
action: null,
},
},
},
quickLinks: {
title: null,
sendChat: null,
embedDocument: null,
createWorkspace: null,
},
exploreMore: {
title: null,
features: {
customAgents: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
slashCommands: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
systemPrompts: {
title: null,
description: null,
primaryAction: null,
secondaryAction: null,
},
},
},
announcements: {
title: null,
},
resources: {
title: null,
links: {
docs: null,
star: null,
},
},
},
};
export default TRANSLATIONS;

80
frontend/src/locales/zh/common.js
View File

@ -705,6 +705,86 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError: "请在开始聊天前创建一个工作区。",
checklist: {
title: "入门指南",
tasksLeft: "剩余任务",
completed: "你正在成为AnythingLLM专家的路上！",
dismiss: "关闭",
tasks: {
create_workspace: {
title: "创建工作区",
description: "创建你的第一个工作区以开始使用",
action: "创建",
},
send_chat: {
title: "发送聊天",
description: "开始与你的AI助手对话",
action: "聊天",
},
embed_document: {
title: "嵌入文档",
description: "添加你的第一个文档到工作区",
action: "嵌入",
},
setup_system_prompt: {
title: "设置系统提示",
description: "配置你的AI助手的行为",
action: "设置",
},
define_slash_command: {
title: "定义斜杠命令",
description: "为你的助手创建自定义命令",
action: "定义",
},
visit_community: {
title: "访问社区中心",
description: "探索社区资源和模板",
action: "浏览",
},
},
},
quickLinks: {
title: "快捷链接",
sendChat: "发送聊天",
embedDocument: "嵌入文档",
createWorkspace: "创建工作区",
},
exploreMore: {
title: "探索更多功能",
features: {
customAgents: {
title: "自定义AI代理",
description: "无需编程即可构建强大的AI代理和自动化流程。",
primaryAction: "使用@agent聊天",
secondaryAction: "构建代理流程",
},
slashCommands: {
title: "斜杠命令",
description: "使用自定义斜杠命令节省时间并注入提示。",
primaryAction: "创建斜杠命令",
secondaryAction: "在中心探索",
},
systemPrompts: {
title: "系统提示",
description: "修改系统提示以自定义工作区的AI回复。",
primaryAction: "修改系统提示",
secondaryAction: "管理提示变量",
},
},
},
announcements: {
title: "更新与公告",
},
resources: {
title: "资源",
links: {
docs: "文档",
star: "在Github上加星标",
},
},
},
};
export default TRANSLATIONS;

80
frontend/src/locales/zh_TW/common.js
View File

@ -708,6 +708,86 @@ const TRANSLATIONS = {
},
},
},
"main-page": {
noWorkspaceError: "請先建立工作空間才能開始對話。",
checklist: {
title: "開始使用",
tasksLeft: "個任務未完成",
completed: "你已經走在成為AnythingLLM專家的路上！",
dismiss: "關閉",
tasks: {
create_workspace: {
title: "建立工作空間",
description: "建立你的第一個工作空間來開始使用",
action: "建立",
},
send_chat: {
title: "發送對話",
description: "開始與你的AI助理對話",
action: "對話",
},
embed_document: {
title: "嵌入文件",
description: "將你的第一個文件添加到工作空間",
action: "嵌入",
},
setup_system_prompt: {
title: "設置系統提示",
description: "設定你的AI助理的行為模式",
action: "設置",
},
define_slash_command: {
title: "定義斜線命令",
description: "為你的助理創建自定義命令",
action: "定義",
},
visit_community: {
title: "訪問社群中心",
description: "探索社群資源和模板",
action: "瀏覽",
},
},
},
quickLinks: {
title: "快速連結",
sendChat: "發送對話",
embedDocument: "嵌入文件",
createWorkspace: "建立工作空間",
},
exploreMore: {
title: "探索更多功能",
features: {
customAgents: {
title: "自定義AI代理",
description: "無需編碼即可建立強大的AI代理和自動化流程。",
primaryAction: "使用@代理進行對話",
secondaryAction: "建立代理流程",
},
slashCommands: {
title: "斜線命令",
description: "節省時間並使用自定義斜線命令注入提示。",
primaryAction: "創建斜線命令",
secondaryAction: "在中心探索",
},
systemPrompts: {
title: "系統提示",
description: "修改系統提示以自定義工作空間的AI回覆。",
primaryAction: "修改系統提示",
secondaryAction: "管理提示變數",
},
},
},
announcements: {
title: "更新與公告",
},
resources: {
title: "資源",
links: {
docs: "文檔",
star: "在Github上加星標",
},
},
},
};
export default TRANSLATIONS;

BIN
frontend/src/media/dataConnectors/drupalwiki.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 24 KiB

23
frontend/src/models/dataConnector.js
View File

@ -162,6 +162,29 @@ const DataConnector = {
});
},
},
drupalwiki: {
collect: async function ({ baseUrl, spaceIds, accessToken }) {
return await fetch(`${API_BASE}/ext/drupalwiki`, {
method: "POST",
headers: baseHeaders(),
body: JSON.stringify({
baseUrl,
spaceIds,
accessToken,
}),
})
.then((res) => res.json())
.then((res) => {
if (!res.success) throw new Error(res.reason);
return { data: res.data, error: null };
})
.catch((e) => {
console.error(e);
return { data: null, error: e.message };
});
},
},
};
export default DataConnector;

78
frontend/src/pages/Main/Home/Checklist/constants.js
View File

@ -7,6 +7,8 @@ import {
} from "@phosphor-icons/react";
import SlashCommandIcon from "./ChecklistItem/icons/SlashCommand";
import paths from "@/utils/paths";
import { t } from "i18next";
const noop = () => {};
export const CHECKLIST_UPDATED_EVENT = "anythingllm_checklist_updated";
@ -34,13 +36,16 @@ export const CHECKLIST_HIDDEN = "anythingllm_checklist_dismissed";
* @property {boolean} completed
*/
/** @type {ChecklistItem[]} */
export const CHECKLIST_ITEMS = [
/**
* Function to generate the checklist items
* @returns {ChecklistItem[]}
*/
export const CHECKLIST_ITEMS = () => [
{
id: "create_workspace",
title: "Create a workspace",
description: "Create your first workspace to get started",
action: "Create",
title: t("main-page.checklist.tasks.create_workspace.title"),
description: t("main-page.checklist.tasks.create_workspace.description"),
action: t("main-page.checklist.tasks.create_workspace.action"),
handler: ({ showNewWsModal = noop }) => {
showNewWsModal();
return true;
@ -49,9 +54,9 @@ export const CHECKLIST_ITEMS = [
},
{
id: "send_chat",
title: "Send a chat",
description: "Start a conversation with your AI assistant",
action: "Chat",
title: t("main-page.checklist.tasks.send_chat.title"),
description: t("main-page.checklist.tasks.send_chat.description"),
action: t("main-page.checklist.tasks.send_chat.action"),
handler: ({
workspaces = [],
navigate = noop,
@ -59,11 +64,9 @@ export const CHECKLIST_ITEMS = [
showNewWsModal = noop,
}) => {
if (workspaces.length === 0) {
showToast(
"Please create a workspace before starting a chat.",
"warning",
{ clear: true }
);
showToast(t("main-page.noWorkspaceError"), "warning", {
clear: true,
});
showNewWsModal();
return false;
}
@ -74,9 +77,9 @@ export const CHECKLIST_ITEMS = [
},
{
id: "embed_document",
title: "Embed a document",
description: "Add your first document to your workspace",
action: "Embed",
title: t("main-page.checklist.tasks.embed_document.title"),
description: t("main-page.checklist.tasks.embed_document.description"),
action: t("main-page.checklist.tasks.embed_document.action"),
handler: ({
workspaces = [],
setSelectedWorkspace = noop,
@ -85,11 +88,10 @@ export const CHECKLIST_ITEMS = [
showNewWsModal = noop,
}) => {
if (workspaces.length === 0) {
showToast(
"Please create a workspace before embedding documents.",
"warning",
{ clear: true }
);
debugger;
showToast(t("main-page.noWorkspaceError"), "warning", {
clear: true,
});
showNewWsModal();
return false;
}
@ -101,9 +103,9 @@ export const CHECKLIST_ITEMS = [
},
{
id: "setup_system_prompt",
title: "Set up a system prompt",
description: "Configure your AI assistant's behavior",
action: "Set Up",
title: t("main-page.checklist.tasks.setup_system_prompt.title"),
description: t("main-page.checklist.tasks.setup_system_prompt.description"),
action: t("main-page.checklist.tasks.setup_system_prompt.action"),
handler: ({
workspaces = [],
navigate = noop,
@ -111,11 +113,9 @@ export const CHECKLIST_ITEMS = [
showToast = noop,
}) => {
if (workspaces.length === 0) {
showToast(
"Please create a workspace before setting up system prompts.",
"warning",
{ clear: true }
);
showToast(t("main-page.noWorkspaceError"), "warning", {
clear: true,
});
showNewWsModal();
return false;
}
@ -130,9 +130,11 @@ export const CHECKLIST_ITEMS = [
},
{
id: "define_slash_command",
title: "Define a slash command",
description: "Create custom commands for your assistant",
action: "Define",
title: t("main-page.checklist.tasks.define_slash_command.title"),
description: t(
"main-page.checklist.tasks.define_slash_command.description"
),
action: t("main-page.checklist.tasks.define_slash_command.action"),
handler: ({
workspaces = [],
navigate = noop,
@ -140,11 +142,7 @@ export const CHECKLIST_ITEMS = [
showToast = noop,
}) => {
if (workspaces.length === 0) {
showToast(
"Please create a workspace before setting up slash commands.",
"warning",
{ clear: true }
);
showToast(t("main-page.noWorkspaceError"), "warning", { clear: true });
showNewWsModal();
return false;
}
@ -159,9 +157,9 @@ export const CHECKLIST_ITEMS = [
},
{
id: "visit_community",
title: "Visit Community Hub",
description: "Explore community resources and templates",
action: "Browse",
title: t("main-page.checklist.tasks.visit_community.title"),
description: t("main-page.checklist.tasks.visit_community.description"),
action: t("main-page.checklist.tasks.visit_community.action"),
handler: () => window.open(paths.communityHub.website(), "_blank"),
icon: UsersThree,
},

19
frontend/src/pages/Main/Home/Checklist/index.jsx
View File

@ -17,9 +17,11 @@ import {
} from "./constants";
import ConfettiExplosion from "react-confetti-explosion";
import { safeJsonParse } from "@/utils/request";
import { useTranslation } from "react-i18next";
const MemoizedChecklistItem = React.memo(ChecklistItem);
export default function Checklist() {
const { t } = useTranslation();
const [loading, setLoading] = useState(true);
const [isHidden, setIsHidden] = useState(false);
const [completedCount, setCompletedCount] = useState(0);
@ -70,7 +72,7 @@ export default function Checklist() {
const checklist = window.localStorage.getItem(CHECKLIST_STORAGE_KEY);
const existingChecklist = checklist ? safeJsonParse(checklist, {}) : {};
const isCompleted =
Object.keys(existingChecklist).length === CHECKLIST_ITEMS.length;
Object.keys(existingChecklist).length === CHECKLIST_ITEMS().length;
setIsCompleted(isCompleted);
if (isCompleted) return;
@ -124,7 +126,7 @@ export default function Checklist() {
const completedItems = safeJsonParse(checklist, {});
setCompletedCount(Object.keys(completedItems).length);
setIsCompleted(
Object.keys(completedItems).length === CHECKLIST_ITEMS.length
Object.keys(completedItems).length === CHECKLIST_ITEMS().length
);
} catch (error) {
console.error(error);
@ -155,7 +157,7 @@ export default function Checklist() {
className="bg-[rgba(54,70,61,0.5)] light:bg-[rgba(216,243,234,0.5)] w-full h-full flex items-center justify-center bg-theme-checklist-item-completed-bg/50 rounded-lg"
>
<p className="text-theme-checklist-item-completed-text text-lg font-bold">
You're on your way to becoming an AnythingLLM expert!
{t("main-page.checklist.completed")}
</p>
</div>
</div>
@ -166,11 +168,12 @@ export default function Checklist() {
<div className="flex justify-between items-center mb-4">
<div className="flex items-center gap-x-3">
<h1 className="text-theme-home-text uppercase text-sm font-semibold">
Getting Started
{t("main-page.checklist.title")}
</h1>
{CHECKLIST_ITEMS.length - completedCount > 0 && (
{CHECKLIST_ITEMS().length - completedCount > 0 && (
<p className="text-theme-home-text-secondary text-xs">
{CHECKLIST_ITEMS.length - completedCount} tasks left
{CHECKLIST_ITEMS().length - completedCount}{" "}
{t("main-page.checklist.tasksLeft")}
</p>
)}
</div>
@ -180,12 +183,12 @@ export default function Checklist() {
onClick={handleClose}
className="text-theme-home-text-secondary bg-theme-home-bg-button px-3 py-1 rounded-xl hover:bg-white/10 transition-colors text-xs light:bg-black-100"
>
close
{t("main-page.checklist.dismiss")}
</button>
</div>
</div>
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
{CHECKLIST_ITEMS.map((item) => (
{CHECKLIST_ITEMS().map((item) => (
<MemoizedChecklistItem
key={item.id}
id={item.id}

46
frontend/src/pages/Main/Home/ExploreFeatures/index.jsx
View File

@ -1,8 +1,10 @@
import { useNavigate } from "react-router-dom";
import paths from "@/utils/paths";
import Workspace from "@/models/workspace";
import { useTranslation } from "react-i18next";
export default function ExploreFeatures() {
const { t } = useTranslation();
const navigate = useNavigate();
const chatWithAgent = async () => {
@ -53,32 +55,50 @@ export default function ExploreFeatures() {
return (
<div>
<h1 className="text-theme-home-text uppercase text-sm font-semibold mb-4">
Explore more features
{t("main-page.exploreMore.title")}
</h1>
<div className="w-full grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
<FeatureCard
title="Custom AI Agents"
description="Build powerful AI Agents and automations with no code."
primaryAction="Chat using @agent"
secondaryAction="Build an agent flow"
title={t("main-page.exploreMore.features.customAgents.title")}
description={t(
"main-page.exploreMore.features.customAgents.description"
)}
primaryAction={t(
"main-page.exploreMore.features.customAgents.primaryAction"
)}
secondaryAction={t(
"main-page.exploreMore.features.customAgents.secondaryAction"
)}
onPrimaryAction={chatWithAgent}
onSecondaryAction={buildAgentFlow}
isNew={true}
/>
<FeatureCard
title="Slash Commands"
description="Save time and inject prompts using custom slash commands."
primaryAction="Create a Slash Command"
secondaryAction="Explore on Hub"
title={t("main-page.exploreMore.features.slashCommands.title")}
description={t(
"main-page.exploreMore.features.slashCommands.description"
)}
primaryAction={t(
"main-page.exploreMore.features.slashCommands.primaryAction"
)}
secondaryAction={t(
"main-page.exploreMore.features.slashCommands.secondaryAction"
)}
onPrimaryAction={setSlashCommand}
onSecondaryAction={exploreSlashCommands}
isNew={false}
/>
<FeatureCard
title="System Prompts"
description="Modify the system prompt to customize the AI replies of a workspace."
primaryAction="Modify a System Prompt"
secondaryAction="Manage prompt variables"
title={t("main-page.exploreMore.features.systemPrompts.title")}
description={t(
"main-page.exploreMore.features.systemPrompts.description"
)}
primaryAction={t(
"main-page.exploreMore.features.systemPrompts.primaryAction"
)}
secondaryAction={t(
"main-page.exploreMore.features.systemPrompts.secondaryAction"
)}
onPrimaryAction={setSystemPrompt}
onSecondaryAction={managePromptVariables}
isNew={true}

26
frontend/src/pages/Main/Home/QuickLinks/index.jsx
View File

@ -8,8 +8,10 @@ import { useState } from "react";
import { useNewWorkspaceModal } from "@/components/Modals/NewWorkspace";
import NewWorkspaceModal from "@/components/Modals/NewWorkspace";
import showToast from "@/utils/toast";
import { useTranslation } from "react-i18next";
export default function QuickLinks() {
const { t } = useTranslation();
const navigate = useNavigate();
const { showModal } = useManageWorkspaceModal();
const [selectedWorkspace, setSelectedWorkspace] = useState(null);
@ -25,11 +27,9 @@ export default function QuickLinks() {
const firstWorkspace = workspaces[0];
navigate(paths.workspace.chat(firstWorkspace.slug));
} else {
showToast(
"Please create a workspace before starting a chat.",
"warning",
{ clear: true }
);
showToast(t("main-page.noWorkspaceError"), "warning", {
clear: true,
});
showNewWsModal();
}
};
@ -41,11 +41,9 @@ export default function QuickLinks() {
setSelectedWorkspace(firstWorkspace);
showModal();
} else {
showToast(
"Please create a workspace before embedding documents.",
"warning",
{ clear: true }
);
showToast(t("main-page.noWorkspaceError"), "warning", {
clear: true,
});
showNewWsModal();
}
};
@ -57,7 +55,7 @@ export default function QuickLinks() {
return (
<div>
<h1 className="text-theme-home-text uppercase text-sm font-semibold mb-4">
Quick Links
{t("main-page.quickLinks.title")}
</h1>
<div className="w-full grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
<button
@ -65,21 +63,21 @@ export default function QuickLinks() {
className="h-[45px] text-sm font-semibold bg-theme-home-button-secondary rounded-lg text-theme-home-button-secondary-text flex items-center justify-center gap-x-2.5 transition-all duration-200 hover:bg-theme-home-button-secondary-hover hover:text-theme-home-button-secondary-hover-text"
>
<ChatCenteredDots size={16} />
Send Chat
{t("main-page.quickLinks.sendChat")}
</button>
<button
onClick={embedDocument}
className="h-[45px] text-sm font-semibold bg-theme-home-button-secondary rounded-lg text-theme-home-button-secondary-text flex items-center justify-center gap-x-2.5 transition-all duration-200 hover:bg-theme-home-button-secondary-hover hover:text-theme-home-button-secondary-hover-text"
>
<FileArrowDown size={16} />
Embed a Document
{t("main-page.quickLinks.embedDocument")}
</button>
<button
onClick={createWorkspace}
className="h-[45px] text-sm font-semibold bg-theme-home-button-secondary rounded-lg text-theme-home-button-secondary-text flex items-center justify-center gap-x-2.5 transition-all duration-200 hover:bg-theme-home-button-secondary-hover hover:text-theme-home-button-secondary-hover-text"
>
<Plus size={16} />
Create Workspace
{t("main-page.quickLinks.createWorkspace")}
</button>
</div>

8
frontend/src/pages/Main/Home/Resources/index.jsx
View File

@ -1,11 +1,13 @@
import paths from "@/utils/paths";
import { ArrowCircleUpRight } from "@phosphor-icons/react";
import { useTranslation } from "react-i18next";
export default function Resources() {
const { t } = useTranslation();
return (
<div>
<h1 className="text-theme-home-text uppercase text-sm font-semibold mb-4">
Resources
{t("main-page.resources.title")}
</h1>
<div className="flex gap-x-6">
<a
@ -14,7 +16,7 @@ export default function Resources() {
href={paths.docs()}
className="text-theme-home-text text-sm flex items-center gap-x-2 hover:opacity-70"
>
Docs
{t("main-page.resources.links.docs")}
<ArrowCircleUpRight weight="fill" size={16} />
</a>
<a
@ -23,7 +25,7 @@ export default function Resources() {
rel="noopener noreferrer"
className="text-theme-home-text text-sm flex items-center gap-x-2 hover:opacity-70"
>
Star on Github
{t("main-page.resources.links.star")}
<ArrowCircleUpRight weight="fill" size={16} />
</a>
</div>

5
frontend/src/pages/Main/Home/Updates/index.jsx
View File

@ -1,10 +1,10 @@
import { useEffect, useState } from "react";
import { safeJsonParse } from "@/utils/request";
import { ArrowSquareOut } from "@phosphor-icons/react";
import { Link } from "react-router-dom";
import PlaceholderOne from "@/media/announcements/placeholder-1.png";
import PlaceholderTwo from "@/media/announcements/placeholder-2.png";
import PlaceholderThree from "@/media/announcements/placeholder-3.png";
import { useTranslation } from "react-i18next";
/**
* @typedef {Object} NewsItem
@ -30,13 +30,14 @@ function randomPlaceholder() {
}
export default function Updates() {
const { t } = useTranslation();
const { isLoading, news } = useNewsItems();
if (isLoading || !news?.length) return null;
return (
<div>
<h1 className="text-theme-home-text uppercase text-sm font-semibold mb-4">
Updates & Announcements
{t("main-page.announcements.title")}
</h1>
<div className="w-full grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
{news.map((item, index) => (

4
server/.env.example
View File

@ -311,6 +311,10 @@ TTS_PROVIDER="native"
# See https://docs.anythingllm.com/configuration#simple-sso-passthrough for more information.
# SIMPLE_SSO_ENABLED=1
# Allow scraping of any IP address in collector - must be string "true" to be enabled
# See https://docs.anythingllm.com/configuration#local-ip-address-scraping for more information.
# COLLECTOR_ALLOW_ANY_IP="true"
# Specify the target languages for when using OCR to parse images and PDFs.
# This is a comma separated list of language codes as a string. Unsupported languages will be ignored.
# Default is English. See https://tesseract-ocr.github.io/tessdoc/Data-Files-in-different-versions.html for a list of valid language codes.

18
server/endpoints/api/document/index.js
View File

@ -322,7 +322,11 @@ function apiDocumentEndpoints(app) {
type: 'object',
example: {
"link": "https://anythingllm.com",
"addToWorkspaces": "workspace1,workspace2"
"addToWorkspaces": "workspace1,workspace2",
"scraperHeaders": {
"Authorization": "Bearer token123",
"My-Custom-Header": "value"
}
}
}
}
@ -365,7 +369,11 @@ function apiDocumentEndpoints(app) {
*/
try {
const Collector = new CollectorApi();
const { link, addToWorkspaces = "" } = reqBody(request);
const {
link,
addToWorkspaces = "",
scraperHeaders = {},
} = reqBody(request);
const processingOnline = await Collector.online();
if (!processingOnline) {
@ -379,8 +387,10 @@ function apiDocumentEndpoints(app) {
return;
}
const { success, reason, documents } =
await Collector.processLink(link);
const { success, reason, documents } = await Collector.processLink(
link,
scraperHeaders
);
if (!success) {
response
.status(500)

21
server/endpoints/extensions/index.js
View File

@ -127,6 +127,27 @@ function extensionEndpoints(app) {
}
}
);
app.post(
"/ext/drupalwiki",
[validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const responseFromProcessor =
await new CollectorApi().forwardExtensionRequest({
endpoint: "/ext/drupalwiki",
method: "POST",
body: request.body,
});
await Telemetry.sendTelemetry("extension_invoked", {
type: "drupalwiki",
});
response.status(200).json(responseFromProcessor);
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
}
module.exports = { extensionEndpoints };

4
server/jobs/sync-watched-documents.js
View File

@ -34,7 +34,7 @@ const { DocumentSyncRun } = require('../models/documentSyncRun.js');
continue;
}
if (type === 'link' || type === 'youtube') {
if (['link', 'youtube'].includes(type)) {
const response = await collector.forwardExtensionRequest({
endpoint: "/ext/resync-source-document",
method: "POST",
@ -46,7 +46,7 @@ const { DocumentSyncRun } = require('../models/documentSyncRun.js');
newContent = response?.content;
}
if (type === 'confluence' || type === 'github' || type === 'gitlab') {
if (['confluence', 'github', 'gitlab', 'drupalwiki'].includes(type)) {
const response = await collector.forwardExtensionRequest({
endpoint: "/ext/resync-source-document",
method: "POST",

10
server/models/documentSyncQueue.js
View File

@ -10,7 +10,14 @@ const { Telemetry } = require("./telemetry");
const DocumentSyncQueue = {
featureKey: "experimental_live_file_sync",
// update the validFileTypes and .canWatch properties when adding elements here.
validFileTypes: ["link", "youtube", "confluence", "github", "gitlab"],
validFileTypes: [
"link",
"youtube",
"confluence",
"github",
"gitlab",
"drupalwiki",
],
defaultStaleAfter: 604800000,
maxRepeatFailures: 5, // How many times a run can fail in a row before pruning.
writable: [],
@ -52,6 +59,7 @@ const DocumentSyncQueue = {
if (chunkSource.startsWith("confluence://")) return true; // If is a confluence document link
if (chunkSource.startsWith("github://")) return true; // If is a GitHub file reference
if (chunkSource.startsWith("gitlab://")) return true; // If is a GitLab file reference
if (chunkSource.startsWith("drupalwiki://")) return true; // If is a DrupalWiki document link
return false;
},

6
server/swagger/openapi.json
View File

@ -1092,7 +1092,11 @@
"type": "object",
"example": {
"link": "https://anythingllm.com",
"addToWorkspaces": "workspace1,workspace2"
"addToWorkspaces": "workspace1,workspace2",
"scraperHeaders": {
"Authorization": "Bearer token123",
"My-Custom-Header": "value"
}
}
}
}

64
server/utils/collectorApi/index.js
View File

@ -1,5 +1,14 @@
const { EncryptionManager } = require("../EncryptionManager");
/**
* @typedef {Object} CollectorOptions
* @property {string} whisperProvider - The provider to use for whisper, defaults to "local"
* @property {string} WhisperModelPref - The model to use for whisper if set.
* @property {string} openAiKey - The API key to use for OpenAI interfacing, mostly passed to OAI Whisper provider.
* @property {Object} ocr - The OCR options
* @property {{allowAnyIp: "true"|null|undefined}} runtimeSettings - The runtime settings that are passed to the collector. Persisted across requests.
*/
// When running locally will occupy the 0.0.0.0 hostname space but when deployed inside
// of docker this endpoint is not exposed so it is only on the Docker instances internal network
// so no additional security is needed on the endpoint directly. Auth is done however by the express
@ -15,6 +24,10 @@ class CollectorApi {
console.log(`\x1b[36m[CollectorApi]\x1b[0m ${text}`, ...args);
}
/**
* Attach options to the request passed to the collector API
* @returns {CollectorOptions}
*/
#attachOptions() {
return {
whisperProvider: process.env.WHISPER_PROVIDER || "local",
@ -23,6 +36,9 @@ class CollectorApi {
ocr: {
langList: process.env.TARGET_OCR_LANG || "eng",
},
runtimeSettings: {
allowAnyIp: process.env.COLLECTOR_ALLOW_ANY_IP ?? "false",
},
};
}
@ -45,6 +61,12 @@ class CollectorApi {
});
}
/**
* Process a document
* - Will append the options to the request body
* @param {string} filename - The filename of the document to process
* @returns {Promise<Object>} - The response from the collector API
*/
async processDocument(filename = "") {
if (!filename) return false;
@ -75,10 +97,22 @@ class CollectorApi {
});
}
async processLink(link = "") {
/**
* Process a link
* - Will append the options to the request body
* @param {string} link - The link to process
* @param {{[key: string]: string}} scraperHeaders - Custom headers to apply to the web-scraping request URL
* @returns {Promise<Object>} - The response from the collector API
*/
async processLink(link = "", scraperHeaders = {}) {
if (!link) return false;
const data = JSON.stringify({ link });
const data = JSON.stringify({
link,
scraperHeaders,
options: this.#attachOptions(),
});
return await fetch(`${this.endpoint}/process-link`, {
method: "POST",
headers: {
@ -101,8 +135,19 @@ class CollectorApi {
});
}
/**
* Process raw text as a document for the collector
* - Will append the options to the request body
* @param {string} textContent - The text to process
* @param {Object} metadata - The metadata to process
* @returns {Promise<Object>} - The response from the collector API
*/
async processRawText(textContent = "", metadata = {}) {
const data = JSON.stringify({ textContent, metadata });
const data = JSON.stringify({
textContent,
metadata,
options: this.#attachOptions(),
});
return await fetch(`${this.endpoint}/process-raw-text`, {
method: "POST",
headers: {
@ -151,10 +196,21 @@ class CollectorApi {
});
}
/**
* Get the content of a link only in a specific format
* - Will append the options to the request body
* @param {string} link - The link to get the content of
* @param {"text"|"html"} captureAs - The format to capture the content as
* @returns {Promise<Object>} - The response from the collector API
*/
async getLinkContent(link = "", captureAs = "text") {
if (!link) return false;
const data = JSON.stringify({ link, captureAs });
const data = JSON.stringify({
link,
captureAs,
options: this.#attachOptions(),
});
return await fetch(`${this.endpoint}/util/get-link`, {
method: "POST",
headers: {

3
server/utils/helpers/updateENV.js
View File

@ -958,6 +958,9 @@ function dumpENV() {
// OCR Language Support
"TARGET_OCR_LANG",
// Collector API common ENV - allows bypassing URL validation checks
"COLLECTOR_ALLOW_ANY_IP",
];
// Simple sanitization of each value to prevent ENV injection via newline or quote escaping.