michael/merlyn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update writeToServerDocuments to take config object (#4213)

Browse Source
This commit is contained in:
Timothy Carambat 2025-07-29 17:53:05 -07:00 committed by GitHub
parent 7692775942
commit 70a07b743b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 81 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
collector/processLink/convert/generic.js
View File

@ -62,10 +62,10 @@ async function scrapeGenericUrl({
token_count_estimate: tokenizeString(content),
};
const document = writeToServerDocuments(
const document = writeToServerDocuments({
data,
`url-${slugify(filename)}-${data.id}`
);
filename: `url-${slugify(filename)}-${data.id}`,
});
console.log(`[SUCCESS]: URL ${link} converted & ready for embedding.\n`);
return { success: true, reason: null, documents: [document] };
}

6
collector/processRawText/index.js
View File

@ -58,10 +58,10 @@ async function processRawText(textContent, metadata) {
token_count_estimate: tokenizeString(textContent),
};
const document = writeToServerDocuments(
const document = writeToServerDocuments({
data,
`raw-${stripAndSlug(metadata.title)}-${data.id}`
);
filename: `raw-${stripAndSlug(metadata.title)}-${data.id}`,
});
console.log(`[SUCCESS]: Raw text and metadata saved & ready for embedding.\n`);
return { success: true, reason: null, documents: [document] };
}

6
collector/processSingleFile/convert/asAudio.js
View File

@ -59,10 +59,10 @@ async function asAudio({ fullFilePath = "", filename = "", options = {} }) {
token_count_estimate: tokenizeString(content),
};
const document = writeToServerDocuments(
const document = writeToServerDocuments({
data,
`${slugify(filename)}-${data.id}`
);
filename: `${slugify(filename)}-${data.id}`,
});
trashFile(fullFilePath);
console.log(
`[SUCCESS]: ${filename} transcribed, converted & ready for embedding.\n`

6
collector/processSingleFile/convert/asDocx.js
View File

@ -45,10 +45,10 @@ async function asDocX({ fullFilePath = "", filename = "" }) {
token_count_estimate: tokenizeString(content),
};
const document = writeToServerDocuments(
const document = writeToServerDocuments({
data,
`${slugify(filename)}-${data.id}`
);
filename: `${slugify(filename)}-${data.id}`,
});
trashFile(fullFilePath);
console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`);
return { success: true, reason: null, documents: [document] };

6
collector/processSingleFile/convert/asEPub.js
View File

@ -43,10 +43,10 @@ async function asEPub({ fullFilePath = "", filename = "" }) {
token_count_estimate: tokenizeString(content),
};
const document = writeToServerDocuments(
const document = writeToServerDocuments({
data,
`${slugify(filename)}-${data.id}`
);
filename: `${slugify(filename)}-${data.id}`,
});
trashFile(fullFilePath);
console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`);
return { success: true, reason: null, documents: [document] };

6
collector/processSingleFile/convert/asImage.js
View File

@ -38,10 +38,10 @@ async function asImage({ fullFilePath = "", filename = "", options = {} }) {
token_count_estimate: tokenizeString(content),
};
const document = writeToServerDocuments(
const document = writeToServerDocuments({
data,
`${slugify(filename)}-${data.id}`
);
filename: `${slugify(filename)}-${data.id}`,
});
trashFile(fullFilePath);
console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`);
return { success: true, reason: null, documents: [document] };

6
collector/processSingleFile/convert/asMbox.js
View File

@ -57,10 +57,10 @@ async function asMbox({ fullFilePath = "", filename = "" }) {
};
item++;
const document = writeToServerDocuments(
const document = writeToServerDocuments({
data,
`${slugify(filename)}-${data.id}-msg-${item}`
);
filename: `${slugify(filename)}-${data.id}-msg-${item}`,
});
documents.push(document);
}

6
collector/processSingleFile/convert/asOfficeMime.js
View File

@ -41,10 +41,10 @@ async function asOfficeMime({ fullFilePath = "", filename = "" }) {
token_count_estimate: tokenizeString(content),
};
const document = writeToServerDocuments(
const document = writeToServerDocuments({
data,
`${slugify(filename)}-${data.id}`
);
filename: `${slugify(filename)}-${data.id}`,
});
trashFile(fullFilePath);
console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`);
return { success: true, reason: null, documents: [document] };

6
collector/processSingleFile/convert/asPDF/index.js
View File

@ -62,10 +62,10 @@ async function asPdf({ fullFilePath = "", filename = "", options = {} }) {
token_count_estimate: tokenizeString(content),
};
const document = writeToServerDocuments(
const document = writeToServerDocuments({
data,
`${slugify(filename)}-${data.id}`
);
filename: `${slugify(filename)}-${data.id}`,
});
trashFile(fullFilePath);
console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`);
return { success: true, reason: null, documents: [document] };

6
collector/processSingleFile/convert/asTxt.js
View File

@ -41,10 +41,10 @@ async function asTxt({ fullFilePath = "", filename = "" }) {
token_count_estimate: tokenizeString(content),
};
const document = writeToServerDocuments(
const document = writeToServerDocuments({
data,
`${slugify(filename)}-${data.id}`
);
filename: `${slugify(filename)}-${data.id}`,
});
trashFile(fullFilePath);
console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`);
return { success: true, reason: null, documents: [document] };

10
collector/processSingleFile/convert/asXlsx.js
View File

@ -64,11 +64,11 @@ async function asXlsx({ fullFilePath = "", filename = "" }) {
token_count_estimate: tokenizeString(content),
};
const document = writeToServerDocuments(
sheetData,
`sheet-${slugify(name)}`,
outFolderPath
);
const document = writeToServerDocuments({
data: sheetData,
filename: `sheet-${slugify(name)}`,
destinationOverride: outFolderPath,
});
documents.push(document);
console.log(
`[SUCCESS]: Sheet "${name}" converted & ready for embedding.`

6
collector/utils/extensions/Confluence/index.js
View File

@ -116,7 +116,11 @@ async function loadConfluence(
const fileName = sanitizeFileName(
`${slugify(doc.metadata.title)}-${data.id}`
);
writeToServerDocuments(data, fileName, outFolderPath);
writeToServerDocuments({
data,
filename: fileName,
destinationOverride: outFolderPath,
});
});
return {

6
collector/utils/extensions/DrupalWiki/DrupalWiki/index.js
View File

@ -202,7 +202,11 @@ class DrupalWiki {
console.log(
`[DrupalWiki Loader]: Saving page '${page.title}' (${page.id}) to '${this.storagePath}/${fileName}'`
);
writeToServerDocuments(data, fileName, this.storagePath);
writeToServerDocuments({
data,
filename: fileName,
destinationOverride: this.storagePath,
});
}
/**

6
collector/utils/extensions/ObsidianVault/index.js
View File

@ -66,7 +66,11 @@ async function loadObsidianVault({ files = [] }) {
const targetFileName = sanitizeFileName(
`${slugify(file.name)}-${data.id}`
);
writeToServerDocuments(data, targetFileName, outFolderPath);
writeToServerDocuments({
data,
filename: targetFileName,
destinationOverride: outFolderPath,
});
results.push({ file: file.path, status: "success" });
} catch (e) {
console.error(`Failed to process ${file.path}:`, e);

8
collector/utils/extensions/RepoLoader/GithubRepo/index.js
View File

@ -71,11 +71,11 @@ async function loadGithubRepo(args, response) {
console.log(
`[GitHub Loader]: Saving ${doc.metadata.source} to ${outFolder}`
);
writeToServerDocuments(
writeToServerDocuments({
data,
`${slugify(doc.metadata.source)}-${data.id}`,
outFolderPath
);
filename: `${slugify(doc.metadata.source)}-${data.id}`,
destinationOverride: outFolderPath,
});
}
return {

8
collector/utils/extensions/RepoLoader/GitlabRepo/index.js
View File

@ -98,11 +98,11 @@ async function loadGitlabRepo(args, response) {
`[GitLab Loader]: Saving ${doc.metadata.source} to ${outFolder}`
);
writeToServerDocuments(
writeToServerDocuments({
data,
sanitizeFileName(`${slugify(doc.metadata.source)}-${data.id}`),
outFolderPath
);
filename: sanitizeFileName(`${slugify(doc.metadata.source)}-${data.id}`),
destinationOverride: outFolderPath,
});
}
return {

6
collector/utils/extensions/WebsiteDepth/index.js
View File

@ -125,7 +125,11 @@ async function bulkScrapePages(links, outFolderPath) {
token_count_estimate: tokenizeString(content),
};
writeToServerDocuments(data, data.title, outFolderPath);
writeToServerDocuments({
data,
filename: data.title,
destinationOverride: outFolderPath,
});
scrapedData.push(data);
console.log(`Successfully scraped ${link}.`);

8
collector/utils/extensions/YoutubeTranscript/index.js
View File

@ -116,11 +116,11 @@ async function loadYouTubeTranscript({ url }) {
};
console.log(`[YouTube Loader]: Saving ${metadata.title} to ${outFolder}`);
writeToServerDocuments(
writeToServerDocuments({
data,
sanitizeFileName(`${slugify(metadata.title)}-${data.id}`),
outFolderPath
);
filename: sanitizeFileName(`${slugify(metadata.title)}-${data.id}`),
destinationOverride: outFolderPath,
});
return {
success: true,

18
collector/utils/files/index.js
View File

@ -96,11 +96,21 @@ function createdDate(filepath) {
}
}
function writeToServerDocuments(
/**
* Writes a document to the server documents folder.
* @param {Object} params - The parameters for the function.
* @param {Object} params.data - The data to write to the file. Must look like a document object.
* @param {string} params.filename - The name of the file to write to.
* @param {string|null} params.destinationOverride - A forced destination to write to - will be honored if provided.
* @returns {Object} - The data with the location added.
*/
function writeToServerDocuments({
data = {},
filename,
destinationOverride = null
) {
filename = null,
destinationOverride = null,
}) {
if (!filename) throw new Error("Filename is required!");
let destination = null;
if (destinationOverride) destination = path.resolve(destinationOverride);
else destination = path.resolve(documentsFolder, "custom-documents");