f7b90571be
* Add capability to web scraping feature for document creation to download and parse statically hosted files * lint * Remove unneeded comment * Simplified process by using key of ACCEPTED_MIMES to validate the response content type, as a result unlocked all supported files * Add TODO comments for future implementation of asDoc.js to handle standard MS Word files in constants.js * Return captureAs argument to be exposed by scrapeGenericUrl and passed into getPageContent | Return explicit argument of captureAs into scrapeGenericUrl in processLink fn * Return debug log for scrapeGenericUrl * Change conditional to a guard clause. * Add error handling, validation, and JSDOC to getContentType helper fn * remove unneeded comments * Simplify URL validation by reusing module * Rename downloadFileToHotDir to downloadURIToFile and moved up to a global module | Add URL valuidation to downloadURIToFile * refactor * add support for webp remove unused imports --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
88 lines
2.6 KiB
JavaScript
88 lines
2.6 KiB
JavaScript
const path = require("path");
|
|
const fs = require("fs");
|
|
const {
|
|
WATCH_DIRECTORY,
|
|
SUPPORTED_FILETYPE_CONVERTERS,
|
|
} = require("../utils/constants");
|
|
const {
|
|
trashFile,
|
|
isTextType,
|
|
normalizePath,
|
|
isWithin,
|
|
} = require("../utils/files");
|
|
const RESERVED_FILES = ["__HOTDIR__.md"];
|
|
|
|
/**
|
|
* Process a single file and return the documents
|
|
* @param {string} targetFilename - The filename to process
|
|
* @param {Object} options - The options for the file processing
|
|
* @param {boolean} options.parseOnly - If true, the file will not be saved as a document even when `writeToServerDocuments` is called in the handler. Must be explicitly set to true to use.
|
|
* @param {Object} metadata - The metadata for the file processing
|
|
* @returns {Promise<{success: boolean, reason: string, documents: Object[]}>} - The documents from the file processing
|
|
*/
|
|
async function processSingleFile(targetFilename, options = {}, metadata = {}) {
|
|
const fullFilePath = path.resolve(
|
|
WATCH_DIRECTORY,
|
|
normalizePath(targetFilename)
|
|
);
|
|
if (!isWithin(path.resolve(WATCH_DIRECTORY), fullFilePath))
|
|
return {
|
|
success: false,
|
|
reason: "Filename is a not a valid path to process.",
|
|
documents: [],
|
|
};
|
|
|
|
if (RESERVED_FILES.includes(targetFilename))
|
|
return {
|
|
success: false,
|
|
reason: "Filename is a reserved filename and cannot be processed.",
|
|
documents: [],
|
|
};
|
|
if (!fs.existsSync(fullFilePath))
|
|
return {
|
|
success: false,
|
|
reason: "File does not exist in upload directory.",
|
|
documents: [],
|
|
};
|
|
|
|
const fileExtension = path.extname(fullFilePath).toLowerCase();
|
|
if (fullFilePath.includes(".") && !fileExtension) {
|
|
return {
|
|
success: false,
|
|
reason: `No file extension found. This file cannot be processed.`,
|
|
documents: [],
|
|
};
|
|
}
|
|
|
|
let processFileAs = fileExtension;
|
|
if (!SUPPORTED_FILETYPE_CONVERTERS.hasOwnProperty(fileExtension)) {
|
|
if (isTextType(fullFilePath)) {
|
|
console.log(
|
|
`\x1b[33m[Collector]\x1b[0m The provided filetype of ${fileExtension} does not have a preset and will be processed as .txt.`
|
|
);
|
|
processFileAs = ".txt";
|
|
} else {
|
|
trashFile(fullFilePath);
|
|
return {
|
|
success: false,
|
|
reason: `File extension ${fileExtension} not supported for parsing and cannot be assumed as text file type.`,
|
|
documents: [],
|
|
};
|
|
}
|
|
}
|
|
|
|
const FileTypeProcessor = require(SUPPORTED_FILETYPE_CONVERTERS[
|
|
processFileAs
|
|
]);
|
|
return await FileTypeProcessor({
|
|
fullFilePath,
|
|
filename: targetFilename,
|
|
options,
|
|
metadata,
|
|
});
|
|
}
|
|
|
|
module.exports = {
|
|
processSingleFile,
|
|
};
|