michael/merlyn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rescope validatedLink to local var

Browse Source
This commit is contained in:
timothycarambat 2025-10-07 12:08:53 -07:00
parent cf3fbcbf0f
commit 5173c75113
2 changed files with 9 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
collector/extensions/index.js
View File

@ -119,9 +119,9 @@ function extensions(app) {
try {
const websiteDepth = require("../utils/extensions/WebsiteDepth");
const { url, depth = 1, maxLinks = 20 } = reqBody(request);
url = validateURL(url);
if (!validURL(url)) throw new Error("Not a valid URL.");
const scrapedData = await websiteDepth(url, depth, maxLinks);
const validatedUrl = validateURL(url);
if (!validURL(validatedUrl)) throw new Error("Not a valid URL.");
const scrapedData = await websiteDepth(validatedUrl, depth, maxLinks);
response.status(200).json({ success: true, data: scrapedData });
} catch (e) {
console.error(e);

12
collector/processLink/index.js
View File

@ -11,10 +11,10 @@ const { validateURL } = require("../utils/url");
* @returns {Promise<{success: boolean, content: string}>} - Response from collector
*/
async function processLink(link, scraperHeaders = {}, metadata = {}) {
link = validateURL(link);
if (!validURL(link)) return { success: false, reason: "Not a valid URL." };
const validatedLink = validateURL(link);
if (!validURL(validatedLink)) return { success: false, reason: "Not a valid URL." };
return await scrapeGenericUrl({
link,
link: validatedLink,
captureAs: "text",
scraperHeaders,
metadata,
@ -30,10 +30,10 @@ async function processLink(link, scraperHeaders = {}, metadata = {}) {
* @returns {Promise<{success: boolean, content: string}>} - Response from collector
*/
async function getLinkText(link, captureAs = "text") {
link = validateURL(link);
if (!validURL(link)) return { success: false, reason: "Not a valid URL." };
const validatedLink = validateURL(link);
if (!validURL(validatedLink)) return { success: false, reason: "Not a valid URL." };
return await scrapeGenericUrl({
link,
link: validatedLink,
captureAs,
saveAsDocument: false,
});