michael/merlyn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handle non-ascii characters in single and bulk link scraper URLs (#2495)

Browse Source 
handle non-ascii characters in urls
This commit is contained in:
Sean Hatfield 2024-10-17 17:04:00 -07:00 committed by GitHub
parent 93d7ce6d34
commit 41522cdfb4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
collector/processLink/convert/generic.js
View File

@ -27,7 +27,8 @@ async function scrapeGenericUrl(link, textOnly = false) {
}
const url = new URL(link);
const filename = (url.host + "-" + url.pathname).replace(".", "_");
const decodedPathname = decodeURIComponent(url.pathname);
const filename = `${url.hostname}${decodedPathname.replace(/\//g, '_')}`;
const data = {
id: v4(),

3
collector/utils/extensions/WebsiteDepth/index.js
View File

@ -108,7 +108,8 @@ async function bulkScrapePages(links, outFolderPath) {
}
const url = new URL(link);
const filename = (url.host + "-" + url.pathname).replace(".", "_");
const decodedPathname = decodeURIComponent(url.pathname);
const filename = `${url.hostname}${decodedPathname.replace(/\//g, '_')}`;
const data = {
id: v4(),