michael/merlyn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix loop logic for fetchNextPage use in GitLabLoader (#4662)

Browse Source 
resolves #4626
closes #4627
This commit is contained in:
Timothy Carambat 2025-11-19 13:53:26 -08:00 committed by GitHub
parent ea336de3c3
commit b3b261e15d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js
View File

@ -199,6 +199,7 @@ class GitLabRepoLoader {
let branchesPage = []; let branchesPage = [];
while ((branchesPage = await this.fetchNextPage(branchesRequestData))) { while ((branchesPage = await this.fetchNextPage(branchesRequestData))) {
if (!Array.isArray(branchesPage) || !branchesPage?.length) break;
this.branches.push(...branchesPage.map((branch) => branch.name)); this.branches.push(...branchesPage.map((branch) => branch.name));
} }
return this.#branchPrefSort(this.branches); return this.#branchPrefSort(this.branches);
@ -221,6 +222,7 @@ class GitLabRepoLoader {
let filesPage = null; let filesPage = null;
let pagePromises = []; let pagePromises = [];
while ((filesPage = await this.fetchNextPage(filesRequestData))) { while ((filesPage = await this.fetchNextPage(filesRequestData))) {
if (!Array.isArray(filesPage) || !filesPage?.length) break;
// Fetch all the files that are not ignored in parallel. // Fetch all the files that are not ignored in parallel.
pagePromises = filesPage pagePromises = filesPage
.filter((file) => { .filter((file) => {
@ -258,6 +260,7 @@ class GitLabRepoLoader {
let issuesPage = null; let issuesPage = null;
let pagePromises = []; let pagePromises = [];
while ((issuesPage = await this.fetchNextPage(issuesRequestData))) { while ((issuesPage = await this.fetchNextPage(issuesRequestData))) {
if (!Array.isArray(issuesPage) || !issuesPage?.length) break;
// Fetch all the issues in parallel. // Fetch all the issues in parallel.
pagePromises = issuesPage.map(async (issue) => { pagePromises = issuesPage.map(async (issue) => {
const discussionsRequestData = { const discussionsRequestData = {
@ -269,6 +272,7 @@ class GitLabRepoLoader {
while ( while (
(discussionPage = await this.fetchNextPage(discussionsRequestData)) (discussionPage = await this.fetchNextPage(discussionsRequestData))
) { ) {
if (!Array.isArray(discussionPage) || !discussionPage?.length) break;
discussions.push( discussions.push(
...discussionPage.map(({ notes }) => ...discussionPage.map(({ notes }) =>
notes.map( notes.map(
@ -308,6 +312,7 @@ ${body}`
}; };
const wikiPages = await this.fetchNextPage(wikiRequestData); const wikiPages = await this.fetchNextPage(wikiRequestData);
if (!Array.isArray(wikiPages)) return [];
console.log(`Total wiki pages fetched: ${wikiPages.length}`); console.log(`Total wiki pages fetched: ${wikiPages.length}`);
return wikiPages; return wikiPages;
} }