diff --git a/backend/src/serverless/integrations/coordinators/devtoCoordinator.ts b/backend/src/serverless/integrations/coordinators/devtoCoordinator.ts index 6d3f74f76a..9a727365b0 100644 --- a/backend/src/serverless/integrations/coordinators/devtoCoordinator.ts +++ b/backend/src/serverless/integrations/coordinators/devtoCoordinator.ts @@ -19,7 +19,7 @@ async function devtoCoordinator(): Promise { integrationId: integration.id, tenant: integration.tenantId.toString(), onboarding: false, - state: { endpoint: '', page: '' }, + state: { endpoint: '', page: '', endpoints: [] }, args: {}, } diff --git a/backend/src/serverless/integrations/coordinators/discordCoordinator.ts b/backend/src/serverless/integrations/coordinators/discordCoordinator.ts index 524603ce4d..19887e558f 100644 --- a/backend/src/serverless/integrations/coordinators/discordCoordinator.ts +++ b/backend/src/serverless/integrations/coordinators/discordCoordinator.ts @@ -16,7 +16,7 @@ async function discordCoordinator(): Promise { sleep: Math.floor(Math.random() * 1200), tenant: integration.tenantId.toString(), onboarding: false, - state: { endpoint: '', page: '' }, + state: { endpoint: '', page: '', endpoints: [] }, args: { guildId: integration.integrationIdentifier, channels: integration.settings.channels || [], diff --git a/backend/src/serverless/integrations/coordinators/slackCoordinator.ts b/backend/src/serverless/integrations/coordinators/slackCoordinator.ts index 861633460b..5f458ecec9 100644 --- a/backend/src/serverless/integrations/coordinators/slackCoordinator.ts +++ b/backend/src/serverless/integrations/coordinators/slackCoordinator.ts @@ -16,7 +16,7 @@ async function slackCoordinator(): Promise { sleep: 0, tenant: integration.tenantId.toString(), onboarding: false, - state: { endpoint: '', page: '' }, + state: { endpoint: '', page: '', endpoints: [] }, args: { channels: integration.settings.channels || [], }, diff --git a/backend/src/serverless/integrations/coordinators/twitterCoordinator.ts b/backend/src/serverless/integrations/coordinators/twitterCoordinator.ts index 4da9049645..fa3df7d000 100644 --- a/backend/src/serverless/integrations/coordinators/twitterCoordinator.ts +++ b/backend/src/serverless/integrations/coordinators/twitterCoordinator.ts @@ -16,7 +16,7 @@ async function twitterCoordinator(): Promise { sleep: 0, tenant: integration.tenantId.toString(), onboarding: false, - state: { endpoint: '', page: '' }, + state: { endpoint: '', page: '', endpoints: [] }, args: { profileId: integration.integrationIdentifier, hashtags: integration.settings.hashtags, diff --git a/backend/src/serverless/integrations/coordinators/twitterReachCoordinator.ts b/backend/src/serverless/integrations/coordinators/twitterReachCoordinator.ts index 686d9255e9..9d37a8aa96 100644 --- a/backend/src/serverless/integrations/coordinators/twitterReachCoordinator.ts +++ b/backend/src/serverless/integrations/coordinators/twitterReachCoordinator.ts @@ -15,7 +15,7 @@ async function twitterReachCoordinator(): Promise { sleep: 0, tenant: microservice.tenantId.toString(), onboarding: false, - state: { endpoint: '', page: '' }, + state: { endpoint: '', page: '', endpoints: [] }, args: { profileId: microservice.microserviceIdentifier, }, diff --git a/backend/src/serverless/integrations/iterators/__tests__/baseIteration.test.ts b/backend/src/serverless/integrations/iterators/__tests__/baseIteration.test.ts index ae25f87b99..a93a8286c2 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/baseIteration.test.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/baseIteration.test.ts @@ -34,6 +34,7 @@ describe('BaseIterator tests', () => { expect(next).toStrictEqual({ endpoint: 'endpoint1', page: '123', + endpoints: ['endpoint1', 'endpoint2', 'endpoint3'], }) }) @@ -46,6 +47,7 @@ describe('BaseIterator tests', () => { expect(next).toStrictEqual({ endpoint: 'endpoint2', page: '', + endpoints: ['endpoint2', 'endpoint3'], }) }) @@ -70,6 +72,7 @@ describe('BaseIterator tests', () => { expect(next).toStrictEqual({ endpoint: 'endpoint2', page: '', + endpoints: ['endpoint2', 'endpoint3'], }) }) @@ -82,6 +85,7 @@ describe('BaseIterator tests', () => { expect(next).toStrictEqual({ endpoint: 'endpoint2', page: '', + endpoints: ['endpoint2', 'endpoint3'], }) }) @@ -112,6 +116,7 @@ describe('BaseIterator tests', () => { expect(next).toStrictEqual({ endpoint: 'endpoint2', page: '', + endpoints: ['endpoint2', 'endpoint3'], }) }) }) @@ -130,10 +135,11 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() + expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint3', page: '' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: [] }, ]) expect(out).toStrictEqual(success) }) @@ -168,14 +174,14 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, - { endpoint: 'endpoint1', page: 'p2' }, - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint2', page: 'p1' }, - { endpoint: 'endpoint2', page: 'p2' }, - { endpoint: 'endpoint3', page: '' }, - { endpoint: 'endpoint3', page: 'p1' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: 'p1', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: 'p2', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: 'p1', endpoints: [] }, ]) expect(out).toStrictEqual(success) }) @@ -201,11 +207,11 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, - { endpoint: 'endpoint1', page: 'p2' }, - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint3', page: '' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: [] }, ]) expect(out).toStrictEqual(success) }) @@ -224,14 +230,15 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, TestIterator.limitReachedState, - { endpoint: 'endpoint1', page: 'p2' }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, ]) expect(iter.state).toStrictEqual({ endpoint: 'endpoint1', page: 'p2', + endpoints: ['endpoint1', 'endpoint2', 'endpoint3'], }) expect(out).toStrictEqual(limitReached) }) @@ -257,14 +264,15 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, TestIterator.limitReachedState, - { endpoint: 'endpoint1', page: 'p2' }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, ]) expect(iter.state).toStrictEqual({ endpoint: 'endpoint1', page: 'p2', + endpoints: ['endpoint1', 'endpoint2', 'endpoint3'], }) expect(out).toStrictEqual(limitReached) }) @@ -290,11 +298,11 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, - { endpoint: 'endpoint1', page: 'p2' }, - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint3', page: '' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: [] }, ]) expect(iter.state).toStrictEqual(TestIterator.endState) expect(out).toStrictEqual(success) @@ -324,17 +332,18 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, - { endpoint: 'endpoint1', page: 'p2' }, - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint3', page: '' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: ['endpoint3'] }, TestIterator.limitReachedState, - { endpoint: 'endpoint3', page: 'p1' }, + { endpoint: 'endpoint3', page: 'p1', endpoints: ['endpoint3'] }, ]) expect(iter.state).toStrictEqual({ endpoint: 'endpoint3', page: 'p1', + endpoints: ['endpoint3'], }) expect(out).toStrictEqual(limitReached) }) @@ -352,11 +361,12 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn, { endpoint: 'endpoint2', page: '', + endpoints: [], }) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint3', page: '' }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: [] }, ]) expect(out).toStrictEqual(success) }) @@ -379,13 +389,14 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn, { endpoint: 'endpoint2', page: 'p1', + endpoints: ['endpoint2', 'endpoint3'], }) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint2', page: 'p1' }, - { endpoint: 'endpoint2', page: 'p2' }, - { endpoint: 'endpoint3', page: '' }, - { endpoint: 'endpoint3', page: 'p1' }, + { endpoint: 'endpoint2', page: 'p1', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: 'p2', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: 'p1', endpoints: [] }, ]) expect(out).toStrictEqual(success) }) @@ -408,18 +419,20 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn, { endpoint: 'endpoint2', page: 'p1', + endpoints: ['endpoint2', 'endpoint3'], }) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint2', page: 'p1' }, - { endpoint: 'endpoint2', page: 'p2' }, - { endpoint: 'endpoint3', page: '' }, + { endpoint: 'endpoint2', page: 'p1', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: 'p2', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: ['endpoint3'] }, TestIterator.limitReachedState, - { endpoint: 'endpoint3', page: 'p1' }, + { endpoint: 'endpoint3', page: 'p1', endpoints: ['endpoint3'] }, ]) expect(iter.state).toStrictEqual({ endpoint: 'endpoint3', page: 'p1', + endpoints: ['endpoint3'], }) expect(out).toStrictEqual(limitReached) }) @@ -449,26 +462,28 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, TestIterator.limitReachedState, - { endpoint: 'endpoint1', page: 'p2' }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, // still all endpoints because next is not called ]) expect(iter.state).toStrictEqual({ endpoint: 'endpoint1', page: 'p2', + endpoints: ['endpoint1', 'endpoint2', 'endpoint3'], }) expect(out).toStrictEqual(limitReached) const iter2 = new TestIterator(itFn2, { endpoint: 'endpoint1', page: 'p2', + endpoints: [], }) const out2 = await iter2.iterate() expect(iter2.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: 'p2' }, - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint3', page: '' }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: [] }, ]) expect(out2).toStrictEqual(success) }) @@ -510,12 +525,12 @@ describe('BaseIterator tests', () => { const out = await iter.iterate() expect(out).toStrictEqual(TestIterator.success) expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, - { endpoint: 'endpoint1', page: 'p2' }, - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint3', page: '' }, - { endpoint: 'endpoint3', page: 'p3' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: 'p3', endpoints: [] }, ]) }) }) @@ -561,27 +576,27 @@ describe('BaseIterator tests', () => { describe('isFinished tests', () => { const endpoints = ['endpoint1', 'endpoint2', 'endpoint3'] it('Should return not finished for a nextPage given', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true) const nextPage = 'here' const isFinished = iter.isFinished(endpoints, 'endpoint2', nextPage) expect(isFinished).toBe(false) }) it('Should return not finished for not nextPage given but in the middle of endpoints', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true) const nextPage = undefined const isFinished = iter.isFinished(endpoints, 'endpoint2', nextPage) expect(isFinished).toBe(false) }) it('Should return not finished for page given and last endpoint', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true) const isFinished = iter.isFinished(endpoints, 'endpoint3', 'here') expect(isFinished).toBe(false) }) it('Should return finished for no nextPage given and last endpoint', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true) const isFinished = iter.isFinished(endpoints, 'endpoint3', undefined) expect(isFinished).toBe(true) }) @@ -589,47 +604,51 @@ describe('BaseIterator tests', () => { describe('Global limit reached before execution tests', () => { it('Should stop before iteration when the global limit or larger is provided as init count', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true, 18) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true, 18) const out = await iter.iterate() expect(out).toStrictEqual(TestIterator.success) expect(iter.audits).toStrictEqual([]) }) it('Should stop before iteration when the global limit or larger is provided as init count', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true, 25) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true, 25) const out = await iter.iterate() expect(out).toStrictEqual(TestIterator.success) expect(iter.audits).toStrictEqual([]) }) it('Should start before iteration when a small limit is provided as init count', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true, 5) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true, 5) const out = await iter.iterate() expect(out).toStrictEqual(TestIterator.success) expect(iter.audits).toStrictEqual([ { endpoint: 'endpoint1', page: '', + endpoints: ['endpoint2', 'endpoint3'], }, { endpoint: 'endpoint2', page: '', + endpoints: ['endpoint3'], }, { endpoint: 'endpoint3', page: '', + endpoints: [], }, ]) }) it('Should stop when a limit almost as the global limit is provided', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true, 17) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true, 17) const out = await iter.iterate() expect(out).toStrictEqual(TestIterator.success) expect(iter.audits).toStrictEqual([ { endpoint: 'endpoint1', page: '', + endpoints: [], }, ]) }) diff --git a/backend/src/serverless/integrations/iterators/__tests__/baseIteratorStatic.test.ts b/backend/src/serverless/integrations/iterators/__tests__/baseIteratorStatic.test.ts index c931164c43..01b77b4837 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/baseIteratorStatic.test.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/baseIteratorStatic.test.ts @@ -5,35 +5,35 @@ describe('Integrations worker static tests', () => { const endpoints = ['endpoint1', 'endpoint2', 'endpoint3'] describe('Get iterator tests', () => { it('Should return the endpoints to iterate for a standard state', async () => { - const state = { endpoint: 'endpoint2', page: 'here' } + const state = { endpoint: 'endpoint2', page: 'here', endpoints: [] } const iterator = BaseIterator.getEndPointsIterator(endpoints, state) expect(iterator).toStrictEqual(['endpoint2', 'endpoint3']) }) it('Should return the endpoints to iterate for a state with no page', async () => { - const state = { endpoint: 'endpoint2', page: '' } + const state = { endpoint: 'endpoint2', page: '', endpoints: [] } const iterator = BaseIterator.getEndPointsIterator(endpoints, state) expect(iterator).toStrictEqual(['endpoint2', 'endpoint3']) }) it('Should return the whole endpoints when the state is in the first endpoint with a page', async () => { - const state = { endpoint: 'endpoint1', page: 'here' } + const state = { endpoint: 'endpoint1', page: 'here', endpoints: [] } const iterator = BaseIterator.getEndPointsIterator(endpoints, state) expect(iterator).toStrictEqual(endpoints) }) it('Should return the last endpoint when the state is in the last endpoint', async () => { - const state = { endpoint: 'endpoint3', page: '' } + const state = { endpoint: 'endpoint3', page: '', endpoints: [] } const iterator = BaseIterator.getEndPointsIterator(endpoints, state) expect(iterator).toStrictEqual(['endpoint3']) }) it('Should throw an error when state.endpoint is not in endpoints', async () => { - const state = { endpoint: 'endpoint4', page: '' } + const state = { endpoint: 'endpoint4', page: '', endpoints: [] } try { BaseIterator.getEndPointsIterator(endpoints, state) } catch (error: any) { @@ -48,10 +48,12 @@ describe('Integrations worker static tests', () => { const state = BaseIterator.initState(endpoints, { endpoint: '', page: '', + endpoints: [], }) expect(state).toStrictEqual({ endpoint: 'endpoint1', page: '', + endpoints: ['endpoint1', 'endpoint2', 'endpoint3'], }) }) @@ -59,6 +61,7 @@ describe('Integrations worker static tests', () => { const givenState = { endpoint: 'endpoint2', page: 'here', + endpoints: [], } const state = BaseIterator.initState(endpoints, givenState) expect(state).toStrictEqual(givenState) diff --git a/backend/src/serverless/integrations/iterators/__tests__/devtoIterator.test.ts b/backend/src/serverless/integrations/iterators/__tests__/devtoIterator.test.ts index c77975904c..668ca0975b 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/devtoIterator.test.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/devtoIterator.test.ts @@ -76,6 +76,7 @@ async function getDevtoIterator(articles: DevtoArticle[]) { { endpoint: '', page: '', + endpoints: [], }, false, ) diff --git a/backend/src/serverless/integrations/iterators/__tests__/discordIterator.test.ts b/backend/src/serverless/integrations/iterators/__tests__/discordIterator.test.ts index 2eb875d9eb..c4e55c9f7a 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/discordIterator.test.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/discordIterator.test.ts @@ -589,7 +589,7 @@ describe('Discord iterator tests', () => { 'guild12345', 'token12345', channels, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const isOver = iter.integrationSpecificIsEndpointFinished('1', recentRecord) @@ -605,7 +605,7 @@ describe('Discord iterator tests', () => { 'guild12345', 'token12345', channels, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const isOver = iter.integrationSpecificIsEndpointFinished('4', recentRecord) @@ -621,7 +621,7 @@ describe('Discord iterator tests', () => { 'guild12345', 'token12345', channels, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const isOver = iter.integrationSpecificIsEndpointFinished('3', recentRecord) @@ -637,7 +637,7 @@ describe('Discord iterator tests', () => { 'guild12345', 'token12345', channels, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const isOver = iter.integrationSpecificIsEndpointFinished('1', oldRecord) @@ -653,7 +653,7 @@ describe('Discord iterator tests', () => { 'guild12345', 'token12345', channels, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const isOver = iter.integrationSpecificIsEndpointFinished('3', oldRecord) @@ -669,7 +669,7 @@ describe('Discord iterator tests', () => { 'guild12345', 'token12345', channels, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const isOver = iter.integrationSpecificIsEndpointFinished('4', oldRecord) @@ -685,7 +685,7 @@ describe('Discord iterator tests', () => { 'guild12345', 'token12345', channels, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const isOverGeneral = iter.isEndpointFinished('4', {}, []) diff --git a/backend/src/serverless/integrations/iterators/__tests__/githubIterator.test.ts b/backend/src/serverless/integrations/iterators/__tests__/githubIterator.test.ts index 9c4f6819d6..e7f1400324 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/githubIterator.test.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/githubIterator.test.ts @@ -38,6 +38,7 @@ async function getGithubIterator(repos: Repos, options: IRepositoryOptions) { { endpoint: '', page: '', + endpoints: [], }, true, ) diff --git a/backend/src/serverless/integrations/iterators/__tests__/slackIterator.test.ts b/backend/src/serverless/integrations/iterators/__tests__/slackIterator.test.ts index 38c931d524..7bd5d72138 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/slackIterator.test.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/slackIterator.test.ts @@ -30,7 +30,7 @@ async function getSlackIterator(members = {}, channels = [{ name: 'dev', id: 'C0 members, integrationId, mockIRepositoryOptions, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, ) } diff --git a/backend/src/serverless/integrations/iterators/__tests__/testIterator.ts b/backend/src/serverless/integrations/iterators/__tests__/testIterator.ts index 8cda4cda8e..cb12b0b631 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/testIterator.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/testIterator.ts @@ -1,11 +1,9 @@ /* eslint @typescript-eslint/no-unused-vars: 0 */ /* eslint class-methods-use-this: 0 */ -import { endpoint } from 'aws-sdk/clients/sns' import moment from 'moment' -import { Endpoint } from 'aws-sdk' import { BaseOutput, parseOutput, IntegrationResponse } from '../../types/iteratorTypes' -import { State } from '../../types/regularTypes' +import { Endpoint, State } from '../../types/regularTypes' import BaseIterator from '../baseIterator' import { AddActivitiesSingle } from '../../types/messageTypes' @@ -19,6 +17,7 @@ export default class TestIterator extends BaseIterator { static limitReachedState: State = { endpoint: '__limit', + endpoints: [], page: '__limit', } @@ -33,7 +32,7 @@ export default class TestIterator extends BaseIterator { */ constructor( transitionFn: Function, - state: State = { endpoint: '', page: '' }, + state: State = { endpoint: '', page: '', endpoints: [] }, onboarding: boolean = false, limitCount: number = 0, ) { @@ -138,4 +137,12 @@ export default class TestIterator extends BaseIterator { async iterate(maxTime: number = 12 * 60): Promise { return super.iterate(maxTime, false) } + + next(currentEndpoint: Endpoint, nextPage: string | undefined, parseOutput: parseOutput): State { + const next = super.next(currentEndpoint, nextPage, parseOutput) + if (this.audits.length > 0) { + this.audits[this.audits.length - 1].endpoints = next.endpoints + } + return next + } } diff --git a/backend/src/serverless/integrations/iterators/__tests__/twitterIterator.test.ts b/backend/src/serverless/integrations/iterators/__tests__/twitterIterator.test.ts index d1efec2698..7199890637 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/twitterIterator.test.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/twitterIterator.test.ts @@ -47,14 +47,14 @@ describe('Integrations worker static tests', () => { it('It should return the proper message', async () => { const iter = new TwitterIterator('tenant12345', 'profile12345', 'token', ['#1', '#2']) const message: TwitterIntegrationMessage = iter.getSQSBody( - { endpoint: 'hashtag/#1', page: '1' }, + { endpoint: 'hashtag/#1', page: '1', endpoints: [] }, 42, ) expect(message).toStrictEqual({ tenant: 'tenant12345', integration: PlatformType.TWITTER, onboarding: false, - state: { endpoint: 'hashtag/#1', page: '1' }, + state: { endpoint: 'hashtag/#1', page: '1', endpoints: [] }, sleep: 42, args: { profileId: 'profile12345', @@ -653,7 +653,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const { activities } = iter.parseActivities(followers, 'followers') @@ -947,7 +947,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, 10, existingFollowers, @@ -1000,7 +1000,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, 10, existingFollowers, @@ -1058,7 +1058,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, 10, existingFollowers, @@ -1098,7 +1098,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, 10, existingFollowers, @@ -1134,7 +1134,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, 10, existingFollowers, @@ -1154,7 +1154,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, 10, existingFollowers, @@ -1171,7 +1171,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, 10, existingFollowers, @@ -1220,7 +1220,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, 10, existingFollowers, @@ -1250,7 +1250,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, 10, existingFollowers, @@ -1278,7 +1278,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, 10, existingFollowers, @@ -1308,7 +1308,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, initialCount, ) @@ -1361,7 +1361,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, initialCount, ) @@ -1414,7 +1414,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, initialCount, ) @@ -1431,7 +1431,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, initialCount, ) @@ -1443,6 +1443,7 @@ describe('Integrations worker static tests', () => { expect(next).toStrictEqual({ endpoint: 'followers', page: 'p124', + endpoints: ['followers', 'mentions', 'hashtag/#1', 'hashtag/#2'], }) }) @@ -1453,7 +1454,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, initialCount, ) @@ -1471,6 +1472,7 @@ describe('Integrations worker static tests', () => { const iter = new TwitterIterator('tenant12345', 'profile12345', 'token', ['#1', '#2'], { endpoint: '', page: '', + endpoints: [], }) const date = iter.getAfterDate() expect(moment(date).unix()).toBeCloseTo( diff --git a/backend/src/serverless/integrations/iterators/baseIterator.ts b/backend/src/serverless/integrations/iterators/baseIterator.ts index e26e74a498..3ec5febd52 100644 --- a/backend/src/serverless/integrations/iterators/baseIterator.ts +++ b/backend/src/serverless/integrations/iterators/baseIterator.ts @@ -26,6 +26,7 @@ export default abstract class BaseIterator { startTimestamp: number static endState: State = { + endpoints: [], endpoint: '__finished', page: '__finished', } @@ -50,13 +51,19 @@ export default abstract class BaseIterator { constructor( tenant: string, endPoints: Endpoints, - state: State = { endpoint: '', page: '' }, + state: State = { endpoint: '', page: '', endpoints: [] }, onboarding: boolean = false, globalLimit: number = Infinity, limitCount: number = 0, ) { this.tenant = tenant - this.endpoints = endPoints + + if (state.endpoints.length > 0 && state.endpoint) { + this.endpoints = state.endpoints + } else { + this.endpoints = endPoints + } + this.state = BaseIterator.initState(this.endpoints, state) this.startTimestamp = moment().utc().unix() @@ -107,6 +114,7 @@ export default abstract class BaseIterator { while (this.state !== BaseIterator.endState) { // Get the current endpoint and page const { endpoint, page } = this.state + // Get the response from the API and parse it. // We also get the date of the latest activity const response: IntegrationResponse = await this.get(endpoint, page) @@ -138,6 +146,7 @@ export default abstract class BaseIterator { } // Get the next state this.state = this.next(endpoint, response.nextPage, parseOutput) + this.endpoints = this.state.endpoints // If we are not done if (this.state !== BaseIterator.endState) { @@ -237,8 +246,9 @@ export default abstract class BaseIterator { return lodash.isEqual(startState, { endpoint: '', page: '', + endpoints: [], }) - ? { endpoint: endpoints[0], page: '' } + ? { endpoint: endpoints[0], page: '', endpoints } : startState } @@ -292,6 +302,7 @@ export default abstract class BaseIterator { : { endpoint: this.endpointsIterator[this.endpointsIterator.indexOf(currentEndpoint) + 1], page: '', + endpoints: this.endpoints.slice(this.endpoints.indexOf(currentEndpoint) + 1), } } // If we do not have a next page, return the next endpoint with an empty page and 0 for number @@ -301,6 +312,9 @@ export default abstract class BaseIterator { ? currentEndpoint : this.endpointsIterator[this.endpointsIterator.indexOf(currentEndpoint) + 1], page: nextPage || '', + endpoints: nextPage + ? this.endpoints + : this.endpoints.slice(this.endpoints.indexOf(currentEndpoint) + 1), } } diff --git a/backend/src/serverless/integrations/iterators/devtoIterator.ts b/backend/src/serverless/integrations/iterators/devtoIterator.ts index 353f8e306e..615c5e15bc 100644 --- a/backend/src/serverless/integrations/iterators/devtoIterator.ts +++ b/backend/src/serverless/integrations/iterators/devtoIterator.ts @@ -36,7 +36,7 @@ export default class DevtoIterator extends BaseIterator { articles: DevtoArticle[], userContext: IRepositoryOptions, integrationId: string, - state: State = { endpoint: '', page: '' }, + state: State = { endpoint: '', page: '', endpoints: [] }, onboarding: boolean = false, ) { const endpoints: Endpoints = articles.map((a) => a.id.toString()) diff --git a/backend/src/serverless/integrations/iterators/discordIterator.ts b/backend/src/serverless/integrations/iterators/discordIterator.ts index d26f71c2ae..db6f5b0aee 100644 --- a/backend/src/serverless/integrations/iterators/discordIterator.ts +++ b/backend/src/serverless/integrations/iterators/discordIterator.ts @@ -20,6 +20,7 @@ import { PlatformType } from '../../../utils/platforms' export default class DiscordIterator extends BaseIterator { static limitReachedState: State = { + endpoints: [], endpoint: '__limit', page: '__limit', } @@ -56,7 +57,7 @@ export default class DiscordIterator extends BaseIterator { guildId: string, botToken: string, channels: Channels, - state: State = { endpoint: '', page: '' }, + state: State = { endpoint: '', page: '', endpoints: [] }, onboarding: boolean = false, ) { // Endpoints are the fixed endpoints plus the channels diff --git a/backend/src/serverless/integrations/iterators/githubIterator.ts b/backend/src/serverless/integrations/iterators/githubIterator.ts index 4dff257100..ddd27ec76d 100644 --- a/backend/src/serverless/integrations/iterators/githubIterator.ts +++ b/backend/src/serverless/integrations/iterators/githubIterator.ts @@ -27,6 +27,7 @@ import Error400 from '../../../errors/Error400' export default class GithubIterator extends BaseIterator { static limitReachedState: State = { + endpoints: [], endpoint: '__limit', page: '__limit', } @@ -56,16 +57,21 @@ export default class GithubIterator extends BaseIterator { tenant: string, repos: Repos, accessToken: string, - state: State = { endpoint: '', page: '' }, + state: State = { endpoints: [], endpoint: '', page: '' }, onboarding: boolean = false, ) { - const endpoints: Endpoints = repos.reduce((acc, repo) => { - const repoEndpoints = GithubIterator.fixedEndpoints.map( - (endpoint) => `${repo.name}|${endpoint}`, - ) - acc.push(...repoEndpoints) - return acc - }, []) + let endpoints: Endpoints + if (state.endpoints.length === 0) { + endpoints = repos.reduce((acc, repo) => { + const repoEndpoints = GithubIterator.fixedEndpoints.map( + (endpoint) => `${repo.name}|${endpoint}`, + ) + acc.push(...repoEndpoints) + return acc + }, []) + } else { + endpoints = state.endpoints + } super(tenant, endpoints, state, onboarding, GithubIterator.globalLimit) this.repos = repos @@ -95,7 +101,7 @@ export default class GithubIterator extends BaseIterator { const stargazersQuery = new StargazersQuery(this.getRepoByName(repoName), this.accessToken) result = await stargazersQuery.getSinglePage(page) - result.data = result.data.filter((i) => (i as any).node.login) + result.data = result.data.filter((i) => (i as any).node?.login) break } case 'pulls': { @@ -106,7 +112,7 @@ export default class GithubIterator extends BaseIterator { result = await pullRequestsQuery.getSinglePage(page) // filter out activities without authors (such as bots) - result.data = result.data.filter((i) => (i as any).author.login) + result.data = result.data.filter((i) => (i as any).author?.login) // add each PR as separate endpoint for comments as form repoName|pull-comments|id result.data.map((pr) => @@ -127,7 +133,7 @@ export default class GithubIterator extends BaseIterator { result = await pullRequestCommentsQuery.getSinglePage(page) - result.data = result.data.filter((i) => (i as any).author.login) + result.data = result.data.filter((i) => (i as any).author?.login) break } case 'issue-comments': { @@ -139,7 +145,7 @@ export default class GithubIterator extends BaseIterator { ) result = await issueCommentsQuery.getSinglePage(page) - result.data = result.data.filter((i) => (i as any).author.login) + result.data = result.data.filter((i) => (i as any).author?.login) break } case 'issues': { @@ -147,7 +153,7 @@ export default class GithubIterator extends BaseIterator { result = await issuesQuery.getSinglePage(page) // filter out activities without authors (such as bots) - result.data = result.data.filter((i) => (i as any).author.login) + result.data = result.data.filter((i) => (i as any).author?.login) // add each issue as separate endpoint for comments as form repoName|issue-comments|id result.data.map((issue) => @@ -163,7 +169,7 @@ export default class GithubIterator extends BaseIterator { result = await forksQuery.getSinglePage(page) // filter out activities without authors (such as bots) -- may not the case for forks, but filter out anyways - result.data = result.data.filter((i) => (i as any).owner.login) + result.data = result.data.filter((i) => (i as any).owner?.login) break } @@ -174,7 +180,7 @@ export default class GithubIterator extends BaseIterator { ) result = await discussionsQuery.getSinglePage(page) - result.data = result.data.filter((i) => (i as any).author.login) + result.data = result.data.filter((i) => (i as any).author?.login) for (const discussion of result.data) { if ((discussion as any).comments.totalCount > 0) { @@ -195,7 +201,7 @@ export default class GithubIterator extends BaseIterator { ) result = await discussionCommentsQuery.getSinglePage(page) - result.data = result.data.filter((i) => (i as any).author.login) + result.data = result.data.filter((i) => (i as any).author?.login) break } @@ -448,7 +454,7 @@ export default class GithubIterator extends BaseIterator { url: record.url ? record.url : '', repo: this.getRepoByName(repo).url, state: record.state.toLowerCase(), - title: record.title, + title: record.title.replace(/\0/g, ''), }, communityMember: this.parseMember(record.author), score: GitHubGrid.issueOpened.score, @@ -513,7 +519,7 @@ export default class GithubIterator extends BaseIterator { url: record.url ? record.url : '', repo: this.getRepoByName(repo).url, state: record.state.toLowerCase(), - title: record.title, + title: record.title.replace(/\0/g, ''), }, communityMember: this.parseMember(record.author), score: GitHubGrid.pullRequestOpened.score, diff --git a/backend/src/serverless/integrations/iterators/slackIterator.ts b/backend/src/serverless/integrations/iterators/slackIterator.ts index 94ab661d10..5f1d78305b 100644 --- a/backend/src/serverless/integrations/iterators/slackIterator.ts +++ b/backend/src/serverless/integrations/iterators/slackIterator.ts @@ -27,6 +27,7 @@ import { PlatformType } from '../../../utils/platforms' export default class SlackIterator extends BaseIterator { static limitReachedState: State = { + endpoints: [], endpoint: '__limit', page: '__limit', } @@ -72,13 +73,14 @@ export default class SlackIterator extends BaseIterator { members: Object, integrationId: string, userContext: IRepositoryOptions, - state: State = { endpoint: '', page: '' }, + state: State = { endpoint: '', page: '', endpoints: [] }, onboarding: boolean = false, ) { + let endpoints: Endpoints = state.endpoints // Endpoints are the fixed endpoints plus the channels - const endpoints: Endpoints = SlackIterator.fixedEndpoints.concat( - channels.map((channel) => channel.id), - ) + if (state.endpoints.length === 0) { + endpoints = SlackIterator.fixedEndpoints.concat(channels.map((channel) => channel.id)) + } super(tenant, endpoints, state, onboarding, SlackIterator.globalLimit) diff --git a/backend/src/serverless/integrations/iterators/twitterIterator.ts b/backend/src/serverless/integrations/iterators/twitterIterator.ts index 721d04c6b4..a044e14b2b 100644 --- a/backend/src/serverless/integrations/iterators/twitterIterator.ts +++ b/backend/src/serverless/integrations/iterators/twitterIterator.ts @@ -25,6 +25,7 @@ import { PlatformType } from '../../../utils/platforms' export default class TwitterIterator extends BaseIterator { static limitReachedState: State = { + endpoints: [], endpoint: '__limit', page: '__limit', } @@ -57,7 +58,7 @@ export default class TwitterIterator extends BaseIterator { profileId: string, accessToken: string, hashtags: Array, - state: State = { endpoint: '', page: '' }, + state: State = { endpoint: '', page: '', endpoints: [] }, onboarding: boolean = false, tweetCount: number = 0, followers: Set = new Set(), diff --git a/backend/src/serverless/integrations/iterators/twitterReachIterator.ts b/backend/src/serverless/integrations/iterators/twitterReachIterator.ts index 6ddfbdb8d2..97c1c8fa22 100644 --- a/backend/src/serverless/integrations/iterators/twitterReachIterator.ts +++ b/backend/src/serverless/integrations/iterators/twitterReachIterator.ts @@ -23,7 +23,7 @@ import { PlatformType } from '../../../utils/platforms' export default class TwitterReachIterator extends BaseIterator { static readonly TWITTER_API_MAX_USERNAME_LENGTH = 15 - static limitReachedState: State = { endpoint: '__limit', page: '__limit' } + static limitReachedState: State = { endpoint: '__limit', page: '__limit', endpoints: [] } static maxRetrospect: number = Number(process.env.TWITTER_MAX_RETROSPECT_IN_SECONDS || 7380) @@ -48,7 +48,7 @@ export default class TwitterReachIterator extends BaseIterator { profileId: string, accessToken: string, members: Array, - state: State = { endpoint: '', page: '' }, + state: State = { endpoint: '', page: '', endpoints: [] }, ) { super(tenant, members, state, false) diff --git a/backend/src/serverless/integrations/types/regularTypes.ts b/backend/src/serverless/integrations/types/regularTypes.ts index bf3af3d222..eefbcb7414 100644 --- a/backend/src/serverless/integrations/types/regularTypes.ts +++ b/backend/src/serverless/integrations/types/regularTypes.ts @@ -21,6 +21,7 @@ export type Endpoint = string export type Endpoints = Array export type State = { - endpoint: Endpoint + endpoints: Endpoints + endpoint: string page: string } diff --git a/backend/src/serverless/integrations/workers/githubWorker.ts b/backend/src/serverless/integrations/workers/githubWorker.ts index ab88b4e60d..4a6c3d56fb 100644 --- a/backend/src/serverless/integrations/workers/githubWorker.ts +++ b/backend/src/serverless/integrations/workers/githubWorker.ts @@ -3,6 +3,9 @@ import IntegrationRepository from '../../../database/repositories/integrationRep import GithubIterator from '../iterators/githubIterator' import { PlatformType } from '../../../utils/platforms' import { IntegrationsMessage } from '../types/messageTypes' +import StargazersQuery from '../usecases/github/graphql/stargazers' +import BaseIterator from '../iterators/baseIterator' +import { IRepositoryOptions } from '../../../database/repositories/IRepositoryOptions' async function githubWorker(body: IntegrationsMessage) { try { @@ -13,19 +16,60 @@ async function githubWorker(body: IntegrationsMessage) { const integration = await IntegrationRepository.findByPlatform(PlatformType.GITHUB, userContext) - const githubIterator = new GithubIterator( - tenant, - integration.settings.repos, - integration.token, - state, - onboarding, - ) + if (integration.settings.repos.length > 0) { + const githubIterator = new GithubIterator( + tenant, + await getAvailableRepos(integration, userContext), + integration.token, + state, + onboarding, + ) + + return await githubIterator.iterate() + } - return await githubIterator.iterate() + return BaseIterator.success } catch (err) { console.log('Error in github worker, ', err) throw err } } +/** + * Gets the repos available with given token. + * Repositories can become unavailable if it's deleted after + * making a github integration. + * If a repo is not available, it will be removed from integration.settings + * @param integration + * @param userContext + * @returns + */ +async function getAvailableRepos(integration, userContext: IRepositoryOptions) { + const availableRepos = [] + let hasUnavailableRepos = false + for (const repo of integration.settings.repos) { + try { + // we don't need to get default 100 item per page, just 1 is enough to check if repo is available + const stargazersQuery = new StargazersQuery(repo, integration.token, 1) + await stargazersQuery.getSinglePage('') + availableRepos.push(repo) + } catch (e) { + console.log(`Repo ${repo.name} will not be parsed. It is not available with the github token`) + hasUnavailableRepos = true + } + } + + integration.settings.repos = availableRepos + + if (hasUnavailableRepos) { + await IntegrationRepository.update( + integration.id, + { settings: integration.settings }, + userContext, + ) + } + + return availableRepos +} + export default githubWorker diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 56ad2a7bd1..eb43fe3ec1 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -262,6 +262,7 @@ export default class IntegrationService { state: { endpoint: '', page: '', + endpoints: [], }, tenant: integration.tenantId.toString(), sleep: 0, @@ -294,6 +295,7 @@ export default class IntegrationService { state: { endpoint: '', page: '', + endpoints: [], }, tenant: integration.tenantId.toString(), sleep: 0, @@ -336,7 +338,7 @@ export default class IntegrationService { integrationId: integration.id, tenant: integration.tenantId.toString(), onboarding: true, - state: { endpoint: '', page: '' }, + state: { endpoint: '', page: '', endpoints: [] }, args: {}, } @@ -364,6 +366,7 @@ export default class IntegrationService { state: { endpoint: '', page: '', + endpoints: [], }, tenant: integration.tenantId.toString(), sleep: 0, @@ -417,6 +420,7 @@ export default class IntegrationService { state: { endpoint: '', page: '', + endpoints: [], }, tenant: integration.tenantId.toString(), sleep: 0,