GitHub integration: check repo can be queried before starting parsing

backend/src/serverless/integrations/coordinators/devtoCoordinator.ts

@@ -19,7 +19,7 @@ async function devtoCoordinator(): Promise<void> {
       integrationId: integration.id,
       tenant: integration.tenantId.toString(),
       onboarding: false,
-      state: { endpoint: '', page: '' },
+      state: { endpoint: '', page: '', endpoints: [] },
       args: {},
     }
 

backend/src/serverless/integrations/coordinators/discordCoordinator.ts

@@ -16,7 +16,7 @@ async function discordCoordinator(): Promise<void> {
       sleep: Math.floor(Math.random() * 1200),
       tenant: integration.tenantId.toString(),
       onboarding: false,
-      state: { endpoint: '', page: '' },
+      state: { endpoint: '', page: '', endpoints: [] },
       args: {
         guildId: integration.integrationIdentifier,
         channels: integration.settings.channels || [],

backend/src/serverless/integrations/coordinators/slackCoordinator.ts

@@ -16,7 +16,7 @@ async function slackCoordinator(): Promise<void> {
       sleep: 0,
       tenant: integration.tenantId.toString(),
       onboarding: false,
-      state: { endpoint: '', page: '' },
+      state: { endpoint: '', page: '', endpoints: [] },
       args: {
         channels: integration.settings.channels || [],
       },

backend/src/serverless/integrations/coordinators/twitterCoordinator.ts

@@ -16,7 +16,7 @@ async function twitterCoordinator(): Promise<void> {
       sleep: 0,
       tenant: integration.tenantId.toString(),
       onboarding: false,
-      state: { endpoint: '', page: '' },
+      state: { endpoint: '', page: '', endpoints: [] },
       args: {
         profileId: integration.integrationIdentifier,
         hashtags: integration.settings.hashtags,

backend/src/serverless/integrations/coordinators/twitterReachCoordinator.ts

@@ -15,7 +15,7 @@ async function twitterReachCoordinator(): Promise<void> {
       sleep: 0,
       tenant: microservice.tenantId.toString(),
       onboarding: false,
-      state: { endpoint: '', page: '' },
+      state: { endpoint: '', page: '', endpoints: [] },
       args: {
         profileId: microservice.microserviceIdentifier,
       },

backend/src/serverless/integrations/iterators/__tests__/baseIteratorStatic.test.ts

@@ -5,35 +5,35 @@ describe('Integrations worker static tests', () => {
   const endpoints = ['endpoint1', 'endpoint2', 'endpoint3']
   describe('Get iterator tests', () => {
     it('Should return the endpoints to iterate for a standard state', async () => {
-      const state = { endpoint: 'endpoint2', page: 'here' }
+      const state = { endpoint: 'endpoint2', page: 'here', endpoints: [] }
       const iterator = BaseIterator.getEndPointsIterator(endpoints, state)
 
       expect(iterator).toStrictEqual(['endpoint2', 'endpoint3'])
     })
 
     it('Should return the endpoints to iterate for a state with no page', async () => {
-      const state = { endpoint: 'endpoint2', page: '' }
+      const state = { endpoint: 'endpoint2', page: '', endpoints: [] }
       const iterator = BaseIterator.getEndPointsIterator(endpoints, state)
 
       expect(iterator).toStrictEqual(['endpoint2', 'endpoint3'])
     })
 
     it('Should return the whole endpoints when the state is in the first endpoint with a page', async () => {
-      const state = { endpoint: 'endpoint1', page: 'here' }
+      const state = { endpoint: 'endpoint1', page: 'here', endpoints: [] }
       const iterator = BaseIterator.getEndPointsIterator(endpoints, state)
 
       expect(iterator).toStrictEqual(endpoints)
     })
 
     it('Should return the last endpoint when the state is in the last endpoint', async () => {
-      const state = { endpoint: 'endpoint3', page: '' }
+      const state = { endpoint: 'endpoint3', page: '', endpoints: [] }
       const iterator = BaseIterator.getEndPointsIterator(endpoints, state)
 
       expect(iterator).toStrictEqual(['endpoint3'])
     })
 
     it('Should throw an error when state.endpoint is not in endpoints', async () => {
-      const state = { endpoint: 'endpoint4', page: '' }
+      const state = { endpoint: 'endpoint4', page: '', endpoints: [] }
       try {
         BaseIterator.getEndPointsIterator(endpoints, state)
       } catch (error: any) {
@@ -48,17 +48,20 @@ describe('Integrations worker static tests', () => {
       const state = BaseIterator.initState(endpoints, {
         endpoint: '',
         page: '',
+        endpoints: [],
       })
       expect(state).toStrictEqual({
         endpoint: 'endpoint1',
         page: '',
+        endpoints: ['endpoint1', 'endpoint2', 'endpoint3'],
       })
     })
 
     it('Should return the given state', async () => {
       const givenState = {
         endpoint: 'endpoint2',
         page: 'here',
+        endpoints: [],
       }
       const state = BaseIterator.initState(endpoints, givenState)
       expect(state).toStrictEqual(givenState)

backend/src/serverless/integrations/iterators/__tests__/devtoIterator.test.ts

@@ -76,6 +76,7 @@ async function getDevtoIterator(articles: DevtoArticle[]) {
     {
       endpoint: '',
       page: '',
+      endpoints: [],
     },
     false,
   )

backend/src/serverless/integrations/iterators/__tests__/discordIterator.test.ts

@@ -589,7 +589,7 @@ describe('Discord iterator tests', () => {
           'guild12345',
           'token12345',
           channels,
-          { endpoint: '', page: '' },
+          { endpoint: '', page: '', endpoints: [] },
           true,
         )
         const isOver = iter.integrationSpecificIsEndpointFinished('1', recentRecord)
@@ -605,7 +605,7 @@ describe('Discord iterator tests', () => {
           'guild12345',
           'token12345',
           channels,
-          { endpoint: '', page: '' },
+          { endpoint: '', page: '', endpoints: [] },
           true,
         )
         const isOver = iter.integrationSpecificIsEndpointFinished('4', recentRecord)
@@ -621,7 +621,7 @@ describe('Discord iterator tests', () => {
           'guild12345',
           'token12345',
           channels,
-          { endpoint: '', page: '' },
+          { endpoint: '', page: '', endpoints: [] },
           true,
         )
         const isOver = iter.integrationSpecificIsEndpointFinished('3', recentRecord)
@@ -637,7 +637,7 @@ describe('Discord iterator tests', () => {
           'guild12345',
           'token12345',
           channels,
-          { endpoint: '', page: '' },
+          { endpoint: '', page: '', endpoints: [] },
           true,
         )
         const isOver = iter.integrationSpecificIsEndpointFinished('1', oldRecord)
@@ -653,7 +653,7 @@ describe('Discord iterator tests', () => {
           'guild12345',
           'token12345',
           channels,
-          { endpoint: '', page: '' },
+          { endpoint: '', page: '', endpoints: [] },
           true,
         )
         const isOver = iter.integrationSpecificIsEndpointFinished('3', oldRecord)
@@ -669,7 +669,7 @@ describe('Discord iterator tests', () => {
           'guild12345',
           'token12345',
           channels,
-          { endpoint: '', page: '' },
+          { endpoint: '', page: '', endpoints: [] },
           true,
         )
         const isOver = iter.integrationSpecificIsEndpointFinished('4', oldRecord)
@@ -685,7 +685,7 @@ describe('Discord iterator tests', () => {
           'guild12345',
           'token12345',
           channels,
-          { endpoint: '', page: '' },
+          { endpoint: '', page: '', endpoints: [] },
           true,
         )
         const isOverGeneral = iter.isEndpointFinished('4', {}, [])

backend/src/serverless/integrations/iterators/__tests__/githubIterator.test.ts

@@ -38,6 +38,7 @@ async function getGithubIterator(repos: Repos, options: IRepositoryOptions) {
     {
       endpoint: '',
       page: '',
+      endpoints: [],
     },
     true,
   )

backend/src/serverless/integrations/iterators/__tests__/slackIterator.test.ts

@@ -30,7 +30,7 @@ async function getSlackIterator(members = {}, channels = [{ name: 'dev', id: 'C0
     members,
     integrationId,
     mockIRepositoryOptions,
-    { endpoint: '', page: '' },
+    { endpoint: '', page: '', endpoints: [] },
     false,
   )
 }

backend/src/serverless/integrations/iterators/__tests__/testIterator.ts

@@ -1,11 +1,9 @@
 /* eslint @typescript-eslint/no-unused-vars: 0 */
 /* eslint class-methods-use-this: 0 */
 
-import { endpoint } from 'aws-sdk/clients/sns'
 import moment from 'moment'
-import { Endpoint } from 'aws-sdk'
 import { BaseOutput, parseOutput, IntegrationResponse } from '../../types/iteratorTypes'
-import { State } from '../../types/regularTypes'
+import { Endpoint, State } from '../../types/regularTypes'
 
 import BaseIterator from '../baseIterator'
 import { AddActivitiesSingle } from '../../types/messageTypes'
@@ -19,6 +17,7 @@ export default class TestIterator extends BaseIterator {
 
   static limitReachedState: State = {
     endpoint: '__limit',
+    endpoints: [],
     page: '__limit',
   }
 
@@ -33,7 +32,7 @@ export default class TestIterator extends BaseIterator {
    */
   constructor(
     transitionFn: Function,
-    state: State = { endpoint: '', page: '' },
+    state: State = { endpoint: '', page: '', endpoints: [] },
     onboarding: boolean = false,
     limitCount: number = 0,
   ) {
@@ -138,4 +137,12 @@ export default class TestIterator extends BaseIterator {
   async iterate(maxTime: number = 12 * 60): Promise<BaseOutput> {
     return super.iterate(maxTime, false)
   }
+
+  next(currentEndpoint: Endpoint, nextPage: string | undefined, parseOutput: parseOutput): State {
+    const next = super.next(currentEndpoint, nextPage, parseOutput)
+    if (this.audits.length > 0) {
+      this.audits[this.audits.length - 1].endpoints = next.endpoints
+    }
+    return next
+  }
 }