Fixed an issue that occurred when fetching twitter followers. (#270)

Author: Uros Marolt

backend/src/serverless/integrations/services/integrations/twitterIntegrationService.ts

@@ -61,6 +61,8 @@ export class TwitterIntegrationService extends IntegrationServiceBase {
     stream: IIntegrationStream,
     context: IStepContext,
   ): Promise<IProcessStreamResults> {
+    const log = this.logger(context)
+
     const { fn, arg } = TwitterIntegrationService.getUsecase(
       stream.value,
       context.pipelineData.profileId,
@@ -85,6 +87,19 @@ export class TwitterIntegrationService extends IntegrationServiceBase {
       : undefined
     const sleep = limit <= 1 ? timeUntilReset : undefined
 
+    if (records === undefined) {
+      log.error(
+        {
+          stream: stream.value,
+          page: stream.metadata.page,
+          profileId: context.pipelineData.profileId,
+        },
+        'No records returned!',
+      )
+
+      throw new Error(`No records returned for stream ${stream.value}!`)
+    }
+
     if (records.length === 0) {
       return {
         operations: [],

backend/src/serverless/integrations/usecases/slack/getMember.ts

@@ -21,14 +21,10 @@ async function getMembers(
 
     const response = await axios(config)
     const member = response.data.user
-    const limit = 100
-    const timeUntilReset = 0
     const nextPage = response.data.response_metadata?.next_cursor || ''
     return {
       records: member,
       nextPage,
-      limit,
-      timeUntilReset,
     }
   } catch (err) {
     if (err && err.response && err.response.status === 429 && err.response.headers['Retry-After']) {

backend/src/serverless/integrations/usecases/twitter/getFollowers.ts

@@ -1,4 +1,4 @@
-import axios from 'axios'
+import axios, { AxiosRequestConfig } from 'axios'
 import moment from 'moment'
 import { TwitterGetFollowersInput, TwitterGetFollowersOutput } from '../../types/twitterTypes'
 import { Logger } from '../../../../utils/logging'
@@ -13,17 +13,30 @@ const getFollowers = async (
   logger: Logger,
 ): Promise<TwitterGetFollowersOutput> => {
   try {
-    const config = {
+    const config: AxiosRequestConfig<any> = {
       method: 'get',
-      url: `https://api.twitter.com/2/users/${input.profileId}/followers?user.fields=name,description,location,public_metrics,url,verified,profile_image_url&pagination_token=D903MLRBG6U1EZZZ`,
+      url: `https://api.twitter.com/2/users/${input.profileId}/followers`,
+      params: {
+        'user.fields': 'name,description,location,public_metrics,url,verified,profile_image_url',
+      },
       headers: {
         Authorization: `Bearer ${input.token}`,
       },
     }
+
+    if (input.perPage) {
+      config.params.max_results = input.perPage
+    }
+
+    if (input.page) {
+      config.params.pagination_token = input.page
+    }
+
     const response = await axios(config)
     const limit = parseInt(response.headers['x-rate-limit-remaining'], 10)
     const resetTs = parseInt(response.headers['x-rate-limit-reset'], 10) * 1000
     const timeUntilReset = moment(resetTs).diff(moment(), 'seconds')
+
     return {
       records: response.data.data,
       nextPage: response.data.meta.next_token || '',