From 7c8ecec811d35db3870cca23b66ef8fd1404e327 Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Fri, 26 Aug 2022 18:13:23 +0200 Subject: [PATCH 1/7] Fix EagleEye keyword filter (#9) --- .../eagleEyeContentRepository.test.ts | 55 ++++++++++++++++++- .../repositories/eagleEyeContentRepository.ts | 3 +- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts b/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts index 3933dce84b..2a2e3d2bfe 100644 --- a/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts +++ b/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts @@ -477,6 +477,59 @@ describe('eagleEyeContentRepository tests', () => { await addAll(mockIRepositoryOptions) + const k1 = { + sourceId: 'sourceIdk1', + vectorId: 'sourceIdk1', + status: null, + platform: 'hacker_news', + title: 'title', + userAttributes: { github: 'hey', twitter: 'ho' }, + text: 'text', + postAttributes: { + score: 10, + }, + url: 'url', + timestamp: new Date(), + username: 'username', + keywords: ['keyword1'], + similarityScore: 0.9, + } + + await new EagleEyeContentService(mockIRepositoryOptions).upsert(k1) + + const k2 = { + sourceId: 'sourceIdk2', + vectorId: 'sourceIdk2', + status: null, + platform: 'hacker_news', + title: 'title', + userAttributes: { github: 'hey', twitter: 'ho' }, + text: 'text', + postAttributes: { + score: 10, + }, + url: 'url', + timestamp: new Date(), + username: 'username', + keywords: ['keyword2'], + similarityScore: 0.9, + } + + try { + await EagleEyeContentRepository.findAndCountAll( + { + filter: { + keywords: 'keyword1,keyword2', + }, + }, + mockIRepositoryOptions, + ) + } catch (e) { + console.log(e) + } + + await new EagleEyeContentService(mockIRepositoryOptions).upsert(k2) + expect( ( await EagleEyeContentRepository.findAndCountAll( @@ -488,7 +541,7 @@ describe('eagleEyeContentRepository tests', () => { mockIRepositoryOptions, ) ).count, - ).toBe(2) + ).toBe(5) }) }) diff --git a/backend/src/database/repositories/eagleEyeContentRepository.ts b/backend/src/database/repositories/eagleEyeContentRepository.ts index 28e18aa058..c056ef1a1b 100644 --- a/backend/src/database/repositories/eagleEyeContentRepository.ts +++ b/backend/src/database/repositories/eagleEyeContentRepository.ts @@ -211,9 +211,10 @@ export default class EagleEyeContentRepository { } if (filter.keywords) { + // Overlap will take a post where any keyword matches any of the filter keywords whereAnd.push({ keywords: { - [Op.contains]: filter.keywords.split(','), + [Op.overlap]: filter.keywords.split(','), }, }) } From f8ba553b0b6f8b11e615619110ccafbac8757ec0 Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 28 Aug 2022 20:32:32 +0200 Subject: [PATCH 2/7] Weekly analytics emails for all workspace users (#10) --- .../__tests__/userRepository.test.ts | 120 ++++++++++++++++++ .../database/repositories/userRepository.ts | 25 ++++ .../workers/weeklyAnalyticsEmailsWorker.ts | 68 +++++----- .../nodejs/nodeMicroserviceSQS.ts | 7 +- .../microservices/nodejs/serverless.yml | 4 +- start.sh | 2 +- 6 files changed, 188 insertions(+), 38 deletions(-) create mode 100644 backend/src/database/repositories/__tests__/userRepository.test.ts diff --git a/backend/src/database/repositories/__tests__/userRepository.test.ts b/backend/src/database/repositories/__tests__/userRepository.test.ts new file mode 100644 index 0000000000..a18315d05a --- /dev/null +++ b/backend/src/database/repositories/__tests__/userRepository.test.ts @@ -0,0 +1,120 @@ +import UserRepository from '../userRepository' +import SequelizeTestUtils from '../../utils/sequelizeTestUtils' +import Error404 from '../../../errors/Error404' +import Roles from '../../../security/roles' + +const db = null + +describe('UserRepository tests', () => { + beforeEach(async () => { + await SequelizeTestUtils.wipeDatabase(db) + }) + + afterAll((done) => { + // Closing the DB connection allows Jest to exit successfully. + SequelizeTestUtils.closeConnection(db) + done() + }) + + describe('findAllUsersOfTenant method', () => { + it('Should find all related users of a tenant successfully', async () => { + // Getting options already creates one random user + const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) + + let allUsersOfTenant = ( + await UserRepository.findAllUsersOfTenant(mockIRepositoryOptions.currentTenant.id) + ).map((u) => SequelizeTestUtils.objectWithoutKey(u, 'tenants')) + + expect(allUsersOfTenant).toStrictEqual([ + mockIRepositoryOptions.currentUser.get({ plain: true }), + ]) + + // add more users to the test tenant + const randomUser2 = await SequelizeTestUtils.getRandomUser() + const user2 = await mockIRepositoryOptions.database.user.create(randomUser2) + + await mockIRepositoryOptions.database.tenantUser.create({ + roles: [Roles.values.admin], + status: 'active', + tenantId: mockIRepositoryOptions.currentTenant.id, + userId: user2.id, + }) + + allUsersOfTenant = ( + await UserRepository.findAllUsersOfTenant(mockIRepositoryOptions.currentTenant.id) + ).map((u) => SequelizeTestUtils.objectWithoutKey(u, 'tenants')) + + expect(allUsersOfTenant).toStrictEqual([ + mockIRepositoryOptions.currentUser.get({ plain: true }), + user2.get({ plain: true }), + ]) + + const randomUser3 = await SequelizeTestUtils.getRandomUser() + const user3 = await mockIRepositoryOptions.database.user.create(randomUser3) + + await mockIRepositoryOptions.database.tenantUser.create({ + roles: [Roles.values.admin], + status: 'active', + tenantId: mockIRepositoryOptions.currentTenant.id, + userId: user3.id, + }) + + allUsersOfTenant = ( + await UserRepository.findAllUsersOfTenant(mockIRepositoryOptions.currentTenant.id) + ).map((u) => SequelizeTestUtils.objectWithoutKey(u, 'tenants')) + + expect(allUsersOfTenant).toStrictEqual([ + mockIRepositoryOptions.currentUser.get({ plain: true }), + user2.get({ plain: true }), + user3.get({ plain: true }), + ]) + + // add other users and tenants that are non related to previous couples + await SequelizeTestUtils.getTestIRepositoryOptions(db) + + // users of the previous tenant should be the same + allUsersOfTenant = ( + await UserRepository.findAllUsersOfTenant(mockIRepositoryOptions.currentTenant.id) + ).map((u) => SequelizeTestUtils.objectWithoutKey(u, 'tenants')) + + expect(allUsersOfTenant).toStrictEqual([ + mockIRepositoryOptions.currentUser.get({ plain: true }), + user2.get({ plain: true }), + user3.get({ plain: true }), + ]) + + const tenantUsers = await mockIRepositoryOptions.database.tenantUser.findAll({ + tenantId: mockIRepositoryOptions.currentTenant.id, + }) + + // remove last user added to the tenant + await tenantUsers[2].destroy({ force: true }) + + allUsersOfTenant = ( + await UserRepository.findAllUsersOfTenant(mockIRepositoryOptions.currentTenant.id) + ).map((u) => SequelizeTestUtils.objectWithoutKey(u, 'tenants')) + + expect(allUsersOfTenant).toStrictEqual([ + mockIRepositoryOptions.currentUser.get({ plain: true }), + user2.get({ plain: true }), + ]) + + // remove first user added to the tenant + await tenantUsers[0].destroy({ force: true }) + + allUsersOfTenant = ( + await UserRepository.findAllUsersOfTenant(mockIRepositoryOptions.currentTenant.id) + ).map((u) => SequelizeTestUtils.objectWithoutKey(u, 'tenants')) + + expect(allUsersOfTenant).toStrictEqual([user2.get({ plain: true })]) + + // remove the last remaining user from the tenant + await tenantUsers[1].destroy({ force: true }) + + // function now should be throwing Error404 + await expect(() => + UserRepository.findAllUsersOfTenant(mockIRepositoryOptions.currentTenant.id), + ).rejects.toThrowError(new Error404()) + }) + }) +}) diff --git a/backend/src/database/repositories/userRepository.ts b/backend/src/database/repositories/userRepository.ts index 16c6b67d24..7079439f68 100644 --- a/backend/src/database/repositories/userRepository.ts +++ b/backend/src/database/repositories/userRepository.ts @@ -31,6 +31,31 @@ export default class UserRepository { return this._populateRelations(record, options) } + /** + * Finds all users of a tenant. + * @param tenantId + * @returns + */ + static async findAllUsersOfTenant(tenantId: string): Promise { + const options = await SequelizeRepository.getDefaultIRepositoryOptions() + + const records = await options.database.user.findAll({ + include: [ + { + model: options.database.tenantUser, + as: 'tenants', + where: { tenantId }, + }, + ], + }) + + if (records.length === 0) { + throw new Error404() + } + + return this._populateRelationsForRows(records, options) + } + static async create(data, options: IRepositoryOptions) { const currentUser = SequelizeRepository.getCurrentUser(options) diff --git a/backend/src/serverless/microservices/nodejs/analytics/workers/weeklyAnalyticsEmailsWorker.ts b/backend/src/serverless/microservices/nodejs/analytics/workers/weeklyAnalyticsEmailsWorker.ts index c975af2956..968cf854bc 100644 --- a/backend/src/serverless/microservices/nodejs/analytics/workers/weeklyAnalyticsEmailsWorker.ts +++ b/backend/src/serverless/microservices/nodejs/analytics/workers/weeklyAnalyticsEmailsWorker.ts @@ -9,9 +9,11 @@ import { AnalyticsEmailsOutput } from '../../messageTypes' import { platformDisplayNames } from '../../../../../utils/platformDisplayNames' import getStage from '../../../../../services/helpers/getStage' import { s3 } from '../../../../../services/aws' +import UserRepository from '../../../../../database/repositories/userRepository' /** - * Sends weekly analytics of a given tenant to the user email. + * Sends weekly analytics emails of a given tenant + * to all users of the tenant. * Data sent is for the last week. * @param tenantId */ @@ -78,44 +80,48 @@ async function weeklyAnalyticsEmailsWorker(tenantId: string): Promise 0, - hotConversationsCount: hotConversations.length, - }, - tenant: { - name: userContext.currentTenant.name, - }, - user: { - name: userContext.currentUser.firstName, - }, - } + const allTenantUsers = await UserRepository.findAllUsersOfTenant(tenantId) const advancedSuppressionManager = { groupId: parseInt(getConfig().SENDGRID_WEEKLY_ANALYTICS_UNSUBSCRIBE_GROUP_ID, 10), groupsToDisplay: [parseInt(getConfig().SENDGRID_WEEKLY_ANALYTICS_UNSUBSCRIBE_GROUP_ID, 10)], } - console.log(`SENDING EMAIL!!!! to: ${userContext.currentUser.email}`) - console.log(`sendgrid template object: `) - console.log(data) + for (const user of allTenantUsers) { + if (user.email && user.emailVerified) { + const userFirstName = user.firstName ? user.firstName : user.email.split('@')[0] + + const data = { + analytics: { + dateRangeStart: dateTimeStart.format('D MMMM, YYYY'), + dateRangeEnd: dateTimeEnd.format('D MMMM, YYYY'), + activeMembers, + newMembers, + activitiesTracked: newActivities, + conversationsStarted: newConversations, + hotConversations, + hasHotConversations: hotConversations.length > 0, + hotConversationsCount: hotConversations.length, + }, + tenant: { + name: userContext.currentTenant.name, + }, + user: { + name: userFirstName, + }, + } - new EmailSender(EmailSender.TEMPLATES.WEEKLY_ANALYTICS, data).sendTo( - userContext.currentUser.email, - advancedSuppressionManager, - ) + await new EmailSender(EmailSender.TEMPLATES.WEEKLY_ANALYTICS, data).sendTo( + user.email, + advancedSuppressionManager, + ) - new EmailSender(EmailSender.TEMPLATES.WEEKLY_ANALYTICS, data).sendTo( - 'team@crowd.dev', - advancedSuppressionManager, - ) + await new EmailSender(EmailSender.TEMPLATES.WEEKLY_ANALYTICS, data).sendTo( + 'team@crowd.dev', + advancedSuppressionManager, + ) + } + } return { status: 200, emailSent: true } } diff --git a/backend/src/serverless/microservices/nodejs/nodeMicroserviceSQS.ts b/backend/src/serverless/microservices/nodejs/nodeMicroserviceSQS.ts index f11a039fa9..7680a65bb2 100644 --- a/backend/src/serverless/microservices/nodejs/nodeMicroserviceSQS.ts +++ b/backend/src/serverless/microservices/nodejs/nodeMicroserviceSQS.ts @@ -1,10 +1,7 @@ import moment from 'moment' import { NodeMicroserviceMessage } from './messageTypes' import { getConfig } from '../../../config' - -const { SQS } = require('aws-sdk') - -const sqs = new SQS() +import { sqs } from '../../../services/aws' /** * Send a message to the node microservice queue @@ -23,7 +20,7 @@ async function sendNodeMicroserviceMessage(body: NodeMicroserviceMessage): Promi await sqs .sendMessage({ - QueueUrl: getConfig().QUEUE_URL, + QueueUrl: getConfig().NODE_MICROSERVICES_SQS_URL, MessageGroupId: messageGroupId, MessageDeduplicationId: messageDeduplicationId, MessageBody: JSON.stringify(body), diff --git a/backend/src/serverless/microservices/nodejs/serverless.yml b/backend/src/serverless/microservices/nodejs/serverless.yml index cef6f312aa..f6b7fdb920 100644 --- a/backend/src/serverless/microservices/nodejs/serverless.yml +++ b/backend/src/serverless/microservices/nodejs/serverless.yml @@ -72,6 +72,8 @@ constructs: environment: NODE_ENV: ${env:NODE_ENV} NODE_MICROSERVICES_STATEMACHINE_ARN: ${env:NODE_MICROSERVICES_STATEMACHINE_ARN} + LOCALSTACK_HOSTNAME: ${env:LOCALSTACK_HOSTNAME} + LOCALSTACK_PORT: ${env:LOCALSTACK_PORT} functions: weeklyAnalyticsEmailsCoordinator: @@ -83,7 +85,7 @@ functions: NODE_ENV: ${env:NODE_ENV} EDITION: ${env:EDITION} SEGMENT_WRITE_KEY: ${env:SEGMENT_WRITE_KEY} - QUEUE_URL: ${env:NODE_MICROSERVICES_SQS_URL} + NODE_MICROSERVICES_SQS_URL: ${env:NODE_MICROSERVICES_SQS_URL} DATABASE_USERNAME: ${env:DATABASE_USERNAME} DATABASE_DIALECT: ${env:DATABASE_DIALECT} DATABASE_PASSWORD: ${env:DATABASE_PASSWORD} diff --git a/start.sh b/start.sh index 8ae75099d3..8928b421d2 100755 --- a/start.sh +++ b/start.sh @@ -45,6 +45,6 @@ echo $IS_DEV if [[ "$IS_DEV" = false ]]; then cd $CLI_HOME/docker && docker-compose -p crowd up --force-recreate else - bash backend/util/install-all.sh + cd $CLI_HOME && bash $CLI_HOME/backend/util/install-all.sh cd $CLI_HOME/docker && docker-compose -f docker-compose.yaml -f docker-compose.dev.yaml -p crowd up --force-recreate fi From 9e3379c07d6d0dd32d8b813ad4565f1777f757b6 Mon Sep 17 00:00:00 2001 From: anil Date: Wed, 7 Sep 2022 12:00:34 +0300 Subject: [PATCH 3/7] github worker now checks if repo is available before iterating --- .../integrations/workers/githubWorker.ts | 60 ++++++++++++++++--- 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/backend/src/serverless/integrations/workers/githubWorker.ts b/backend/src/serverless/integrations/workers/githubWorker.ts index ab88b4e60d..4a6c3d56fb 100644 --- a/backend/src/serverless/integrations/workers/githubWorker.ts +++ b/backend/src/serverless/integrations/workers/githubWorker.ts @@ -3,6 +3,9 @@ import IntegrationRepository from '../../../database/repositories/integrationRep import GithubIterator from '../iterators/githubIterator' import { PlatformType } from '../../../utils/platforms' import { IntegrationsMessage } from '../types/messageTypes' +import StargazersQuery from '../usecases/github/graphql/stargazers' +import BaseIterator from '../iterators/baseIterator' +import { IRepositoryOptions } from '../../../database/repositories/IRepositoryOptions' async function githubWorker(body: IntegrationsMessage) { try { @@ -13,19 +16,60 @@ async function githubWorker(body: IntegrationsMessage) { const integration = await IntegrationRepository.findByPlatform(PlatformType.GITHUB, userContext) - const githubIterator = new GithubIterator( - tenant, - integration.settings.repos, - integration.token, - state, - onboarding, - ) + if (integration.settings.repos.length > 0) { + const githubIterator = new GithubIterator( + tenant, + await getAvailableRepos(integration, userContext), + integration.token, + state, + onboarding, + ) + + return await githubIterator.iterate() + } - return await githubIterator.iterate() + return BaseIterator.success } catch (err) { console.log('Error in github worker, ', err) throw err } } +/** + * Gets the repos available with given token. + * Repositories can become unavailable if it's deleted after + * making a github integration. + * If a repo is not available, it will be removed from integration.settings + * @param integration + * @param userContext + * @returns + */ +async function getAvailableRepos(integration, userContext: IRepositoryOptions) { + const availableRepos = [] + let hasUnavailableRepos = false + for (const repo of integration.settings.repos) { + try { + // we don't need to get default 100 item per page, just 1 is enough to check if repo is available + const stargazersQuery = new StargazersQuery(repo, integration.token, 1) + await stargazersQuery.getSinglePage('') + availableRepos.push(repo) + } catch (e) { + console.log(`Repo ${repo.name} will not be parsed. It is not available with the github token`) + hasUnavailableRepos = true + } + } + + integration.settings.repos = availableRepos + + if (hasUnavailableRepos) { + await IntegrationRepository.update( + integration.id, + { settings: integration.settings }, + userContext, + ) + } + + return availableRepos +} + export default githubWorker From 3e2f26c1a6c677b9d126b8a4f01747e7cf5479a4 Mon Sep 17 00:00:00 2001 From: anil Date: Wed, 7 Sep 2022 12:00:34 +0300 Subject: [PATCH 4/7] github worker now checks if repo is available before iterating --- .../integrations/workers/githubWorker.ts | 60 ++++++++++++++++--- 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/backend/src/serverless/integrations/workers/githubWorker.ts b/backend/src/serverless/integrations/workers/githubWorker.ts index ab88b4e60d..4a6c3d56fb 100644 --- a/backend/src/serverless/integrations/workers/githubWorker.ts +++ b/backend/src/serverless/integrations/workers/githubWorker.ts @@ -3,6 +3,9 @@ import IntegrationRepository from '../../../database/repositories/integrationRep import GithubIterator from '../iterators/githubIterator' import { PlatformType } from '../../../utils/platforms' import { IntegrationsMessage } from '../types/messageTypes' +import StargazersQuery from '../usecases/github/graphql/stargazers' +import BaseIterator from '../iterators/baseIterator' +import { IRepositoryOptions } from '../../../database/repositories/IRepositoryOptions' async function githubWorker(body: IntegrationsMessage) { try { @@ -13,19 +16,60 @@ async function githubWorker(body: IntegrationsMessage) { const integration = await IntegrationRepository.findByPlatform(PlatformType.GITHUB, userContext) - const githubIterator = new GithubIterator( - tenant, - integration.settings.repos, - integration.token, - state, - onboarding, - ) + if (integration.settings.repos.length > 0) { + const githubIterator = new GithubIterator( + tenant, + await getAvailableRepos(integration, userContext), + integration.token, + state, + onboarding, + ) + + return await githubIterator.iterate() + } - return await githubIterator.iterate() + return BaseIterator.success } catch (err) { console.log('Error in github worker, ', err) throw err } } +/** + * Gets the repos available with given token. + * Repositories can become unavailable if it's deleted after + * making a github integration. + * If a repo is not available, it will be removed from integration.settings + * @param integration + * @param userContext + * @returns + */ +async function getAvailableRepos(integration, userContext: IRepositoryOptions) { + const availableRepos = [] + let hasUnavailableRepos = false + for (const repo of integration.settings.repos) { + try { + // we don't need to get default 100 item per page, just 1 is enough to check if repo is available + const stargazersQuery = new StargazersQuery(repo, integration.token, 1) + await stargazersQuery.getSinglePage('') + availableRepos.push(repo) + } catch (e) { + console.log(`Repo ${repo.name} will not be parsed. It is not available with the github token`) + hasUnavailableRepos = true + } + } + + integration.settings.repos = availableRepos + + if (hasUnavailableRepos) { + await IntegrationRepository.update( + integration.id, + { settings: integration.settings }, + userContext, + ) + } + + return availableRepos +} + export default githubWorker From c2c16dd580214b65d014c25aa3435a79d1a1643d Mon Sep 17 00:00:00 2001 From: anil Date: Wed, 7 Sep 2022 12:31:44 +0300 Subject: [PATCH 5/7] existence check when getting member for all gh endpoints --- .../integrations/iterators/githubIterator.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/src/serverless/integrations/iterators/githubIterator.ts b/backend/src/serverless/integrations/iterators/githubIterator.ts index 4dff257100..7a741f3c6c 100644 --- a/backend/src/serverless/integrations/iterators/githubIterator.ts +++ b/backend/src/serverless/integrations/iterators/githubIterator.ts @@ -95,7 +95,7 @@ export default class GithubIterator extends BaseIterator { const stargazersQuery = new StargazersQuery(this.getRepoByName(repoName), this.accessToken) result = await stargazersQuery.getSinglePage(page) - result.data = result.data.filter((i) => (i as any).node.login) + result.data = result.data.filter((i) => (i as any).node?.login) break } case 'pulls': { @@ -106,7 +106,7 @@ export default class GithubIterator extends BaseIterator { result = await pullRequestsQuery.getSinglePage(page) // filter out activities without authors (such as bots) - result.data = result.data.filter((i) => (i as any).author.login) + result.data = result.data.filter((i) => (i as any).author?.login) // add each PR as separate endpoint for comments as form repoName|pull-comments|id result.data.map((pr) => @@ -127,7 +127,7 @@ export default class GithubIterator extends BaseIterator { result = await pullRequestCommentsQuery.getSinglePage(page) - result.data = result.data.filter((i) => (i as any).author.login) + result.data = result.data.filter((i) => (i as any).author?.login) break } case 'issue-comments': { @@ -139,7 +139,7 @@ export default class GithubIterator extends BaseIterator { ) result = await issueCommentsQuery.getSinglePage(page) - result.data = result.data.filter((i) => (i as any).author.login) + result.data = result.data.filter((i) => (i as any).author?.login) break } case 'issues': { @@ -147,7 +147,7 @@ export default class GithubIterator extends BaseIterator { result = await issuesQuery.getSinglePage(page) // filter out activities without authors (such as bots) - result.data = result.data.filter((i) => (i as any).author.login) + result.data = result.data.filter((i) => (i as any).author?.login) // add each issue as separate endpoint for comments as form repoName|issue-comments|id result.data.map((issue) => @@ -163,7 +163,7 @@ export default class GithubIterator extends BaseIterator { result = await forksQuery.getSinglePage(page) // filter out activities without authors (such as bots) -- may not the case for forks, but filter out anyways - result.data = result.data.filter((i) => (i as any).owner.login) + result.data = result.data.filter((i) => (i as any).owner?.login) break } @@ -174,7 +174,7 @@ export default class GithubIterator extends BaseIterator { ) result = await discussionsQuery.getSinglePage(page) - result.data = result.data.filter((i) => (i as any).author.login) + result.data = result.data.filter((i) => (i as any).author?.login) for (const discussion of result.data) { if ((discussion as any).comments.totalCount > 0) { @@ -195,7 +195,7 @@ export default class GithubIterator extends BaseIterator { ) result = await discussionCommentsQuery.getSinglePage(page) - result.data = result.data.filter((i) => (i as any).author.login) + result.data = result.data.filter((i) => (i as any).author?.login) break } From 06d3059193fce02737e76780696aa547d846d3f0 Mon Sep 17 00:00:00 2001 From: anilb Date: Mon, 12 Sep 2022 16:12:50 +0300 Subject: [PATCH 6/7] Base iterator looses dynamic endpoints information on consecutive runs (#50) --- .../coordinators/devtoCoordinator.ts | 2 +- .../coordinators/discordCoordinator.ts | 2 +- .../coordinators/slackCoordinator.ts | 2 +- .../coordinators/twitterCoordinator.ts | 2 +- .../coordinators/twitterReachCoordinator.ts | 2 +- .../iterators/__tests__/baseIteration.test.ts | 145 ++++++++++-------- .../__tests__/baseIteratorStatic.test.ts | 13 +- .../iterators/__tests__/devtoIterator.test.ts | 1 + .../__tests__/discordIterator.test.ts | 14 +- .../__tests__/githubIterator.test.ts | 1 + .../iterators/__tests__/slackIterator.test.ts | 2 +- .../iterators/__tests__/testIterator.ts | 15 +- .../__tests__/twitterIterator.test.ts | 38 ++--- .../integrations/iterators/baseIterator.ts | 20 ++- .../integrations/iterators/devtoIterator.ts | 2 +- .../integrations/iterators/discordIterator.ts | 3 +- .../integrations/iterators/githubIterator.ts | 22 ++- .../integrations/iterators/slackIterator.ts | 10 +- .../integrations/iterators/twitterIterator.ts | 3 +- .../iterators/twitterReachIterator.ts | 4 +- .../integrations/types/regularTypes.ts | 3 +- backend/src/services/integrationService.ts | 6 +- 22 files changed, 187 insertions(+), 125 deletions(-) diff --git a/backend/src/serverless/integrations/coordinators/devtoCoordinator.ts b/backend/src/serverless/integrations/coordinators/devtoCoordinator.ts index 6d3f74f76a..9a727365b0 100644 --- a/backend/src/serverless/integrations/coordinators/devtoCoordinator.ts +++ b/backend/src/serverless/integrations/coordinators/devtoCoordinator.ts @@ -19,7 +19,7 @@ async function devtoCoordinator(): Promise { integrationId: integration.id, tenant: integration.tenantId.toString(), onboarding: false, - state: { endpoint: '', page: '' }, + state: { endpoint: '', page: '', endpoints: [] }, args: {}, } diff --git a/backend/src/serverless/integrations/coordinators/discordCoordinator.ts b/backend/src/serverless/integrations/coordinators/discordCoordinator.ts index 524603ce4d..19887e558f 100644 --- a/backend/src/serverless/integrations/coordinators/discordCoordinator.ts +++ b/backend/src/serverless/integrations/coordinators/discordCoordinator.ts @@ -16,7 +16,7 @@ async function discordCoordinator(): Promise { sleep: Math.floor(Math.random() * 1200), tenant: integration.tenantId.toString(), onboarding: false, - state: { endpoint: '', page: '' }, + state: { endpoint: '', page: '', endpoints: [] }, args: { guildId: integration.integrationIdentifier, channels: integration.settings.channels || [], diff --git a/backend/src/serverless/integrations/coordinators/slackCoordinator.ts b/backend/src/serverless/integrations/coordinators/slackCoordinator.ts index 861633460b..5f458ecec9 100644 --- a/backend/src/serverless/integrations/coordinators/slackCoordinator.ts +++ b/backend/src/serverless/integrations/coordinators/slackCoordinator.ts @@ -16,7 +16,7 @@ async function slackCoordinator(): Promise { sleep: 0, tenant: integration.tenantId.toString(), onboarding: false, - state: { endpoint: '', page: '' }, + state: { endpoint: '', page: '', endpoints: [] }, args: { channels: integration.settings.channels || [], }, diff --git a/backend/src/serverless/integrations/coordinators/twitterCoordinator.ts b/backend/src/serverless/integrations/coordinators/twitterCoordinator.ts index 4da9049645..fa3df7d000 100644 --- a/backend/src/serverless/integrations/coordinators/twitterCoordinator.ts +++ b/backend/src/serverless/integrations/coordinators/twitterCoordinator.ts @@ -16,7 +16,7 @@ async function twitterCoordinator(): Promise { sleep: 0, tenant: integration.tenantId.toString(), onboarding: false, - state: { endpoint: '', page: '' }, + state: { endpoint: '', page: '', endpoints: [] }, args: { profileId: integration.integrationIdentifier, hashtags: integration.settings.hashtags, diff --git a/backend/src/serverless/integrations/coordinators/twitterReachCoordinator.ts b/backend/src/serverless/integrations/coordinators/twitterReachCoordinator.ts index 686d9255e9..9d37a8aa96 100644 --- a/backend/src/serverless/integrations/coordinators/twitterReachCoordinator.ts +++ b/backend/src/serverless/integrations/coordinators/twitterReachCoordinator.ts @@ -15,7 +15,7 @@ async function twitterReachCoordinator(): Promise { sleep: 0, tenant: microservice.tenantId.toString(), onboarding: false, - state: { endpoint: '', page: '' }, + state: { endpoint: '', page: '', endpoints: [] }, args: { profileId: microservice.microserviceIdentifier, }, diff --git a/backend/src/serverless/integrations/iterators/__tests__/baseIteration.test.ts b/backend/src/serverless/integrations/iterators/__tests__/baseIteration.test.ts index ae25f87b99..a93a8286c2 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/baseIteration.test.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/baseIteration.test.ts @@ -34,6 +34,7 @@ describe('BaseIterator tests', () => { expect(next).toStrictEqual({ endpoint: 'endpoint1', page: '123', + endpoints: ['endpoint1', 'endpoint2', 'endpoint3'], }) }) @@ -46,6 +47,7 @@ describe('BaseIterator tests', () => { expect(next).toStrictEqual({ endpoint: 'endpoint2', page: '', + endpoints: ['endpoint2', 'endpoint3'], }) }) @@ -70,6 +72,7 @@ describe('BaseIterator tests', () => { expect(next).toStrictEqual({ endpoint: 'endpoint2', page: '', + endpoints: ['endpoint2', 'endpoint3'], }) }) @@ -82,6 +85,7 @@ describe('BaseIterator tests', () => { expect(next).toStrictEqual({ endpoint: 'endpoint2', page: '', + endpoints: ['endpoint2', 'endpoint3'], }) }) @@ -112,6 +116,7 @@ describe('BaseIterator tests', () => { expect(next).toStrictEqual({ endpoint: 'endpoint2', page: '', + endpoints: ['endpoint2', 'endpoint3'], }) }) }) @@ -130,10 +135,11 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() + expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint3', page: '' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: [] }, ]) expect(out).toStrictEqual(success) }) @@ -168,14 +174,14 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, - { endpoint: 'endpoint1', page: 'p2' }, - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint2', page: 'p1' }, - { endpoint: 'endpoint2', page: 'p2' }, - { endpoint: 'endpoint3', page: '' }, - { endpoint: 'endpoint3', page: 'p1' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: 'p1', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: 'p2', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: 'p1', endpoints: [] }, ]) expect(out).toStrictEqual(success) }) @@ -201,11 +207,11 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, - { endpoint: 'endpoint1', page: 'p2' }, - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint3', page: '' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: [] }, ]) expect(out).toStrictEqual(success) }) @@ -224,14 +230,15 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, TestIterator.limitReachedState, - { endpoint: 'endpoint1', page: 'p2' }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, ]) expect(iter.state).toStrictEqual({ endpoint: 'endpoint1', page: 'p2', + endpoints: ['endpoint1', 'endpoint2', 'endpoint3'], }) expect(out).toStrictEqual(limitReached) }) @@ -257,14 +264,15 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, TestIterator.limitReachedState, - { endpoint: 'endpoint1', page: 'p2' }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, ]) expect(iter.state).toStrictEqual({ endpoint: 'endpoint1', page: 'p2', + endpoints: ['endpoint1', 'endpoint2', 'endpoint3'], }) expect(out).toStrictEqual(limitReached) }) @@ -290,11 +298,11 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, - { endpoint: 'endpoint1', page: 'p2' }, - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint3', page: '' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: [] }, ]) expect(iter.state).toStrictEqual(TestIterator.endState) expect(out).toStrictEqual(success) @@ -324,17 +332,18 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, - { endpoint: 'endpoint1', page: 'p2' }, - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint3', page: '' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: ['endpoint3'] }, TestIterator.limitReachedState, - { endpoint: 'endpoint3', page: 'p1' }, + { endpoint: 'endpoint3', page: 'p1', endpoints: ['endpoint3'] }, ]) expect(iter.state).toStrictEqual({ endpoint: 'endpoint3', page: 'p1', + endpoints: ['endpoint3'], }) expect(out).toStrictEqual(limitReached) }) @@ -352,11 +361,12 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn, { endpoint: 'endpoint2', page: '', + endpoints: [], }) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint3', page: '' }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: [] }, ]) expect(out).toStrictEqual(success) }) @@ -379,13 +389,14 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn, { endpoint: 'endpoint2', page: 'p1', + endpoints: ['endpoint2', 'endpoint3'], }) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint2', page: 'p1' }, - { endpoint: 'endpoint2', page: 'p2' }, - { endpoint: 'endpoint3', page: '' }, - { endpoint: 'endpoint3', page: 'p1' }, + { endpoint: 'endpoint2', page: 'p1', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: 'p2', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: 'p1', endpoints: [] }, ]) expect(out).toStrictEqual(success) }) @@ -408,18 +419,20 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn, { endpoint: 'endpoint2', page: 'p1', + endpoints: ['endpoint2', 'endpoint3'], }) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint2', page: 'p1' }, - { endpoint: 'endpoint2', page: 'p2' }, - { endpoint: 'endpoint3', page: '' }, + { endpoint: 'endpoint2', page: 'p1', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: 'p2', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: ['endpoint3'] }, TestIterator.limitReachedState, - { endpoint: 'endpoint3', page: 'p1' }, + { endpoint: 'endpoint3', page: 'p1', endpoints: ['endpoint3'] }, ]) expect(iter.state).toStrictEqual({ endpoint: 'endpoint3', page: 'p1', + endpoints: ['endpoint3'], }) expect(out).toStrictEqual(limitReached) }) @@ -449,26 +462,28 @@ describe('BaseIterator tests', () => { const iter = new TestIterator(itFn) const out = await iter.iterate() expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, TestIterator.limitReachedState, - { endpoint: 'endpoint1', page: 'p2' }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, // still all endpoints because next is not called ]) expect(iter.state).toStrictEqual({ endpoint: 'endpoint1', page: 'p2', + endpoints: ['endpoint1', 'endpoint2', 'endpoint3'], }) expect(out).toStrictEqual(limitReached) const iter2 = new TestIterator(itFn2, { endpoint: 'endpoint1', page: 'p2', + endpoints: [], }) const out2 = await iter2.iterate() expect(iter2.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: 'p2' }, - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint3', page: '' }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: [] }, ]) expect(out2).toStrictEqual(success) }) @@ -510,12 +525,12 @@ describe('BaseIterator tests', () => { const out = await iter.iterate() expect(out).toStrictEqual(TestIterator.success) expect(iter.audits).toStrictEqual([ - { endpoint: 'endpoint1', page: '' }, - { endpoint: 'endpoint1', page: 'p1' }, - { endpoint: 'endpoint1', page: 'p2' }, - { endpoint: 'endpoint2', page: '' }, - { endpoint: 'endpoint3', page: '' }, - { endpoint: 'endpoint3', page: 'p3' }, + { endpoint: 'endpoint1', page: '', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p1', endpoints: ['endpoint1', 'endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint1', page: 'p2', endpoints: ['endpoint2', 'endpoint3'] }, + { endpoint: 'endpoint2', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: '', endpoints: ['endpoint3'] }, + { endpoint: 'endpoint3', page: 'p3', endpoints: [] }, ]) }) }) @@ -561,27 +576,27 @@ describe('BaseIterator tests', () => { describe('isFinished tests', () => { const endpoints = ['endpoint1', 'endpoint2', 'endpoint3'] it('Should return not finished for a nextPage given', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true) const nextPage = 'here' const isFinished = iter.isFinished(endpoints, 'endpoint2', nextPage) expect(isFinished).toBe(false) }) it('Should return not finished for not nextPage given but in the middle of endpoints', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true) const nextPage = undefined const isFinished = iter.isFinished(endpoints, 'endpoint2', nextPage) expect(isFinished).toBe(false) }) it('Should return not finished for page given and last endpoint', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true) const isFinished = iter.isFinished(endpoints, 'endpoint3', 'here') expect(isFinished).toBe(false) }) it('Should return finished for no nextPage given and last endpoint', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true) const isFinished = iter.isFinished(endpoints, 'endpoint3', undefined) expect(isFinished).toBe(true) }) @@ -589,47 +604,51 @@ describe('BaseIterator tests', () => { describe('Global limit reached before execution tests', () => { it('Should stop before iteration when the global limit or larger is provided as init count', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true, 18) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true, 18) const out = await iter.iterate() expect(out).toStrictEqual(TestIterator.success) expect(iter.audits).toStrictEqual([]) }) it('Should stop before iteration when the global limit or larger is provided as init count', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true, 25) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true, 25) const out = await iter.iterate() expect(out).toStrictEqual(TestIterator.success) expect(iter.audits).toStrictEqual([]) }) it('Should start before iteration when a small limit is provided as init count', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true, 5) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true, 5) const out = await iter.iterate() expect(out).toStrictEqual(TestIterator.success) expect(iter.audits).toStrictEqual([ { endpoint: 'endpoint1', page: '', + endpoints: ['endpoint2', 'endpoint3'], }, { endpoint: 'endpoint2', page: '', + endpoints: ['endpoint3'], }, { endpoint: 'endpoint3', page: '', + endpoints: [], }, ]) }) it('Should stop when a limit almost as the global limit is provided', async () => { - const iter = new TestIterator((n) => n, { endpoint: '', page: '' }, true, 17) + const iter = new TestIterator((n) => n, { endpoint: '', page: '', endpoints: [] }, true, 17) const out = await iter.iterate() expect(out).toStrictEqual(TestIterator.success) expect(iter.audits).toStrictEqual([ { endpoint: 'endpoint1', page: '', + endpoints: [], }, ]) }) diff --git a/backend/src/serverless/integrations/iterators/__tests__/baseIteratorStatic.test.ts b/backend/src/serverless/integrations/iterators/__tests__/baseIteratorStatic.test.ts index c931164c43..01b77b4837 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/baseIteratorStatic.test.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/baseIteratorStatic.test.ts @@ -5,35 +5,35 @@ describe('Integrations worker static tests', () => { const endpoints = ['endpoint1', 'endpoint2', 'endpoint3'] describe('Get iterator tests', () => { it('Should return the endpoints to iterate for a standard state', async () => { - const state = { endpoint: 'endpoint2', page: 'here' } + const state = { endpoint: 'endpoint2', page: 'here', endpoints: [] } const iterator = BaseIterator.getEndPointsIterator(endpoints, state) expect(iterator).toStrictEqual(['endpoint2', 'endpoint3']) }) it('Should return the endpoints to iterate for a state with no page', async () => { - const state = { endpoint: 'endpoint2', page: '' } + const state = { endpoint: 'endpoint2', page: '', endpoints: [] } const iterator = BaseIterator.getEndPointsIterator(endpoints, state) expect(iterator).toStrictEqual(['endpoint2', 'endpoint3']) }) it('Should return the whole endpoints when the state is in the first endpoint with a page', async () => { - const state = { endpoint: 'endpoint1', page: 'here' } + const state = { endpoint: 'endpoint1', page: 'here', endpoints: [] } const iterator = BaseIterator.getEndPointsIterator(endpoints, state) expect(iterator).toStrictEqual(endpoints) }) it('Should return the last endpoint when the state is in the last endpoint', async () => { - const state = { endpoint: 'endpoint3', page: '' } + const state = { endpoint: 'endpoint3', page: '', endpoints: [] } const iterator = BaseIterator.getEndPointsIterator(endpoints, state) expect(iterator).toStrictEqual(['endpoint3']) }) it('Should throw an error when state.endpoint is not in endpoints', async () => { - const state = { endpoint: 'endpoint4', page: '' } + const state = { endpoint: 'endpoint4', page: '', endpoints: [] } try { BaseIterator.getEndPointsIterator(endpoints, state) } catch (error: any) { @@ -48,10 +48,12 @@ describe('Integrations worker static tests', () => { const state = BaseIterator.initState(endpoints, { endpoint: '', page: '', + endpoints: [], }) expect(state).toStrictEqual({ endpoint: 'endpoint1', page: '', + endpoints: ['endpoint1', 'endpoint2', 'endpoint3'], }) }) @@ -59,6 +61,7 @@ describe('Integrations worker static tests', () => { const givenState = { endpoint: 'endpoint2', page: 'here', + endpoints: [], } const state = BaseIterator.initState(endpoints, givenState) expect(state).toStrictEqual(givenState) diff --git a/backend/src/serverless/integrations/iterators/__tests__/devtoIterator.test.ts b/backend/src/serverless/integrations/iterators/__tests__/devtoIterator.test.ts index c77975904c..668ca0975b 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/devtoIterator.test.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/devtoIterator.test.ts @@ -76,6 +76,7 @@ async function getDevtoIterator(articles: DevtoArticle[]) { { endpoint: '', page: '', + endpoints: [], }, false, ) diff --git a/backend/src/serverless/integrations/iterators/__tests__/discordIterator.test.ts b/backend/src/serverless/integrations/iterators/__tests__/discordIterator.test.ts index 2eb875d9eb..c4e55c9f7a 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/discordIterator.test.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/discordIterator.test.ts @@ -589,7 +589,7 @@ describe('Discord iterator tests', () => { 'guild12345', 'token12345', channels, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const isOver = iter.integrationSpecificIsEndpointFinished('1', recentRecord) @@ -605,7 +605,7 @@ describe('Discord iterator tests', () => { 'guild12345', 'token12345', channels, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const isOver = iter.integrationSpecificIsEndpointFinished('4', recentRecord) @@ -621,7 +621,7 @@ describe('Discord iterator tests', () => { 'guild12345', 'token12345', channels, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const isOver = iter.integrationSpecificIsEndpointFinished('3', recentRecord) @@ -637,7 +637,7 @@ describe('Discord iterator tests', () => { 'guild12345', 'token12345', channels, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const isOver = iter.integrationSpecificIsEndpointFinished('1', oldRecord) @@ -653,7 +653,7 @@ describe('Discord iterator tests', () => { 'guild12345', 'token12345', channels, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const isOver = iter.integrationSpecificIsEndpointFinished('3', oldRecord) @@ -669,7 +669,7 @@ describe('Discord iterator tests', () => { 'guild12345', 'token12345', channels, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const isOver = iter.integrationSpecificIsEndpointFinished('4', oldRecord) @@ -685,7 +685,7 @@ describe('Discord iterator tests', () => { 'guild12345', 'token12345', channels, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const isOverGeneral = iter.isEndpointFinished('4', {}, []) diff --git a/backend/src/serverless/integrations/iterators/__tests__/githubIterator.test.ts b/backend/src/serverless/integrations/iterators/__tests__/githubIterator.test.ts index 9c4f6819d6..e7f1400324 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/githubIterator.test.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/githubIterator.test.ts @@ -38,6 +38,7 @@ async function getGithubIterator(repos: Repos, options: IRepositoryOptions) { { endpoint: '', page: '', + endpoints: [], }, true, ) diff --git a/backend/src/serverless/integrations/iterators/__tests__/slackIterator.test.ts b/backend/src/serverless/integrations/iterators/__tests__/slackIterator.test.ts index 38c931d524..7bd5d72138 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/slackIterator.test.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/slackIterator.test.ts @@ -30,7 +30,7 @@ async function getSlackIterator(members = {}, channels = [{ name: 'dev', id: 'C0 members, integrationId, mockIRepositoryOptions, - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, ) } diff --git a/backend/src/serverless/integrations/iterators/__tests__/testIterator.ts b/backend/src/serverless/integrations/iterators/__tests__/testIterator.ts index 8cda4cda8e..cb12b0b631 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/testIterator.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/testIterator.ts @@ -1,11 +1,9 @@ /* eslint @typescript-eslint/no-unused-vars: 0 */ /* eslint class-methods-use-this: 0 */ -import { endpoint } from 'aws-sdk/clients/sns' import moment from 'moment' -import { Endpoint } from 'aws-sdk' import { BaseOutput, parseOutput, IntegrationResponse } from '../../types/iteratorTypes' -import { State } from '../../types/regularTypes' +import { Endpoint, State } from '../../types/regularTypes' import BaseIterator from '../baseIterator' import { AddActivitiesSingle } from '../../types/messageTypes' @@ -19,6 +17,7 @@ export default class TestIterator extends BaseIterator { static limitReachedState: State = { endpoint: '__limit', + endpoints: [], page: '__limit', } @@ -33,7 +32,7 @@ export default class TestIterator extends BaseIterator { */ constructor( transitionFn: Function, - state: State = { endpoint: '', page: '' }, + state: State = { endpoint: '', page: '', endpoints: [] }, onboarding: boolean = false, limitCount: number = 0, ) { @@ -138,4 +137,12 @@ export default class TestIterator extends BaseIterator { async iterate(maxTime: number = 12 * 60): Promise { return super.iterate(maxTime, false) } + + next(currentEndpoint: Endpoint, nextPage: string | undefined, parseOutput: parseOutput): State { + const next = super.next(currentEndpoint, nextPage, parseOutput) + if (this.audits.length > 0) { + this.audits[this.audits.length - 1].endpoints = next.endpoints + } + return next + } } diff --git a/backend/src/serverless/integrations/iterators/__tests__/twitterIterator.test.ts b/backend/src/serverless/integrations/iterators/__tests__/twitterIterator.test.ts index d1efec2698..7199890637 100644 --- a/backend/src/serverless/integrations/iterators/__tests__/twitterIterator.test.ts +++ b/backend/src/serverless/integrations/iterators/__tests__/twitterIterator.test.ts @@ -47,14 +47,14 @@ describe('Integrations worker static tests', () => { it('It should return the proper message', async () => { const iter = new TwitterIterator('tenant12345', 'profile12345', 'token', ['#1', '#2']) const message: TwitterIntegrationMessage = iter.getSQSBody( - { endpoint: 'hashtag/#1', page: '1' }, + { endpoint: 'hashtag/#1', page: '1', endpoints: [] }, 42, ) expect(message).toStrictEqual({ tenant: 'tenant12345', integration: PlatformType.TWITTER, onboarding: false, - state: { endpoint: 'hashtag/#1', page: '1' }, + state: { endpoint: 'hashtag/#1', page: '1', endpoints: [] }, sleep: 42, args: { profileId: 'profile12345', @@ -653,7 +653,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, ) const { activities } = iter.parseActivities(followers, 'followers') @@ -947,7 +947,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, 10, existingFollowers, @@ -1000,7 +1000,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, 10, existingFollowers, @@ -1058,7 +1058,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, 10, existingFollowers, @@ -1098,7 +1098,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, 10, existingFollowers, @@ -1134,7 +1134,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, 10, existingFollowers, @@ -1154,7 +1154,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, 10, existingFollowers, @@ -1171,7 +1171,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, 10, existingFollowers, @@ -1220,7 +1220,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, 10, existingFollowers, @@ -1250,7 +1250,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, 10, existingFollowers, @@ -1278,7 +1278,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, 10, existingFollowers, @@ -1308,7 +1308,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, initialCount, ) @@ -1361,7 +1361,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, initialCount, ) @@ -1414,7 +1414,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, true, initialCount, ) @@ -1431,7 +1431,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, initialCount, ) @@ -1443,6 +1443,7 @@ describe('Integrations worker static tests', () => { expect(next).toStrictEqual({ endpoint: 'followers', page: 'p124', + endpoints: ['followers', 'mentions', 'hashtag/#1', 'hashtag/#2'], }) }) @@ -1453,7 +1454,7 @@ describe('Integrations worker static tests', () => { 'profile12345', 'token', ['#1', '#2'], - { endpoint: '', page: '' }, + { endpoint: '', page: '', endpoints: [] }, false, initialCount, ) @@ -1471,6 +1472,7 @@ describe('Integrations worker static tests', () => { const iter = new TwitterIterator('tenant12345', 'profile12345', 'token', ['#1', '#2'], { endpoint: '', page: '', + endpoints: [], }) const date = iter.getAfterDate() expect(moment(date).unix()).toBeCloseTo( diff --git a/backend/src/serverless/integrations/iterators/baseIterator.ts b/backend/src/serverless/integrations/iterators/baseIterator.ts index e26e74a498..3ec5febd52 100644 --- a/backend/src/serverless/integrations/iterators/baseIterator.ts +++ b/backend/src/serverless/integrations/iterators/baseIterator.ts @@ -26,6 +26,7 @@ export default abstract class BaseIterator { startTimestamp: number static endState: State = { + endpoints: [], endpoint: '__finished', page: '__finished', } @@ -50,13 +51,19 @@ export default abstract class BaseIterator { constructor( tenant: string, endPoints: Endpoints, - state: State = { endpoint: '', page: '' }, + state: State = { endpoint: '', page: '', endpoints: [] }, onboarding: boolean = false, globalLimit: number = Infinity, limitCount: number = 0, ) { this.tenant = tenant - this.endpoints = endPoints + + if (state.endpoints.length > 0 && state.endpoint) { + this.endpoints = state.endpoints + } else { + this.endpoints = endPoints + } + this.state = BaseIterator.initState(this.endpoints, state) this.startTimestamp = moment().utc().unix() @@ -107,6 +114,7 @@ export default abstract class BaseIterator { while (this.state !== BaseIterator.endState) { // Get the current endpoint and page const { endpoint, page } = this.state + // Get the response from the API and parse it. // We also get the date of the latest activity const response: IntegrationResponse = await this.get(endpoint, page) @@ -138,6 +146,7 @@ export default abstract class BaseIterator { } // Get the next state this.state = this.next(endpoint, response.nextPage, parseOutput) + this.endpoints = this.state.endpoints // If we are not done if (this.state !== BaseIterator.endState) { @@ -237,8 +246,9 @@ export default abstract class BaseIterator { return lodash.isEqual(startState, { endpoint: '', page: '', + endpoints: [], }) - ? { endpoint: endpoints[0], page: '' } + ? { endpoint: endpoints[0], page: '', endpoints } : startState } @@ -292,6 +302,7 @@ export default abstract class BaseIterator { : { endpoint: this.endpointsIterator[this.endpointsIterator.indexOf(currentEndpoint) + 1], page: '', + endpoints: this.endpoints.slice(this.endpoints.indexOf(currentEndpoint) + 1), } } // If we do not have a next page, return the next endpoint with an empty page and 0 for number @@ -301,6 +312,9 @@ export default abstract class BaseIterator { ? currentEndpoint : this.endpointsIterator[this.endpointsIterator.indexOf(currentEndpoint) + 1], page: nextPage || '', + endpoints: nextPage + ? this.endpoints + : this.endpoints.slice(this.endpoints.indexOf(currentEndpoint) + 1), } } diff --git a/backend/src/serverless/integrations/iterators/devtoIterator.ts b/backend/src/serverless/integrations/iterators/devtoIterator.ts index 353f8e306e..615c5e15bc 100644 --- a/backend/src/serverless/integrations/iterators/devtoIterator.ts +++ b/backend/src/serverless/integrations/iterators/devtoIterator.ts @@ -36,7 +36,7 @@ export default class DevtoIterator extends BaseIterator { articles: DevtoArticle[], userContext: IRepositoryOptions, integrationId: string, - state: State = { endpoint: '', page: '' }, + state: State = { endpoint: '', page: '', endpoints: [] }, onboarding: boolean = false, ) { const endpoints: Endpoints = articles.map((a) => a.id.toString()) diff --git a/backend/src/serverless/integrations/iterators/discordIterator.ts b/backend/src/serverless/integrations/iterators/discordIterator.ts index d26f71c2ae..db6f5b0aee 100644 --- a/backend/src/serverless/integrations/iterators/discordIterator.ts +++ b/backend/src/serverless/integrations/iterators/discordIterator.ts @@ -20,6 +20,7 @@ import { PlatformType } from '../../../utils/platforms' export default class DiscordIterator extends BaseIterator { static limitReachedState: State = { + endpoints: [], endpoint: '__limit', page: '__limit', } @@ -56,7 +57,7 @@ export default class DiscordIterator extends BaseIterator { guildId: string, botToken: string, channels: Channels, - state: State = { endpoint: '', page: '' }, + state: State = { endpoint: '', page: '', endpoints: [] }, onboarding: boolean = false, ) { // Endpoints are the fixed endpoints plus the channels diff --git a/backend/src/serverless/integrations/iterators/githubIterator.ts b/backend/src/serverless/integrations/iterators/githubIterator.ts index 7a741f3c6c..83d83b6fad 100644 --- a/backend/src/serverless/integrations/iterators/githubIterator.ts +++ b/backend/src/serverless/integrations/iterators/githubIterator.ts @@ -27,6 +27,7 @@ import Error400 from '../../../errors/Error400' export default class GithubIterator extends BaseIterator { static limitReachedState: State = { + endpoints: [], endpoint: '__limit', page: '__limit', } @@ -56,16 +57,21 @@ export default class GithubIterator extends BaseIterator { tenant: string, repos: Repos, accessToken: string, - state: State = { endpoint: '', page: '' }, + state: State = { endpoints: [], endpoint: '', page: '' }, onboarding: boolean = false, ) { - const endpoints: Endpoints = repos.reduce((acc, repo) => { - const repoEndpoints = GithubIterator.fixedEndpoints.map( - (endpoint) => `${repo.name}|${endpoint}`, - ) - acc.push(...repoEndpoints) - return acc - }, []) + let endpoints: Endpoints + if (state.endpoints.length === 0) { + endpoints = repos.reduce((acc, repo) => { + const repoEndpoints = GithubIterator.fixedEndpoints.map( + (endpoint) => `${repo.name}|${endpoint}`, + ) + acc.push(...repoEndpoints) + return acc + }, []) + } else { + endpoints = state.endpoints + } super(tenant, endpoints, state, onboarding, GithubIterator.globalLimit) this.repos = repos diff --git a/backend/src/serverless/integrations/iterators/slackIterator.ts b/backend/src/serverless/integrations/iterators/slackIterator.ts index 94ab661d10..5f1d78305b 100644 --- a/backend/src/serverless/integrations/iterators/slackIterator.ts +++ b/backend/src/serverless/integrations/iterators/slackIterator.ts @@ -27,6 +27,7 @@ import { PlatformType } from '../../../utils/platforms' export default class SlackIterator extends BaseIterator { static limitReachedState: State = { + endpoints: [], endpoint: '__limit', page: '__limit', } @@ -72,13 +73,14 @@ export default class SlackIterator extends BaseIterator { members: Object, integrationId: string, userContext: IRepositoryOptions, - state: State = { endpoint: '', page: '' }, + state: State = { endpoint: '', page: '', endpoints: [] }, onboarding: boolean = false, ) { + let endpoints: Endpoints = state.endpoints // Endpoints are the fixed endpoints plus the channels - const endpoints: Endpoints = SlackIterator.fixedEndpoints.concat( - channels.map((channel) => channel.id), - ) + if (state.endpoints.length === 0) { + endpoints = SlackIterator.fixedEndpoints.concat(channels.map((channel) => channel.id)) + } super(tenant, endpoints, state, onboarding, SlackIterator.globalLimit) diff --git a/backend/src/serverless/integrations/iterators/twitterIterator.ts b/backend/src/serverless/integrations/iterators/twitterIterator.ts index 721d04c6b4..a044e14b2b 100644 --- a/backend/src/serverless/integrations/iterators/twitterIterator.ts +++ b/backend/src/serverless/integrations/iterators/twitterIterator.ts @@ -25,6 +25,7 @@ import { PlatformType } from '../../../utils/platforms' export default class TwitterIterator extends BaseIterator { static limitReachedState: State = { + endpoints: [], endpoint: '__limit', page: '__limit', } @@ -57,7 +58,7 @@ export default class TwitterIterator extends BaseIterator { profileId: string, accessToken: string, hashtags: Array, - state: State = { endpoint: '', page: '' }, + state: State = { endpoint: '', page: '', endpoints: [] }, onboarding: boolean = false, tweetCount: number = 0, followers: Set = new Set(), diff --git a/backend/src/serverless/integrations/iterators/twitterReachIterator.ts b/backend/src/serverless/integrations/iterators/twitterReachIterator.ts index 6ddfbdb8d2..97c1c8fa22 100644 --- a/backend/src/serverless/integrations/iterators/twitterReachIterator.ts +++ b/backend/src/serverless/integrations/iterators/twitterReachIterator.ts @@ -23,7 +23,7 @@ import { PlatformType } from '../../../utils/platforms' export default class TwitterReachIterator extends BaseIterator { static readonly TWITTER_API_MAX_USERNAME_LENGTH = 15 - static limitReachedState: State = { endpoint: '__limit', page: '__limit' } + static limitReachedState: State = { endpoint: '__limit', page: '__limit', endpoints: [] } static maxRetrospect: number = Number(process.env.TWITTER_MAX_RETROSPECT_IN_SECONDS || 7380) @@ -48,7 +48,7 @@ export default class TwitterReachIterator extends BaseIterator { profileId: string, accessToken: string, members: Array, - state: State = { endpoint: '', page: '' }, + state: State = { endpoint: '', page: '', endpoints: [] }, ) { super(tenant, members, state, false) diff --git a/backend/src/serverless/integrations/types/regularTypes.ts b/backend/src/serverless/integrations/types/regularTypes.ts index bf3af3d222..eefbcb7414 100644 --- a/backend/src/serverless/integrations/types/regularTypes.ts +++ b/backend/src/serverless/integrations/types/regularTypes.ts @@ -21,6 +21,7 @@ export type Endpoint = string export type Endpoints = Array export type State = { - endpoint: Endpoint + endpoints: Endpoints + endpoint: string page: string } diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 56ad2a7bd1..eb43fe3ec1 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -262,6 +262,7 @@ export default class IntegrationService { state: { endpoint: '', page: '', + endpoints: [], }, tenant: integration.tenantId.toString(), sleep: 0, @@ -294,6 +295,7 @@ export default class IntegrationService { state: { endpoint: '', page: '', + endpoints: [], }, tenant: integration.tenantId.toString(), sleep: 0, @@ -336,7 +338,7 @@ export default class IntegrationService { integrationId: integration.id, tenant: integration.tenantId.toString(), onboarding: true, - state: { endpoint: '', page: '' }, + state: { endpoint: '', page: '', endpoints: [] }, args: {}, } @@ -364,6 +366,7 @@ export default class IntegrationService { state: { endpoint: '', page: '', + endpoints: [], }, tenant: integration.tenantId.toString(), sleep: 0, @@ -417,6 +420,7 @@ export default class IntegrationService { state: { endpoint: '', page: '', + endpoints: [], }, tenant: integration.tenantId.toString(), sleep: 0, From 22814c80ebf7c01a7d48f86c8af187a6f6979ecc Mon Sep 17 00:00:00 2001 From: anil Date: Tue, 13 Sep 2022 10:31:37 +0300 Subject: [PATCH 7/7] workaround for null byte strings in issues --- .../src/serverless/integrations/iterators/githubIterator.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/serverless/integrations/iterators/githubIterator.ts b/backend/src/serverless/integrations/iterators/githubIterator.ts index 83d83b6fad..ddd27ec76d 100644 --- a/backend/src/serverless/integrations/iterators/githubIterator.ts +++ b/backend/src/serverless/integrations/iterators/githubIterator.ts @@ -454,7 +454,7 @@ export default class GithubIterator extends BaseIterator { url: record.url ? record.url : '', repo: this.getRepoByName(repo).url, state: record.state.toLowerCase(), - title: record.title, + title: record.title.replace(/\0/g, ''), }, communityMember: this.parseMember(record.author), score: GitHubGrid.issueOpened.score, @@ -519,7 +519,7 @@ export default class GithubIterator extends BaseIterator { url: record.url ? record.url : '', repo: this.getRepoByName(repo).url, state: record.state.toLowerCase(), - title: record.title, + title: record.title.replace(/\0/g, ''), }, communityMember: this.parseMember(record.author), score: GitHubGrid.pullRequestOpened.score,