Skip to content

Commit 09c0478

Browse files
themaroltthemarolt
authored
Automations Optimization, Devto automation activity type frontend fix & cli script for starting premium services (#165)
1 parent ee9dfda commit 09c0478

File tree

24 files changed

+328
-197
lines changed

24 files changed

+328
-197
lines changed

‎backend/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"start:nodejs-worker:dev": "nodemon --watch \"src/**/*.ts\" -e ts,json --exec \"ts-node --transpile-only ./src/bin/nodejs-worker.ts\"",
1313
"start:nodejs-worker:dev:local": "set -a && . ./.env.dist.local && . ./.env.override.local && set +a && SERVICE=nodejs-worker nodemon --watch \"src/**/*.ts\" -e ts,json --exec \"node --inspect=0.0.0.0:9229 -r ts-node/register ./src/bin/nodejs-worker.ts --transpile-only\"",
1414
"build": "tsc && npm run build:documentation && cp package*json dist/ && cp .sequelizerc dist/.sequelizerc ",
15-
"test": "docker-compose -f ../docker/docker-compose.test.yaml up -d && jest --clearCache && set -a && . ./.env.dist.local && . ./.env.test && set +a && NODE_ENV=test SERVICE=test jest --runInBand --verbose",
15+
"test": "../scripts/cli scaffold up-test && jest --clearCache && set -a && . ./.env.dist.local && . ./.env.test && set +a && NODE_ENV=test SERVICE=test jest --runInBand --verbose",
1616
"build:documentation": "copyfiles --flat ./src/documentation/openapi.json ./dist/documentation/",
1717
"start:integrations": "nodemon --watch \"src/**/*.ts\" -e ts,json --exec \"cd src/serverless/integrations && npm run sls-webpack\" --on-change-only",
1818
"start:dbOperations": "nodemon --watch \"src/**/*.ts\" -e ts,json --exec \"cd src/serverless/dbOperations && npm run sls-webpack\" --on-change-only",

‎backend/src/database/repositories/__tests__/memberRepository.test.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -730,6 +730,8 @@ describe('MemberRepository tests', () => {
730730
},
731731
])
732732

733+
await SequelizeTestUtils.refreshMaterializedViews(db)
734+
733735
const members = await MemberRepository.findAndCountAll(
734736
{ filter: {}, orderBy: 'activityCount_DESC' },
735737
mockIRepositoryOptions,
@@ -782,6 +784,8 @@ describe('MemberRepository tests', () => {
782784
mockIRepositoryOptions,
783785
)
784786

787+
await SequelizeTestUtils.refreshMaterializedViews(db)
788+
785789
const members = await MemberRepository.findAndCountAll(
786790
{ filter: { tags: [nodeTag.id, vueTag.id] } },
787791
mockIRepositoryOptions,
@@ -834,6 +838,8 @@ describe('MemberRepository tests', () => {
834838
mockIRepositoryOptions,
835839
)
836840

841+
await SequelizeTestUtils.refreshMaterializedViews(db)
842+
837843
const members = await MemberRepository.findAndCountAll(
838844
{ filter: { tags: [nodeTag.id] } },
839845
mockIRepositoryOptions,
@@ -890,6 +896,8 @@ describe('MemberRepository tests', () => {
890896
mockIRepositoryOptions,
891897
)
892898

899+
await SequelizeTestUtils.refreshMaterializedViews(db)
900+
893901
const members = await MemberRepository.findAndCountAll(
894902
{ filter: { organizations: [crowd.id, pp.id] } },
895903
mockIRepositoryOptions,
@@ -925,6 +933,8 @@ describe('MemberRepository tests', () => {
925933
await MemberRepository.create(user2, mockIRepositoryOptions)
926934
await MemberRepository.create(user3, mockIRepositoryOptions)
927935

936+
await SequelizeTestUtils.refreshMaterializedViews(db)
937+
928938
const members = await MemberRepository.findAndCountAll(
929939
{ filter: { scoreRange: [1, 6] } },
930940
mockIRepositoryOptions,
@@ -960,6 +970,8 @@ describe('MemberRepository tests', () => {
960970
await MemberRepository.create(user2, mockIRepositoryOptions)
961971
await MemberRepository.create(user3, mockIRepositoryOptions)
962972

973+
await SequelizeTestUtils.refreshMaterializedViews(db)
974+
963975
const members = await MemberRepository.findAndCountAll(
964976
{ filter: { scoreRange: [7] } },
965977
mockIRepositoryOptions,
@@ -1122,6 +1134,8 @@ describe('MemberRepository tests', () => {
11221134
},
11231135
])
11241136

1137+
await SequelizeTestUtils.refreshMaterializedViews(db)
1138+
11251139
let members = await MemberRepository.findAndCountAll(
11261140
{
11271141
filter: {},

‎backend/src/database/repositories/__tests__/organizationRepository.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -782,6 +782,8 @@ describe('OrganizationRepository tests', () => {
782782
await createOrganization(piedpiper, mockIRepositoryOptions)
783783
await createOrganization(hooli, mockIRepositoryOptions)
784784

785+
await SequelizeTestUtils.refreshMaterializedViews(db)
786+
785787
const memberId = await (
786788
await MemberRepository.findAndCountAll({}, mockIRepositoryOptions)
787789
).rows[0].id
@@ -811,6 +813,8 @@ describe('OrganizationRepository tests', () => {
811813
await createOrganization(piedpiper, mockIRepositoryOptions)
812814
await createOrganization(hooli, mockIRepositoryOptions)
813815

816+
await SequelizeTestUtils.refreshMaterializedViews(db)
817+
814818
const memberId = await (
815819
await MemberRepository.findAndCountAll({}, mockIRepositoryOptions)
816820
).rows[0].id

‎backend/src/database/repositories/__tests__/taskRepository.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -575,6 +575,8 @@ describe('TaskRepository tests', () => {
575575
await TaskRepository.create(toCreate1, mockIRepositoryOptions)
576576
await TaskRepository.create(toCreate2, mockIRepositoryOptions)
577577

578+
await SequelizeTestUtils.refreshMaterializedViews(db)
579+
578580
const member = (
579581
await MemberRepository.findAndCountAll(
580582
{
@@ -722,6 +724,8 @@ describe('TaskRepository tests', () => {
722724
await TaskRepository.create(toCreate2, mockIRepositoryOptions)
723725
await TaskRepository.create(toCreate3, mockIRepositoryOptions)
724726

727+
await SequelizeTestUtils.refreshMaterializedViews(db)
728+
725729
const members = (
726730
await MemberRepository.findAndCountAll(
727731
{

‎backend/src/database/utils/sequelizeTestUtils.ts‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,36 @@ import SettingsRepository from '../repositories/settingsRepository'
1414
export default class SequelizeTestUtils {
1515
static async wipeDatabase(db) {
1616
db = await this.getDatabase(db)
17-
await db.sequelize.sync({ force: true })
17+
await db.sequelize.query(`
18+
truncate table
19+
tenants,
20+
integrations,
21+
activities,
22+
members,
23+
automations,
24+
"automationExecutions",
25+
conversations,
26+
notes,
27+
reports,
28+
organizations,
29+
"organizationCaches",
30+
settings,
31+
tags,
32+
tasks,
33+
users,
34+
files,
35+
microservices,
36+
"eagleEyeContents",
37+
"auditLogs"
38+
cascade;
39+
`)
40+
}
41+
42+
static async refreshMaterializedViews(db) {
43+
db = await this.getDatabase(db)
44+
await db.sequelize.query(
45+
'refresh materialized view concurrently "memberActivityAggregatesMVs";',
46+
)
1847
}
1948

2049
static async getDatabase(db?) {

‎backend/src/serverless/dbOperations/__tests__/operationsWorker.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ describe('Serverless database operations worker tests', () => {
3030

3131
await worker(tenantId, 'upsert_members', [member])
3232

33+
await SequelizeTestUtils.refreshMaterializedViews(db)
34+
3335
const dbMembers = (await new MemberService(mockIRepositoryOptions).findAndCountAll({})).rows
3436

3537
expect(dbMembers.length).toBe(1)
@@ -53,6 +55,8 @@ describe('Serverless database operations worker tests', () => {
5355

5456
await worker(tenantId, 'upsert_members', members)
5557

58+
await SequelizeTestUtils.refreshMaterializedViews(db)
59+
5660
const dbMembers = (await new MemberService(mockIRepositoryOptions).findAndCountAll({})).rows
5761

5862
expect(dbMembers.length).toBe(2)
@@ -157,6 +161,8 @@ describe('Serverless database operations worker tests', () => {
157161

158162
await worker(tenantId, 'update_members', [{ id: memberId, update: { score: 10 } }])
159163

164+
await SequelizeTestUtils.refreshMaterializedViews(db)
165+
160166
const dbMembers = (await new MemberService(mockIRepositoryOptions).findAndCountAll({})).rows
161167

162168
expect(dbMembers.length).toBe(1)
@@ -192,6 +198,8 @@ describe('Serverless database operations worker tests', () => {
192198
{ id: memberIds[1], update: { score: 3 } },
193199
])
194200

201+
await SequelizeTestUtils.refreshMaterializedViews(db)
202+
195203
const dbMembers = (await new MemberService(mockIRepositoryOptions).findAndCountAll({})).rows
196204

197205
expect(dbMembers.length).toBe(2)
@@ -403,6 +411,8 @@ describe('Serverless database operations worker tests', () => {
403411
[memberIds[1], memberIds[0]],
404412
])
405413

414+
await SequelizeTestUtils.refreshMaterializedViews(db)
415+
406416
const dbMergeMembers = (await new MemberService(mockIRepositoryOptions).findAndCountAll({}))
407417
.rows
408418

@@ -436,6 +446,8 @@ describe('Serverless database operations worker tests', () => {
436446

437447
await worker(tenantId, 'update_members_to_merge', [])
438448

449+
await SequelizeTestUtils.refreshMaterializedViews(db)
450+
439451
const dbMergeMembers = (await new MemberService(mockIRepositoryOptions).findAndCountAll({}))
440452
.rows
441453

‎backend/src/serverless/microservices/nodejs/automation/workers/newActivityWorker.ts‎

Lines changed: 34 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,20 @@ import { prepareMemberPayload } from './newMemberWorker'
1313

1414
/**
1515
* Helper function to check whether a single activity should be processed by automation
16-
* @param activityData Activity data
16+
* @param activity Activity data
1717
* @param automation {AutomationData} Automation data
1818
*/
19-
export const shouldProcessActivity = (activityData, automation: AutomationData): boolean => {
19+
export const shouldProcessActivity = (activity: any, automation: AutomationData): boolean => {
2020
const settings = automation.settings as NewActivitySettings
2121

2222
let process = true
2323

2424
// check whether activity type matches
2525
if (settings.types && settings.types.length > 0) {
26-
if (!settings.types.includes(activityData.type)) {
26+
if (!settings.types.includes(activity.type)) {
2727
console.log(
28-
`Ignoring automation ${automation.id} - Activity ${activityData.id} type '${
29-
activityData.type
28+
`Ignoring automation ${automation.id} - Activity ${activity.id} type '${
29+
activity.type
3030
}' does not match automation setting types: [${settings.types.join(', ')}]`,
3131
)
3232
process = false
@@ -35,10 +35,10 @@ export const shouldProcessActivity = (activityData, automation: AutomationData):
3535

3636
// check whether activity platform matches
3737
if (process && settings.platforms && settings.platforms.length > 0) {
38-
if (!settings.platforms.includes(activityData.platform)) {
38+
if (!settings.platforms.includes(activity.platform)) {
3939
console.log(
40-
`Ignoring automation ${automation.id} - Activity ${activityData.id} platform '${
41-
activityData.platform
40+
`Ignoring automation ${automation.id} - Activity ${activity.id} platform '${
41+
activity.platform
4242
}' does not match automation setting platforms: [${settings.platforms.join(', ')}]`,
4343
)
4444
process = false
@@ -47,27 +47,27 @@ export const shouldProcessActivity = (activityData, automation: AutomationData):
4747

4848
// check whether activity content contains any of the keywords
4949
if (process && settings.keywords && settings.keywords.length > 0) {
50-
const body = (activityData.body as string).toLowerCase()
50+
const body = (activity.body as string).toLowerCase()
5151
if (!settings.keywords.some((keyword) => body.includes(keyword.trim().toLowerCase()))) {
5252
console.log(
5353
`Ignoring automation ${automation.id} - Activity ${
54-
activityData.id
54+
activity.id
5555
} content does not match automation setting keywords: [${settings.keywords.join(', ')}]`,
5656
)
5757
process = false
5858
}
5959
}
6060

61-
if (process && !settings.teamMemberActivities) {
62-
if (
63-
activityData.member.attributes.isTeamMember &&
64-
activityData.member.attributes.isTeamMember.custom
65-
) {
66-
console.log(
67-
`Ignoring automation ${automation.id} - Activity ${activityData.id} belongs to a team member!`,
68-
)
69-
process = false
70-
}
61+
if (
62+
process &&
63+
!settings.teamMemberActivities &&
64+
activity.member.attributes.isTeamMember &&
65+
activity.member.attributes.isTeamMember.custom
66+
) {
67+
console.log(
68+
`Ignoring automation ${automation.id} - Activity ${activity.id} belongs to a team member!`,
69+
)
70+
process = false
7171
}
7272

7373
return process
@@ -102,8 +102,9 @@ export const prepareActivityPayload = (activity: any): any => {
102102
*
103103
* @param tenantId tenant unique ID
104104
* @param activityId activity unique ID
105+
* @param activityData activity data
105106
*/
106-
export default async (tenantId: string, activityId: string): Promise<void> => {
107+
export default async (tenantId: string, activityId?: string, activityData?: any): Promise<void> => {
107108
// console.log(`New activity automation trigger detected with activity id: ${activityId}!`)
108109

109110
const userContext = await getUserContext(tenantId)
@@ -117,19 +118,25 @@ export default async (tenantId: string, activityId: string): Promise<void> => {
117118

118119
if (automations.length > 0) {
119120
console.log(`Found ${automations.length} automations to process!`)
120-
const activityData = await ActivityRepository.findById(activityId, userContext)
121+
let activity: any | undefined = activityData
122+
123+
if (activity === undefined) {
124+
activity = await ActivityRepository.findById(activityId, userContext)
125+
}
121126

122127
for (const automation of automations) {
123-
if (shouldProcessActivity(activityData, automation)) {
124-
console.log(`Activity ${activityId} is being processed by automation ${automation.id}!`)
128+
if (shouldProcessActivity(activity, automation)) {
129+
console.log(
130+
`Activity ${activity.activityId} is being processed by automation ${automation.id}!`,
131+
)
125132

126133
switch (automation.type) {
127134
case AutomationType.WEBHOOK:
128135
await sendWebhookProcessRequest(
129136
tenantId,
130-
automation.id,
131-
activityData.id,
132-
prepareActivityPayload(activityData),
137+
automation,
138+
activity.id,
139+
prepareActivityPayload(activity),
133140
)
134141
break
135142
default:

‎backend/src/serverless/microservices/nodejs/automation/workers/newMemberWorker.ts‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,14 @@ import {
99
} from '../../../../../types/automationTypes'
1010
import MemberRepository from '../../../../../database/repositories/memberRepository'
1111
import { sendWebhookProcessRequest } from './util'
12+
import { MemberAutomationData } from '../../messageTypes'
1213

1314
/**
1415
* Helper function to check whether a single member should be processed by automation
1516
* @param member Member data
1617
* @param automation {AutomationData} Automation data
1718
*/
18-
export const shouldProcessMember = (member, automation: AutomationData): boolean => {
19+
export const shouldProcessMember = (member: any, automation: AutomationData): boolean => {
1920
const settings = automation.settings as NewMemberSettings
2021

2122
let process = true
@@ -57,15 +58,19 @@ export const prepareMemberPayload = (member: any): any => {
5758

5859
return copy
5960
}
60-
6161
/**
6262
* Check whether this member matches any automations for tenant.
6363
* If so emit automation process messages to NodeJS microservices SQS queue.
6464
*
6565
* @param tenantId tenant unique ID
66-
* @param memberId community member unique ID
66+
* @param memberId tenant member ID
67+
* @param memberData community member data
6768
*/
68-
export default async (tenantId: string, memberId: string): Promise<void> => {
69+
export default async (
70+
tenantId: string,
71+
memberId?: string,
72+
memberData?: MemberAutomationData,
73+
): Promise<void> => {
6974
// console.log(`New member automation trigger detected with member id: ${memberId}!`)
7075

7176
const userContext = await getUserContext(tenantId)
@@ -79,17 +84,21 @@ export default async (tenantId: string, memberId: string): Promise<void> => {
7984

8085
if (automations.length > 0) {
8186
console.log(`Found ${automations.length} automations to process!`)
82-
const member = await MemberRepository.findById(memberId, userContext, true, false)
87+
88+
let member: any | undefined = memberData
89+
if (member === undefined) {
90+
member = await MemberRepository.findById(memberId, userContext)
91+
}
8392

8493
for (const automation of automations) {
8594
if (shouldProcessMember(member, automation)) {
86-
console.log(`Member ${memberId} is being processed by automation ${automation.id}!`)
95+
console.log(`Member ${member.id} is being processed by automation ${automation.id}!`)
8796

8897
switch (automation.type) {
8998
case AutomationType.WEBHOOK:
9099
await sendWebhookProcessRequest(
91100
tenantId,
92-
automation.id,
101+
automation,
93102
member.id,
94103
prepareMemberPayload(member),
95104
)

0 commit comments

Comments
 (0)