diff --git a/backend/package-lock.json b/backend/package-lock.json index acbc06f49f..08ede12419 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -1598,6 +1598,8 @@ "name": "@crowd/common", "version": "1.0.0", "dependencies": { + "@crowd/logging": "file:../logging", + "@crowd/types": "file:../types", "uuid": "^9.0.0" }, "devDependencies": { @@ -3131,9 +3133,12 @@ "@crowd/common": "file:../common", "@crowd/logging": "file:../logging", "@crowd/types": "file:../types", + "@octokit/auth-app": "^3.6.1", + "@octokit/graphql": "^4.8.0", "axios": "^1.4.0", "he": "^1.2.0", - "sanitize-html": "^2.10.0" + "sanitize-html": "^2.10.0", + "verify-github-webhook": "^1.0.1" }, "devDependencies": { "@types/he": "^1.2.0", @@ -36067,6 +36072,8 @@ "@crowd/common": { "version": "file:../services/libs/common", "requires": { + "@crowd/logging": "file:../logging", + "@crowd/types": "file:../types", "@types/node": "^18.16.3", "@typescript-eslint/eslint-plugin": "^5.59.2", "@typescript-eslint/parser": "^5.59.2", @@ -36954,6 +36961,8 @@ "@crowd/common": "file:../common", "@crowd/logging": "file:../logging", "@crowd/types": "file:../types", + "@octokit/auth-app": "^3.6.1", + "@octokit/graphql": "^4.8.0", "@types/he": "^1.2.0", "@types/node": "^18.16.3", "@types/sanitize-html": "^2.9.0", @@ -36966,12 +36975,15 @@ "he": "^1.2.0", "prettier": "^2.8.8", "sanitize-html": "^2.10.0", - "typescript": "^5.0.4" + "typescript": "^5.0.4", + "verify-github-webhook": "^1.0.1" }, "dependencies": { "@crowd/common": { "version": "file:../services/libs/common", "requires": { + "@crowd/logging": "file:../logging", + "@crowd/types": "file:../types", "@types/node": "^18.16.3", "@typescript-eslint/eslint-plugin": "^5.59.2", "@typescript-eslint/parser": "^5.59.2", @@ -37874,6 +37886,8 @@ "@crowd/common": { "version": "file:../services/libs/common", "requires": { + "@crowd/logging": "file:../logging", + "@crowd/types": "file:../types", "@types/node": "^18.16.3", "@typescript-eslint/eslint-plugin": "^5.59.2", "@typescript-eslint/parser": "^5.59.2", @@ -41653,6 +41667,8 @@ "@crowd/common": { "version": "file:../services/libs/common", "requires": { + "@crowd/logging": "file:../logging", + "@crowd/types": "file:../types", "@types/node": "^18.16.3", "@typescript-eslint/eslint-plugin": "^5.59.2", "@typescript-eslint/parser": "^5.59.2", @@ -45337,6 +45353,8 @@ "@crowd/common": { "version": "file:../services/libs/common", "requires": { + "@crowd/logging": "file:../logging", + "@crowd/types": "file:../types", "@types/node": "^18.16.3", "@typescript-eslint/eslint-plugin": "^5.59.2", "@typescript-eslint/parser": "^5.59.2", @@ -46239,6 +46257,8 @@ "@crowd/common": { "version": "file:../services/libs/common", "requires": { + "@crowd/logging": "file:../logging", + "@crowd/types": "file:../types", "@types/node": "^18.16.3", "@typescript-eslint/eslint-plugin": "^5.59.2", "@typescript-eslint/parser": "^5.59.2", @@ -50599,6 +50619,8 @@ "@crowd/common": { "version": "file:../services/libs/common", "requires": { + "@crowd/logging": "file:../logging", + "@crowd/types": "file:../types", "@types/node": "^18.16.3", "@typescript-eslint/eslint-plugin": "^5.59.2", "@typescript-eslint/parser": "^5.59.2", @@ -51501,6 +51523,8 @@ "@crowd/common": { "version": "file:../services/libs/common", "requires": { + "@crowd/logging": "file:../logging", + "@crowd/types": "file:../types", "@types/node": "^18.16.3", "@typescript-eslint/eslint-plugin": "^5.59.2", "@typescript-eslint/parser": "^5.59.2", diff --git a/backend/src/database/migrations/U1692866303__segmentActivityChannels.sql b/backend/src/database/migrations/U1692866303__segmentActivityChannels.sql new file mode 100644 index 0000000000..8bfa332aad --- /dev/null +++ b/backend/src/database/migrations/U1692866303__segmentActivityChannels.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS "public"."segmentActivityChannels" CASCADE; \ No newline at end of file diff --git a/backend/src/database/migrations/V1692866303__segmentActivityChannels.sql b/backend/src/database/migrations/V1692866303__segmentActivityChannels.sql new file mode 100644 index 0000000000..3402b594c9 --- /dev/null +++ b/backend/src/database/migrations/V1692866303__segmentActivityChannels.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS "public"."segmentActivityChannels" ( + "id" UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + "tenantId" UUID NOT NULL REFERENCES public.tenants(id) ON UPDATE CASCADE ON DELETE CASCADE, + "segmentId" UUID NOT NULL REFERENCES public.segments(id) ON UPDATE CASCADE ON DELETE CASCADE, + "platform" TEXT NOT NULL, + "channel" TEXT NOT NULL +); + +CREATE UNIQUE INDEX unique_segment_activity_channel + ON "public"."segmentActivityChannels" ("tenantId", "segmentId", "platform", "channel"); + +INSERT INTO "public"."segmentActivityChannels" ("segmentId", "tenantId", "platform", "channel") + SELECT + "id", + "tenantId", + jsonb_object_keys("activityChannels"), + TRIM('"' FROM jsonb_array_elements("activityChannels"->jsonb_object_keys("activityChannels"))::TEXT) + FROM "public"."segments"; diff --git a/backend/src/database/repositories/segmentRepository.ts b/backend/src/database/repositories/segmentRepository.ts index 66f654ecd9..cc50bba250 100644 --- a/backend/src/database/repositories/segmentRepository.ts +++ b/backend/src/database/repositories/segmentRepository.ts @@ -182,39 +182,71 @@ class SegmentRepository extends RepositoryBase< 'sourceId', 'sourceParentId', 'customActivityTypes', - 'activityChannels', ].includes(key), ) - let segmentUpdateQuery = `UPDATE segments SET ` - const replacements = {} as any + if (updateFields.length > 0) { + let segmentUpdateQuery = `UPDATE segments SET ` + const replacements = {} as any - for (const field of updateFields) { - segmentUpdateQuery += ` "${field}" = :${field} ` - replacements[field] = data[field] + for (const field of updateFields) { + segmentUpdateQuery += ` "${field}" = :${field} ` + replacements[field] = data[field] - if (updateFields[updateFields.length - 1] !== field) { - segmentUpdateQuery += ', ' + if (updateFields[updateFields.length - 1] !== field) { + segmentUpdateQuery += ', ' + } } - } - segmentUpdateQuery += ` WHERE id = :id and "tenantId" = :tenantId ` - replacements.tenantId = this.options.currentTenant.id - replacements.id = id + segmentUpdateQuery += ` WHERE id = :id and "tenantId" = :tenantId ` + replacements.tenantId = this.options.currentTenant.id + replacements.id = id - if (replacements.customActivityTypes) { - replacements.customActivityTypes = JSON.stringify(replacements.customActivityTypes) - } + if (replacements.customActivityTypes) { + replacements.customActivityTypes = JSON.stringify(replacements.customActivityTypes) + } - if (replacements.activityChannels) { - replacements.activityChannels = JSON.stringify(replacements.activityChannels) + await this.options.database.sequelize.query(segmentUpdateQuery, { + replacements, + type: QueryTypes.UPDATE, + transaction, + }) } - await this.options.database.sequelize.query(segmentUpdateQuery, { - replacements, - type: QueryTypes.UPDATE, - transaction, - }) + if (data.activityChannels && typeof data.activityChannels === 'object') { + if (Object.keys(data.activityChannels).length > 0) { + const replacements = {} + let valuePlaceholders = '' + Object.keys(data.activityChannels).forEach((platform) => { + data.activityChannels[platform].forEach((channel, i) => { + valuePlaceholders += data.activityChannels[platform] + .map( + () => + `(:tenantId_${platform}_${i}, :segmentId_${platform}_${i}, :platform_${platform}_${i}, :channel_${platform}_${i})`, + ) + .join(', ') + + replacements[`tenantId_${platform}_${i}`] = this.options.currentTenant.id + replacements[`segmentId_${platform}_${i}`] = id + replacements[`platform_${platform}_${i}`] = platform + replacements[`channel_${platform}_${i}`] = channel + }) + }) + + await this.options.database.sequelize.query( + ` + INSERT INTO "segmentActivityChannels" ("tenantId", "segmentId", "platform", "channel") + VALUES ${valuePlaceholders} + ON CONFLICT DO NOTHING; + `, + { + replacements, + type: QueryTypes.INSERT, + transaction, + }, + ) + } + } return this.findById(id) } @@ -298,11 +330,22 @@ class SegmentRepository extends RepositoryBase< const transaction = this.transaction const records = await this.options.database.sequelize.query( - `SELECT * - FROM segments - WHERE id in (:ids) - and "tenantId" = :tenantId; - `, + `SELECT + s.*, + json_agg(sac."activityChannels") AS "activityChannels" + FROM segments s + LEFT JOIN ( + SELECT + "tenantId", + json_build_object(concat(platform), json_agg(sac.channel)) AS "activityChannels" + FROM "segmentActivityChannels" sac + WHERE "tenantId" = :tenantId + GROUP BY "tenantId", "platform" + ) sac + ON sac."tenantId" = s."tenantId" + WHERE id in (:ids) + AND s."tenantId" = :tenantId + GROUP BY s.id;`, { replacements: { ids, @@ -313,6 +356,10 @@ class SegmentRepository extends RepositoryBase< }, ) + records.forEach((row) => { + row.activityChannels = Object.assign({}, ...row.activityChannels) + }) + return records.map((sr) => SegmentRepository.populateRelations(sr)) } @@ -333,11 +380,22 @@ class SegmentRepository extends RepositoryBase< const transaction = this.transaction const records = await this.options.database.sequelize.query( - `SELECT * - FROM segments - WHERE id = :id - and "tenantId" = :tenantId; - `, + `SELECT + s.*, + json_agg(sac."activityChannels") AS "activityChannels" + FROM segments s + LEFT JOIN ( + SELECT + "tenantId", + json_build_object(concat(platform), json_agg(sac.channel)) AS "activityChannels" + FROM "segmentActivityChannels" sac + WHERE "tenantId" = :tenantId + GROUP BY "tenantId", "platform" + ) sac + ON sac."tenantId" = s."tenantId" + WHERE s.id = :id + AND s."tenantId" = :tenantId + GROUP BY s.id;`, { replacements: { id, @@ -353,6 +411,7 @@ class SegmentRepository extends RepositoryBase< } const record = records[0] + record.activityChannels = Object.assign({}, ...record.activityChannels) if (SegmentRepository.isProjectGroup(record)) { // find projects @@ -569,17 +628,27 @@ class SegmentRepository extends RepositoryBase< const subprojects = await this.options.database.sequelize.query( ` - select s.*, - count(*) over () as "totalCount" - from segments s - where s."grandparentSlug" is not null - and s."parentSlug" is not null - and s."tenantId" = :tenantId - ${searchQuery} - GROUP BY s."id" - ORDER BY s."name" - ${this.getPaginationString(criteria)}; - `, + SELECT + COUNT(DISTINCT s.id) AS count, + s.*, + json_agg(sac."activityChannels") AS "activityChannels" + FROM segments s + LEFT JOIN ( + SELECT + "tenantId", + json_build_object(concat(platform), json_agg(sac.channel)) AS "activityChannels" + FROM "segmentActivityChannels" sac + WHERE "tenantId" = :tenantId + GROUP BY "tenantId", "platform" + ) sac + ON sac."tenantId" = s."tenantId" + WHERE s."grandparentSlug" IS NOT NULL + AND s."parentSlug" IS NOT NULL + AND s."tenantId" = :tenantId + ${searchQuery} + GROUP BY s.id + ${this.getPaginationString(criteria)}; + `, { replacements: { tenantId: this.currentTenant.id, @@ -592,15 +661,14 @@ class SegmentRepository extends RepositoryBase< }, ) - const count = subprojects.length > 0 ? Number.parseInt(subprojects[0].totalCount, 10) : 0 + const count = subprojects.length > 0 ? Number.parseInt(subprojects[0].count, 10) : 0 - const integrationsBySegments = await this.queryIntegrationsForSubprojects(subprojects) - - const rows = subprojects.map((i) => { - let subproject = removeFieldsFromObject(i, 'totalCount') - subproject = SegmentRepository.populateRelations(subproject) - subproject.integrations = integrationsBySegments[subproject.id] || [] - return subproject + const rows = subprojects + rows.forEach((row, i) => { + rows[i].activityChannels = rows[i].activityChannels[0] + if (rows[i].activityChannels === null) { + rows[i].activityChannels = {} + } }) // TODO: Add member count to segments after implementing member relations @@ -657,14 +725,17 @@ class SegmentRepository extends RepositoryBase< static getActivityChannels(options: IRepositoryOptions) { const channels = {} + for (const segment of options.currentSegments) { - for (const platform of Object.keys(segment.activityChannels)) { - if (!channels[platform]) { - channels[platform] = new Set(segment.activityChannels[platform]) - } else { - segment.activityChannels[platform].forEach((ch) => - (channels[platform] as Set).add(ch), - ) + if (segment.activityChannels) { + for (const platform of Object.keys(segment.activityChannels)) { + if (!channels[platform]) { + channels[platform] = new Set(segment.activityChannels[platform]) + } else { + segment.activityChannels[platform].forEach((ch) => + (channels[platform] as Set).add(ch), + ) + } } } } diff --git a/backend/src/services/__tests__/activityService.test.ts b/backend/src/services/__tests__/activityService.test.ts index ab08387ead..ca22d15482 100644 --- a/backend/src/services/__tests__/activityService.test.ts +++ b/backend/src/services/__tests__/activityService.test.ts @@ -864,9 +864,7 @@ describe('ActivityService tests', () => { } await new ActivityService(mockIRepositoryOptions).upsert(activity) - const activityChannels = SegmentRepository.getActivityChannels(mockIRepositoryOptions) - expect(activityChannels[activity.platform].includes(activity.channel)).toBe(true) }) @@ -932,6 +930,7 @@ describe('ActivityService tests', () => { member: memberCreated.id, score: 1, } + await new ActivityService(mockIRepositoryOptions).upsert(activity) const activityChannels = SegmentRepository.getActivityChannels(mockIRepositoryOptions) expect(activityChannels[activity1.platform].length).toBe(1) diff --git a/services/apps/data_sink_worker/src/service/settings.repo.ts b/services/apps/data_sink_worker/src/service/settings.repo.ts index a1264e77fb..f37d205958 100644 --- a/services/apps/data_sink_worker/src/service/settings.repo.ts +++ b/services/apps/data_sink_worker/src/service/settings.repo.ts @@ -62,64 +62,18 @@ export default class SettingsRepository extends RepositoryBase { - const existingData = await this.db().oneOrNone( - `select "activityChannels" from "segments" where "tenantId" = $(tenantId) and id = $(segmentId)`, + await this.db().result( + ` + INSERT INTO "segmentActivityChannels" ("tenantId", "segmentId", "platform", "channel") VALUES + ($(tenantId), $(segmentId), $(platform), $(channel)) + ON CONFLICT DO NOTHING; + `, { tenantId, segmentId, + platform, + channel, }, ) - - if (existingData) { - const channels = existingData.activityChannels - - if (channels && channels[platform] && channels[platform].includes(channel)) { - return - } else { - await this.db().result( - ` - update segments - set "activityChannels" = - case - -- If platform exists, and channel does not exist, add it - when "activityChannels" ? $(platform) - and not ($(channel) = any (select jsonb_array_elements_text("activityChannels" -> $(platform)))) then - jsonb_set( - "activityChannels", - array [$(platform)::text], - "activityChannels" -> $(platform) || jsonb_build_array($(channel)) - ) - -- If platform does not exist, create it - when not ("activityChannels" ? $(platform)) or "activityChannels" is null then - coalesce("activityChannels", '{}'::jsonb) || - jsonb_build_object($(platform), jsonb_build_array($(channel))) - -- Else, do nothing - else - "activityChannels" - end - where "tenantId" = $(tenantId) - and id = $(segmentId) - and case - -- If platform exists, and channel does not exist, add it - when "activityChannels" ? $(platform) - and not ($(channel) = any (select jsonb_array_elements_text("activityChannels" -> $(platform)))) then - 1 - -- If platform does not exist, create it - when not ("activityChannels" ? $(platform)) or "activityChannels" is null then - 1 - -- Else, do nothing - else - 0 - end = 1 - `, - { - tenantId, - segmentId, - platform, - channel, - }, - ) - } - } } }