Added the Git integration to segments, and tweaks for the GitHub integration (#994)

Author: Joan Reyero

backend/package-lock.json

@@ -81,6 +81,7 @@
     "cors": "2.8.5",
     "cron": "^2.1.0",
     "cron-time-generator": "^1.3.0",
+    "crowd-sentiment": "^1.1.7",
     "crypto-js": "^4.1.1",
     "discord.js": "^14.7.1",
     "dotenv": "8.2.0",

backend/package.json

@@ -155,6 +155,25 @@ class IntegrationRepository {
     await this._createAuditLog(AuditLogRepository.DELETE, record, record, options)
   }
 
+  static async findAllByPlatform(platform, options: IRepositoryOptions) {
+    const transaction = SequelizeRepository.getTransaction(options)
+
+    const include = []
+
+    const currentTenant = SequelizeRepository.getCurrentTenant(options)
+
+    const records = await options.database.integration.findAll({
+      where: {
+        platform,
+        tenantId: currentTenant.id,
+      },
+      include,
+      transaction,
+    })
+
+    return records.map((record) => record.get({ plain: true }))
+  }
+
   static async findByPlatform(platform, options: IRepositoryOptions) {
     const transaction = SequelizeRepository.getTransaction(options)
 

backend/src/database/repositories/integrationRepository.ts

@@ -56,7 +56,6 @@ class MemberRepository {
     const transaction = SequelizeRepository.getTransaction(options)
 
     const segment = SequelizeRepository.getStrictlySingleActiveSegment(options)
-
     const record = await options.database.member.create(
       {
         ...lodash.pick(data, [

backend/src/database/repositories/memberRepository.ts

@@ -66,7 +66,6 @@ export default class SequelizeRepository {
         `This operation can have exactly one segment. Found ${options.currentSegments.length} segments.`,
       )
     }
-
     return options.currentSegments[0]
   }
 

backend/src/database/repositories/sequelizeRepository.ts

@@ -2,6 +2,7 @@ import { getServiceChildLogger } from '@crowd/logging'
 import { KUBE_MODE } from '../../conf/index'
 import bulkOperations from './operationsWorker'
 import getUserContext from '../../database/utils/getUserContext'
+import SegmentRepository from '../../database/repositories/segmentRepository'
 
 const log = getServiceChildLogger('dbOperations.handler')
 
@@ -22,6 +23,8 @@ export async function consumer(event) {
   }
 
   const context = await getUserContext(tenantId)
+  const segmentRepository = new SegmentRepository(context)
+  context.currentSegments = [await segmentRepository.findById(event.segments[0])]
 
   const result = await bulkOperations(event.operation, event.records, context)
 

backend/src/serverless/dbOperations/handler.ts

@@ -233,7 +233,9 @@ export class GithubIntegrationService extends IntegrationServiceBase {
           }))
         }
 
-        newStreams = [...prCommentStreams, ...prReviewThreads, ...prCommitsStreams]
+        // It is very important to keep commits first. Otherwise, we have problems
+        // creating conversations if the Git integration has already ran for those data points.
+        newStreams = [...prCommitsStreams, ...prCommentStreams, ...prReviewThreads]
         break
       case GithubStreamType.PULL_COMMENTS: {
         const pullRequestNumber = stream.metadata.prNumber

backend/src/serverless/integrations/services/integrations/githubIntegrationService.ts

@@ -554,6 +554,57 @@ describe('ActivityService tests', () => {
       expect(activityCreated2.body).toBe(activity2.body)
     })
 
+    it('Should keep isMainBranch as true', async () => {
+      const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db)
+      await populateSegments(mockIRepositoryOptions)
+      const memberService = new MemberService(mockIRepositoryOptions)
+      const activityService = new ActivityService(mockIRepositoryOptions)
+
+      const cm = await memberService.upsert({
+        username: {
+          [PlatformType.DISCORD]: 'test',
+        },
+        platform: PlatformType.DISCORD,
+      })
+
+      const activity1 = {
+        type: 'message',
+        timestamp: '2020-05-27T15:13:30Z',
+        username: 'test',
+        member: cm.id,
+        platform: PlatformType.DISCORD,
+        sourceId: 'sourceId#1',
+        attributes: {
+          isMainBranch: true,
+          other: 'other',
+        },
+      }
+
+      await activityService.upsert(activity1)
+
+      const activity2 = {
+        type: 'message',
+        timestamp: '2022-05-27T15:13:30Z',
+        username: 'test',
+        member: cm.id,
+        platform: PlatformType.DISCORD,
+        sourceId: 'sourceId#1',
+        body: 'What is love?',
+        attributes: {
+          isMainBranch: false,
+          other2: 'other2',
+        },
+      }
+
+      const activityCreated2 = await activityService.upsert(activity2)
+
+      expect(activityCreated2.attributes).toStrictEqual({
+        isMainBranch: true,
+        other: 'other',
+        other2: 'other2',
+      })
+    })
+
     it('Should create various conversations successfully with given parent-child relationships of activities [descending timestamp order]', async () => {
       const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db)
       await populateSegments(mockIRepositoryOptions)

backend/src/services/__tests__/activityService.test.ts

@@ -1,8 +1,9 @@
 import { LoggerBase, logExecutionTime } from '@crowd/logging'
 import { Blob } from 'buffer'
+import vader from 'crowd-sentiment'
 import { Transaction } from 'sequelize/types'
 import { PlatformType } from '@crowd/types'
-import { IS_DEV_ENV, IS_TEST_ENV } from '../conf'
+import { IS_DEV_ENV, IS_TEST_ENV, GITHUB_CONFIG } from '../conf'
 import ActivityRepository from '../database/repositories/activityRepository'
 import MemberAttributeSettingsRepository from '../database/repositories/memberAttributeSettingsRepository'
 import MemberRepository from '../database/repositories/memberRepository'
@@ -20,6 +21,8 @@ import MemberService from './memberService'
 import SegmentRepository from '../database/repositories/segmentRepository'
 import SegmentService from './segmentService'
 
+const IS_GITHUB_COMMIT_DATA_ENABLED = GITHUB_CONFIG.isCommitDataEnabled === 'true'
+
 export default class ActivityService extends LoggerBase {
   options: IServiceOptions
 
@@ -94,10 +97,24 @@ export default class ActivityService extends LoggerBase {
         const toUpdate = merge(existing, data, {
           // eslint-disable-next-line @typescript-eslint/no-unused-vars
           timestamp: (oldValue, _newValue) => oldValue,
+          attributes: (oldValue, newValue) => {
+            if (oldValue && newValue) {
+              const out = { ...oldValue, ...newValue }
+              // If either of the two has isMainBranch set to true, then set it to true
+              if (oldValue.isMainBranch || newValue.isMainBranch) {
+                out.isMainBranch = true
+              }
+              return out
+            }
+            return newValue
+          },
           // eslint-disable-next-line @typescript-eslint/no-unused-vars
           organizationId: (oldValue, _newValue) => oldValue,
         })
         record = await ActivityRepository.update(id, toUpdate, repositoryOptions)
+        if (data.parent) {
+          await this.addToConversation(record.id, data.parent, transaction)
+        }
       } else {
         if (!data.sentiment) {
           const sentiment = await this.getSentiment(data)
@@ -226,6 +243,31 @@ export default class ActivityService extends LoggerBase {
       }
     }
 
+    // When we implement Kern.ais's sentiment, we will get rid of this. In the meantime, we use Vader
+    // because we don't have an agreement with LF for comprehend.
+    if (IS_GITHUB_COMMIT_DATA_ENABLED) {
+      const text = data.sourceParentId ? data.body : `${data.title} ${data.body}`
+      const sentiment = vader.SentimentIntensityAnalyzer.polarity_scores(text)
+      const compound = Math.round(((sentiment.compound + 1) / 2) * 100)
+      // Some activities are inherently different, we might want to dampen their sentiment
+
+      let label = 'neutral'
+      if (compound < 33) {
+        label = 'negative'
+      } else if (compound > 66) {
+        label = 'positive'
+      }
+
+      return {
+        positive: Math.round(sentiment.pos * 100),
+        negative: Math.round(sentiment.neg * 100),
+        neutral: Math.round(sentiment.neu * 100),
+        mixed: Math.round(sentiment.neu * 100),
+        sentiment: compound,
+        label,
+      }
+    }
+
     try {
       data.body = data.body ?? ''
       data.title = data.title ?? ''

backend/src/services/activityService.ts

@@ -91,6 +91,10 @@ export default class IntegrationService {
     return IntegrationRepository.findByPlatform(platform, this.options)
   }
 
+  async findAllByPlatform(platform) {
+    return IntegrationRepository.findAllByPlatform(platform, this.options)
+  }
+
   async create(data, transaction?: any) {
     try {
       const record = await IntegrationRepository.create(data, {
@@ -291,20 +295,21 @@ export default class IntegrationService {
 
       const repos = await getInstalledRepositories(installToken)
 
-      // If the git integration is configured, we add the repos to the git config
-      let isGitintegrationConfigured
-      try {
-        await this.findByPlatform(PlatformType.GIT)
-        isGitintegrationConfigured = true
-      } catch (err) {
-        isGitintegrationConfigured = false
-      }
-      if (isGitintegrationConfigured) {
-        const gitRemotes = await this.gitGetRemotes()
-        await this.gitConnectOrUpdate({
-          remotes: [...gitRemotes.default, ...repos.map((repo) => repo.cloneUrl)],
-        })
-      }
+      // TODO: I will do this later. For now they can add it manually.
+      // // If the git integration is configured, we add the repos to the git config
+      // let isGitintegrationConfigured
+      // try {
+      //   await this.findByPlatform(PlatformType.GIT)
+      //   isGitintegrationConfigured = true
+      // } catch (err) {
+      //   isGitintegrationConfigured = false
+      // }
+      // if (isGitintegrationConfigured) {
+      //   const gitRemotes = await this.gitGetRemotes()
+      //   await this.gitConnectOrUpdate({
+      //     remotes: [...gitRemotes, ...repos.map((repo) => repo.cloneUrl)],
+      //   })
+      // }
 
       integration = await this.createOrUpdate(
         {
@@ -629,11 +634,15 @@ export default class IntegrationService {
    */
   async gitGetRemotes() {
     try {
-      const integration = await this.findByPlatform(PlatformType.GIT)
-      return {
-        // We are returning this until we have segments
-        default: integration.settings.remotes,
-      }
+      const integrations = await this.findAllByPlatform(PlatformType.GIT)
+      return integrations.reduce((acc, integration) => {
+        const {
+          segmentId,
+          settings: { remotes },
+        } = integration
+        acc[segmentId] = remotes
+        return acc
+      }, {})
     } catch (err) {
       throw new Error400(this.options.language, 'errors.git.noIntegration')
     }