Refactor Slack Alerting and support it in new integration framework

backend/package.json

@@ -52,6 +52,7 @@
     "@crowd/integrations": "file:../services/libs/integrations",
     "@crowd/types": "file:../services/libs/types",
     "@crowd/sqs": "file:../services/libs/sqs",
+    "@crowd/alerting": "file:../services/libs/alerting",
     "@cubejs-client/core": "^0.30.4",
     "@google-cloud/storage": "5.3.0",
     "@octokit/auth-app": "^3.6.1",

backend/src/serverless/integrations/services/integrationRunProcessor.ts

@@ -3,6 +3,7 @@ import { ApiPubSubEmitter } from '@crowd/redis'
 import { Logger, getChildLogger, LoggerBase } from '@crowd/logging'
 import { singleOrDefault } from '@crowd/common'
 import { PlatformType } from '@crowd/types'
+import { sendSlackAlert, SlackAlertTypes } from '@crowd/alerting'
 import IntegrationRepository from '../../../database/repositories/integrationRepository'
 import IntegrationRunRepository from '../../../database/repositories/integrationRunRepository'
 import IntegrationStreamRepository from '../../../database/repositories/integrationStreamRepository'
@@ -28,8 +29,7 @@ import bulkOperations from '../../dbOperations/operationsWorker'
 import UserRepository from '../../../database/repositories/userRepository'
 import EmailSender from '../../../services/emailSender'
 import { i18n } from '../../../i18n'
-import { API_CONFIG } from '../../../conf'
-import { SlackAlertTypes, sendSlackAlert } from '../../../utils/slackAlerts'
+import { API_CONFIG, SLACK_ALERTING_CONFIG } from '../../../conf'
 
 export class IntegrationRunProcessor extends LoggerBase {
   constructor(
@@ -526,7 +526,14 @@ export class IntegrationRunProcessor extends LoggerBase {
           logger.error('Integration ended but we are still processing!')
         }
       } else if (newState === IntegrationRunState.ERROR) {
-        await sendSlackAlert(SlackAlertTypes.INTEGRATION_ERROR, integration, userContext, logger)
+        await sendSlackAlert({
+          slackURL: SLACK_ALERTING_CONFIG.url,
+          alertType: SlackAlertTypes.INTEGRATION_ERROR,
+          integration,
+          userContext,
+          log: logger,
+          frameworkVersion: 'old',
+        })
       }
 
       if (run.onboarding && this.apiPubSubEmitter) {

backend/src/serverless/microservices/nodejs/integration-data-checker/integrationDataCheckerWorker.ts

@@ -1,5 +1,6 @@
 import moment from 'moment'
 import { getServiceChildLogger } from '@crowd/logging'
+import { sendSlackAlert, SlackAlertTypes } from '@crowd/alerting'
 import getUserContext from '../../../../database/utils/getUserContext'
 import IntegrationService from '../../../../services/integrationService'
 import ActivityService from '../../../../services/activityService'
@@ -9,7 +10,7 @@ import {
 } from './integrationDataCheckerSettings'
 import { IRepositoryOptions } from '../../../../database/repositories/IRepositoryOptions'
 import { IntegrationDataCheckerSettings } from './integrationDataCheckerTypes'
-import { sendSlackAlert, SlackAlertTypes } from '../../../../utils/slackAlerts'
+import { SLACK_ALERTING_CONFIG } from '@/conf'
 
 const log = getServiceChildLogger('integrationDataCheckerWorker')
 
@@ -102,7 +103,15 @@ async function sendSlackAlertAction(
   integration,
   userContext: IRepositoryOptions,
 ) {
-  return sendSlackAlert(SlackAlertTypes.DATA_CHECKER, integration, userContext, log, settings)
+  return sendSlackAlert({
+    slackURL: SLACK_ALERTING_CONFIG.url,
+    alertType: SlackAlertTypes.DATA_CHECKER,
+    integration,
+    userContext,
+    log,
+    frameworkVersion: 'old',
+    settings,
+  })
 }
 
 function generateDate(timeframe) {

services/apps/integration_run_worker/config/custom-environment-variables.json

@@ -22,5 +22,8 @@
   "nango": {
     "url": "CROWD_NANGO_URL",
     "secretKey": "CROWD_NANGO_SECRET_KEY"
+  },
+  "slackAlerting": {
+    "url": "CROWD_SLACK_ALERTING_URL"
   }
 }

services/apps/integration_run_worker/package.json

@@ -23,6 +23,7 @@
     "@crowd/redis": "file:../../libs/redis",
     "@crowd/sqs": "file:../../libs/sqs",
     "@crowd/types": "file:../../libs/types",
+    "@crowd/alerting": "file:../../libs/alerting",
     "@types/config": "^3.3.0",
     "@types/node": "^18.16.3",
     "config": "^3.3.9",

services/apps/integration_run_worker/src/conf/index.ts

@@ -8,6 +8,10 @@ export interface INangoConfig {
   secretKey: string
 }
 
+export interface ISlackAlertingConfig {
+  url: string
+}
+
 let sqsConfig: ISqsClientConfig
 export const SQS_CONFIG = (): ISqsClientConfig => {
   if (sqsConfig) return sqsConfig
@@ -52,6 +56,14 @@ export const NANGO_CONFIG = (): INangoConfig => {
   return nangoConfig
 }
 
+let slackAlertingConfig: ISlackAlertingConfig
+export const SLACK_ALERTING_CONFIG = (): ISlackAlertingConfig => {
+  if (slackAlertingConfig) return slackAlertingConfig
+
+  slackAlertingConfig = config.get<ISlackAlertingConfig>('slackAlerting')
+  return slackAlertingConfig
+}
+
 const platformMap: Map<string, unknown | null> = new Map()
 export const PLATFORM_CONFIG = (platform: string): unknown | undefined => {
   if (platformMap.has(platform)) {

services/apps/integration_run_worker/src/repo/integrationRun.data.ts

@@ -10,6 +10,9 @@ export interface IGenerateStreamsData {
   runId: string
   tenantId: string
   hasSampleData: boolean
+  plan: string
+  isTrialPlan: boolean
+  name: string
   integrationSettings: unknown
   streamCount: number
 }

services/apps/integration_run_worker/src/repo/integrationRun.repo.ts

@@ -102,6 +102,9 @@ export default class IntegrationRunRepository extends RepositoryBase<Integration
           r."tenantId",
           r.onboarding,
           t."hasSampleData",
+          t."plan",
+          t."isTrialPlan",
+          t."name",
           i.platform                  as "integrationType", 
           i.status                    as "integrationState",
           r.state                     as "runState",

services/apps/integration_run_worker/src/service/integrationRunService.ts

@@ -9,6 +9,8 @@ import { NANGO_CONFIG, PLATFORM_CONFIG } from '../conf'
 import IntegrationRunRepository from '../repo/integrationRun.repo'
 import MemberAttributeSettingsRepository from '../repo/memberAttributeSettings.repo'
 import SampleDataRepository from '../repo/sampleData.repo'
+import { SlackAlertTypes, sendSlackAlert } from '@crowd/alerting'
+import { SLACK_ALERTING_CONFIG } from '../conf'
 
 export default class IntegrationRunService extends LoggerBase {
   private readonly repo: IntegrationRunRepository
@@ -70,6 +72,25 @@ export default class IntegrationRunService extends LoggerBase {
             message: 'Some streams failed!',
           })
 
+          await sendSlackAlert({
+            slackURL: SLACK_ALERTING_CONFIG().url,
+            alertType: SlackAlertTypes.INTEGRATION_ERROR,
+            integration: {
+              id: runInfo.integrationId,
+              platform: runInfo.integrationType,
+              tenantId: runInfo.tenantId,
+            },
+            userContext: {
+              currentTenant: {
+                name: runInfo.name,
+                plan: runInfo.plan,
+                isTrial: runInfo.isTrialPlan,
+              },
+            },
+            log: this.log,
+            frameworkVersion: 'new',
+          })
+
           if (runInfo.onboarding) {
             this.log.warn('Onboarding - marking integration as failed!')
             await this.repo.markIntegration(runId, 'error')

services/libs/alerting/.eslintrc.cjs

@@ -0,0 +1,22 @@
+module.exports = {
+    parser: '@typescript-eslint/parser',
+    extends: [
+      'eslint:recommended',
+      'plugin:@typescript-eslint/recommended',
+      'plugin:prettier/recommended',
+    ],
+    plugins: ['@typescript-eslint', 'prettier'],
+    parserOptions: {
+      ecmaVersion: 2022,
+      sourceType: 'module',
+    },
+    env: {
+      es6: true,
+      node: true,
+    },
+    rules: {
+      'prettier/prettier': 'error',
+      '@typescript-eslint/explicit-module-boundary-types': 'off',
+    },
+  };
+

services/libs/alerting/.prettierignore

@@ -0,0 +1,3 @@
+dist/
+.eslintrc.cjs
+.prettierrc.cjs

services/libs/alerting/.prettierrc.cjs

@@ -0,0 +1,8 @@
+module.exports = {
+    singleQuote: true,
+    arrowParens: 'always',
+    printWidth: 100,
+    trailingComma: 'all',
+    semi: false,
+  };
+