Enhancement/separate service for webhooks c 927

.github/workflows/lf-production-deploy-new.yaml

@@ -11,6 +11,10 @@ on:
         description: Deploy integration-sync-worker service?
         required: true
         type: boolean
+      deploy_webhook_api:
+        description: Deploy webhook-api service?
+        required: true
+        type: boolean
 
 env:
   DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -66,6 +70,27 @@ jobs:
         id: image
         run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
 
+  build-and-push-webhook-api:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.deploy_webhook_api }}
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: webhook-api
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
   deploy-search-sync-worker:
     needs: build-and-push-search-sync-worker
     runs-on: ubuntu-latest
@@ -101,3 +126,21 @@ jobs:
           service: integration-sync-worker
           image: ${{ needs.build-and-push-integration-sync-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+
+  deploy-webhook-api:
+    needs: build-and-push-webhook-api
+    runs-on: ubuntu-latest
+    if: ${{ inputs.deploy_webhook_api }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: webhook-api
+          image: ${{ needs.build-and-push-webhook-api.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}

.github/workflows/lf-staging-deploy-webhook-api.yaml

@@ -0,0 +1,60 @@
+name: LF Staging Deploy Webhook API
+
+on:
+  push:
+    branches:
+      - 'lf-staging/**'
+      - 'lf-staging-**'
+    paths:
+      - 'services/libs/**'
+      - 'services/apps/webhook_api/**'
+
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+  CROWD_CLUSTER: ${{ secrets.LF_STAGING_CLUSTER_NAME }}
+  CROWD_ROLE_ARN: ${{ secrets.LF_STAGING_CLUSTER_ROLE_ARN }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.LF_AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.LF_AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: ${{ secrets.LF_AWS_REGION }}
+  SLACK_CHANNEL: deploys-lf-staging
+  SLACK_WEBHOOK: ${{ secrets.LF_STAGING_SLACK_CHANNEL_HOOK }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: webhook-api
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
+  deploy-webhook-api:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: webhook-api
+          image: ${{ needs.build-and-push.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}

.github/workflows/production-deploy-new.yaml

@@ -11,6 +11,10 @@ on:
         description: Deploy integration-sync-worker service?
         required: true
         type: boolean
+      deploy_webhook_api:
+        description: Deploy webhook-api service?
+        required: true
+        type: boolean
 
 env:
   DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -66,6 +70,27 @@ jobs:
         id: image
         run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
 
+  build-and-push-webhook-api:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.deploy_webhook_api }}
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: webhook-api
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
   deploy-search-sync-worker:
     needs: build-and-push-search-sync-worker
     runs-on: ubuntu-latest
@@ -101,3 +126,21 @@ jobs:
           service: integration-sync-worker
           image: ${{ needs.build-and-push-integration-sync-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+
+  deploy-webhook-api:
+    needs: build-and-push-webhook-api
+    runs-on: ubuntu-latest
+    if: ${{ inputs.deploy_webhook-api }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: webhook-api
+          image: ${{ needs.build-and-push-webhook-api.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}

.github/workflows/staging-deploy-webhook-api.yaml

@@ -0,0 +1,60 @@
+name: Staging Deploy Webhook API
+
+on:
+  push:
+    branches:
+      - 'staging/**'
+      - 'staging-**'
+    paths:
+      - 'services/libs/**'
+      - 'services/apps/webhook_api/**'
+
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+  CROWD_CLUSTER: ${{ secrets.STAGING_CLUSTER_NAME }}
+  CROWD_ROLE_ARN: ${{ secrets.STAGING_CLUSTER_ROLE_ARN }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: ${{ secrets.AWS_REGION }}
+  SLACK_CHANNEL: deploys-staging
+  SLACK_WEBHOOK: ${{ secrets.STAGING_SLACK_CHANNEL_HOOK }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: webhook-api
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
+  deploy-webhook-api:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: webhook-api
+          image: ${{ needs.build-and-push.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}

backend/src/api/webhooks/index.ts

@@ -1,7 +1,7 @@
 import { safeWrap } from '../../middlewares/errorMiddleware'
 
 export default (app) => {
-  app.post(`/github`, safeWrap(require('./github').default))
+  // app.post(`/github`, safeWrap(require('./github').default))
   app.post(`/stripe`, safeWrap(require('./stripe').default))
   app.post(`/sendgrid`, safeWrap(require('./sendgrid').default))
   app.post(`/discourse/:tenantId`, safeWrap(require('./discourse').default))

scripts/builders/webhook-api.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+DOCKERFILE="./services/docker/Dockerfile.webhook_api"
+CONTEXT="../"
+REPO="crowddotdev/webhook-api"

scripts/services/docker/Dockerfile.webhook_api

@@ -0,0 +1,30 @@
+FROM node:16-alpine as builder
+
+RUN apk add --update --no-cache bash
+
+WORKDIR /usr/crowd/app
+
+# build libraries
+COPY ./services/scripts ./services/scripts
+COPY ./services/libs ./services/libs
+RUN cd services/scripts && ./install_lib_packages.sh && ./build_libs.sh && ./clear_lib_packages.sh
+
+FROM node:16-alpine as release
+
+RUN apk add --update --no-cache bash
+
+WORKDIR /usr/crowd/app
+
+ARG INSTALL_DEV_DEPENDENCIES="false"
+
+COPY ./services/scripts ./services/scripts
+COPY --from=builder /usr/crowd/app/services/libs/ ./services/libs/
+RUN if [ "$INSTALL_DEV_DEPENDENCIES" = "false" ]; then export NPM_I_ENV_OPTS="--omit=dev"; fi && cd services/scripts && ./install_lib_packages.sh ${NPM_I_ENV_OPTS}
+
+COPY ./services/apps/webhook_api/package.json ./services/apps/webhook_api/package-lock.json ./services/apps/webhook_api/tsconfig.json  ./services/apps/webhook_api/
+COPY ./services/apps/webhook_api/config ./services/apps/webhook_api/config
+COPY ./services/apps/webhook_api/src ./services/apps/webhook_api/src
+
+RUN if [ "$INSTALL_DEV_DEPENDENCIES" = "false" ]; then export NPM_I_ENV_OPTS="--omit=dev"; fi && cd services/apps/webhook_api && npm ci ${NPM_I_ENV_OPTS}
+
+RUN npm cache clean --force

scripts/services/docker/Dockerfile.webhook_api.dockerignore

@@ -0,0 +1,22 @@
+**/.git
+**/node_modules
+**/venv*
+**/.webpack
+**/.serverless
+**/.cubestore
+**/.env
+**/.env.*
+**/.idea
+**/.vscode
+**/dist
+backend/server-config/
+backend/util/
+backend/src/serverless/microservices/python
+.vscode/
+.github/
+frontend/
+scripts/
+.flake8
+*.md
+Makefile
+backend/

scripts/services/webhook-api.yaml

@@ -0,0 +1,63 @@
+version: '3.1'
+
+x-env-args: &env-args
+  DOCKER_BUILDKIT: 1
+  NODE_ENV: docker
+  SERVICE: webhook-api
+  SHELL: /bin/sh
+
+services:
+  webhook-api:
+    build:
+      context: ../../
+      dockerfile: ./scripts/services/docker/Dockerfile.webhook_api
+    command: 'npm run start'
+    working_dir: /usr/crowd/app/services/apps/webhook_api
+    env_file:
+      - ../../backend/.env.dist.local
+      - ../../backend/.env.dist.composed
+      - ../../backend/.env.override.local
+      - ../../backend/.env.override.composed
+    environment:
+      <<: *env-args
+    ports:
+      - '8082:8082'
+    restart: always
+    networks:
+      - crowd-bridge
+
+  webhook-api-dev:
+    build:
+      context: ../../
+      dockerfile: ./scripts/services/docker/Dockerfile.webhook_api
+      args:
+        INSTALL_DEV_DEPENDENCIES: 'true'
+    command: 'npm run dev'
+    working_dir: /usr/crowd/app/services/apps/webhook_api
+    # user: '${USER_ID}:${GROUP_ID}'
+    env_file:
+      - ../../backend/.env.dist.local
+      - ../../backend/.env.dist.composed
+      - ../../backend/.env.override.local
+      - ../../backend/.env.override.composed
+    environment:
+      <<: *env-args
+    hostname: webhook-api
+    ports:
+      - '8082:8082'
+    networks:
+      - crowd-bridge
+    volumes:
+      - ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src
+      - ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src
+      - ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src
+      - ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src
+      - ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src
+      - ../../services/libs/sentiment/src:/usr/crowd/app/services/libs/sentiment/src
+      - ../../services/libs/sqs/src:/usr/crowd/app/services/libs/sqs/src
+      - ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src
+      - ../../services/apps/webhook_api/src:/usr/crowd/app/services/apps/webhook_api/src
+
+networks:
+  crowd-bridge:
+    external: true