Improve dependencies and PostgreSQL in archetypes

services/apps/template_consumer/package.json

@@ -15,6 +15,7 @@
   },
   "dependencies": {
     "@crowd/archetype-consumer": "file:../../archetypes/consumer",
+    "@crowd/archetype-standard": "file:../../archetypes/standard",
     "@types/node": "^20.8.2",
     "ts-node": "^10.9.1",
     "tsconfig-paths": "^4.2.0",

services/apps/template_consumer/src/main.ts

@@ -1,5 +1,5 @@
-import { Config } from '@crowd/standard'
-import { ServiceConsumer, Options } from '@crowd/consumer'
+import { Config } from '@crowd/archetype-standard'
+import { ServiceConsumer, Options } from '@crowd/archetype-consumer'
 
 const config: Config = {
   producer: {

services/apps/template_consumer/tsconfig.json

@@ -10,7 +10,8 @@
     "esModuleInterop": true,
     "baseUrl": "./src",
     "paths": {
-      "@crowd/*": ["../../../libs/*/src", "../../../archetypes/*/src"]
+      "@crowd/*": ["../../../libs/*/src"],
+      "@crowd/archetype-*": ["../../../archetypes/*/src"]
     }
   },
   "include": ["src/**/*"]

services/apps/template_worker/package.json

@@ -14,6 +14,7 @@
     "tsc-check": "./node_modules/.bin/tsc --noEmit"
   },
   "dependencies": {
+    "@crowd/archetype-standard": "file:../../archetypes/standard",
     "@crowd/archetype-worker": "file:../../archetypes/worker",
     "@temporalio/workflow": "~1.8.6",
     "@types/node": "^20.8.2",

services/apps/template_worker/src/main.ts

@@ -1,5 +1,5 @@
-import { Config } from '@crowd/standard'
-import { ServiceWorker, Options } from '@crowd/worker'
+import { Config } from '@crowd/archetype-standard'
+import { ServiceWorker, Options } from '@crowd/archetype-worker'
 
 const config: Config = {
   producer: {

services/apps/template_worker/tsconfig.json

@@ -10,7 +10,8 @@
     "esModuleInterop": true,
     "baseUrl": "./src",
     "paths": {
-      "@crowd/*": ["../../../libs/*/src", "../../../archetypes/*/src"]
+      "@crowd/*": ["../../../libs/*/src"],
+      "@crowd/archetype-*": ["../../../archetypes/*/src"]
     }
   },
   "include": ["src/**/*"]

services/archetypes/consumer/src/index.ts

@@ -1,6 +1,6 @@
 import { Consumer as KafkaConsumer } from 'kafkajs'
 
-import { Config, Service } from '@crowd/standard'
+import { Config, Service } from '@crowd/archetype-standard'
 
 // List all required environment variables, grouped per "component".
 // They are in addition to the ones required by the "standard" archetype.

services/archetypes/consumer/tsconfig.json

@@ -10,7 +10,8 @@
     "esModuleInterop": true,
     "baseUrl": "./src",
     "paths": {
-      "@crowd/*": ["../../../libs/*/src", "../../../archetypes/*/src"]
+      "@crowd/*": ["../../../libs/*/src"],
+      "@crowd/archetype-*": ["../../../archetypes/*/src"]
     }
   },
   "include": ["src/**/*"]

services/archetypes/standard/src/index.ts

@@ -13,16 +13,19 @@ const logger = getServiceLogger()
 
 // List all required environment variables, grouped per "component".
 const envvars = {
-  base: ['CROWD_SERVICE', 'CROWD_UNLEASH_URL', 'CROWD_UNLEASH_API_TOKEN'],
+  base: ['SERVICE', 'CROWD_UNLEASH_URL', 'CROWD_UNLEASH_BACKEND_API_KEY'],
   producer: ['CROWD_KAFKA_BROKERS'],
   temporal: ['CROWD_TEMPORAL_SERVER_URL', 'CROWD_TEMPORAL_NAMESPACE'],
-  redis: ['CROWD_REDIS_HOST', 'CROWD_REDIS_PORT', 'CROWD_REDIS_USER', 'CROWD_REDIS_PASSWORD'],
+  redis: ['CROWD_REDIS_HOST', 'CROWD_REDIS_PORT', 'CROWD_REDIS_USERNAME', 'CROWD_REDIS_PASSWORD'],
 }
 
 /*
 Config is used to configure the service.
 */
 export interface Config {
+  // Additional environment variables required by the service to properly run.
+  envvars?: string[]
+
   // Enable and configure the Kafka producer, if needed.
   producer: {
     enabled: boolean
@@ -64,17 +67,20 @@ export class Service {
   protected _redisClient: RedisClient | null
 
   constructor(config: Config) {
-    this.name = process.env['CROWD_SERVICE']
+    this.name = process.env['SERVICE']
     this.tracer = tracer
     this.log = logger
     this.config = config
     this.integrations = INTEGRATION_SERVICES
 
+    // TODO: Handle SSL and SASL configuration.
     if (config.producer.enabled && process.env['CROWD_KAFKA_BROKERS']) {
       const brokers = process.env['CROWD_KAFKA_BROKERS']
       this._kafka = new Kafka({
         clientId: this.name,
         brokers: brokers.split(','),
+        // sasl
+        // ssl
       })
     }
   }
@@ -143,6 +149,12 @@ export class Service {
       }
     })
 
+    this.config.envvars.forEach((envvar) => {
+      if (!process.env[envvar]) {
+        missing.push(envvar)
+      }
+    })
+
     // Only validate Kafka-related environment variables if enabled.
     if (this.config.producer.enabled) {
       envvars.producer.forEach((envvar) => {
@@ -189,7 +201,7 @@ export class Service {
       appName: this.name,
       url: process.env['CROWD_UNLEASH_URL'],
       customHeaders: {
-        Authorization: process.env['CROWD_UNLEASH_API_TOKEN'],
+        Authorization: process.env['CROWD_UNLEASH_BACKEND_API_KEY'],
       },
     })
 
@@ -224,7 +236,7 @@ export class Service {
         this._redisClient = await getRedisClient({
           host: process.env['CROWD_REDIS_HOST'],
           port: process.env['CROWD_REDIS_PORT'],
-          username: process.env['CROWD_REDIS_USER'],
+          username: process.env['CROWD_REDIS_USERNAME'],
           password: process.env['CROWD_REDIS_PASSWORD'],
         })
 

services/archetypes/standard/tsconfig.json

@@ -10,7 +10,8 @@
     "esModuleInterop": true,
     "baseUrl": "./src",
     "paths": {
-      "@crowd/*": ["../../../libs/*/src", "../../../archetypes/*/src"]
+      "@crowd/*": ["../../../libs/*/src"],
+      "@crowd/archetype-*": ["../../../archetypes/*/src"]
     }
   },
   "include": ["src/**/*"]

services/archetypes/worker/src/index.ts

@@ -2,19 +2,20 @@ import path from 'path'
 
 import { NativeConnection, Worker as TemporalWorker, bundleWorkflowCode } from '@temporalio/worker'
 
-import { Config, Service } from '@crowd/standard'
+import { Config, Service } from '@crowd/archetype-standard'
 import { getDbConnection, DbStore } from '@crowd/database'
 
 // List all required environment variables, grouped per "component".
 // They are in addition to the ones required by the "standard" archetype.
 const envvars = {
   worker: ['CROWD_TEMPORAL_SERVER_URL', 'CROWD_TEMPORAL_NAMESPACE', 'CROWD_TEMPORAL_TASKQUEUE'],
   postgres: [
-    'CROWD_POSTGRES_HOST',
-    'CROWD_POSTGRES_PORT',
-    'CROWD_POSTGRES_USER',
-    'CROWD_POSTGRES_PASSWORD',
-    'CROWD_POSTGRES_DATABASE',
+    'CROWD_DB_READ_HOST',
+    'CROWD_DB_WRITE_HOST',
+    'CROWD_DB_PORT',
+    'CROWD_DB_USERNAME',
+    'CROWD_DB_PASSWORD',
+    'CROWD_DB_DATABASE',
   ],
 }
 
@@ -34,20 +35,24 @@ export class ServiceWorker extends Service {
   readonly options: Options
 
   protected _worker: TemporalWorker
-  protected _postgres: DbStore
+  protected _postgresReader: DbStore
+  protected _postgresWriter: DbStore
 
   constructor(config: Config, opts: Options) {
     super(config)
 
     this.options = opts
   }
 
-  get postgres(): DbStore | null {
+  get postgres(): { reader: DbStore; writer: DbStore } | null {
     if (!this.options.postgres.enabled) {
       return null
     }
 
-    return this._postgres
+    return {
+      reader: this._postgresReader,
+      writer: this._postgresWriter,
+    }
   }
 
   // We first need to ensure a standard service can be initialized given the config
@@ -111,14 +116,28 @@ export class ServiceWorker extends Service {
     if (this.options.postgres.enabled) {
       try {
         const dbConnection = await getDbConnection({
-          host: process.env['CROWD_POSTGRES_HOST'],
-          port: Number(process.env['CROWD_POSTGRES_PORT']),
-          user: process.env['CROWD_POSTGRES_USER'],
-          password: process.env['CROWD_POSTGRES_PASSWORD'],
-          database: process.env['CROWD_POSTGRES_DATABASE'],
+          host: process.env['CROWD_DB_READ_HOST'],
+          port: Number(process.env['CROWD_DB_PORT']),
+          user: process.env['CROWD_DB_USERNAME'],
+          password: process.env['CROWD_DB_PASSWORD'],
+          database: process.env['CROWD_DB_DATABASE'],
+        })
+
+        this._postgresReader = new DbStore(this.log, dbConnection)
+      } catch (err) {
+        throw new Error(err)
+      }
+
+      try {
+        const dbConnection = await getDbConnection({
+          host: process.env['CROWD_DB_WRITE_HOST'],
+          port: Number(process.env['CROWD_DB_PORT']),
+          user: process.env['CROWD_DB_USERNAME'],
+          password: process.env['CROWD_DB_PASSWORD'],
+          database: process.env['CROWD_DB_DATABASE'],
         })
 
-        this._postgres = new DbStore(this.log, dbConnection)
+        this._postgresWriter = new DbStore(this.log, dbConnection)
       } catch (err) {
         throw new Error(err)
       }
@@ -137,9 +156,9 @@ export class ServiceWorker extends Service {
   // Stop allows to gracefully stop the service. Order for closing connections
   // matters. We need to stop the Temporal worker before closing other connections.
   protected override async stop() {
-    this._worker.shutdown()
     if (this.options.postgres.enabled) {
-      this._postgres.dbInstance.end()
+      this._postgresWriter.dbInstance.end()
+      this._postgresReader.dbInstance.end()
     }
 
     await super.stop()

services/archetypes/worker/tsconfig.json

@@ -10,7 +10,8 @@
     "esModuleInterop": true,
     "baseUrl": "./src",
     "paths": {
-      "@crowd/*": ["../../../libs/*/src", "../../../archetypes/*/src"]
+      "@crowd/*": ["../../../libs/*/src"],
+      "@crowd/archetype-*": ["../../../archetypes/*/src"]
     }
   },
   "include": ["src/**/*"]