diff --git a/services/apps/template_consumer/package-lock.json b/services/apps/template_consumer/package-lock.json index 9e1a7447c3..96bb049556 100644 --- a/services/apps/template_consumer/package-lock.json +++ b/services/apps/template_consumer/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "dependencies": { "@crowd/archetype-consumer": "file:../../archetypes/consumer", + "@crowd/archetype-standard": "file:../../archetypes/standard", "@types/node": "^20.8.2", "ts-node": "^10.9.1", "tsconfig-paths": "^4.2.0", @@ -50,7 +51,6 @@ "../../archetypes/standard": { "name": "@crowd/archetype-standard", "version": "1.0.0", - "extraneous": true, "dependencies": { "@crowd/logging": "file:../../libs/logging", "@crowd/redis": "file:../../libs/redis", @@ -263,6 +263,10 @@ "resolved": "../../archetypes/consumer", "link": true }, + "node_modules/@crowd/archetype-standard": { + "resolved": "../../archetypes/standard", + "link": true + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -2795,6 +2799,30 @@ "typescript": "^5.2.2" } }, + "@crowd/archetype-standard": { + "version": "file:../../archetypes/standard", + "requires": { + "@crowd/logging": "file:../../libs/logging", + "@crowd/redis": "file:../../libs/redis", + "@crowd/tracing": "file:../../libs/tracing", + "@temporalio/client": "~1.8.6", + "@types/config": "^3.3.1", + "@types/node": "^20.8.2", + "@typescript-eslint/eslint-plugin": "^6.7.4", + "@typescript-eslint/parser": "^6.7.4", + "asyncapi-validator": "~4.0.0", + "config": "^3.3.9", + "eslint": "^8.50.0", + "eslint-config-prettier": "^9.0.0", + "eslint-plugin-prettier": "^5.0.0", + "kafkajs": "~2.2.4", + "prettier": "^3.0.3", + "ts-node": "^10.9.1", + "tsconfig-paths": "^4.2.0", + "typescript": "^5.2.2", + "unleash-client": "~4.1.1" + } + }, "@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", diff --git a/services/apps/template_consumer/package.json b/services/apps/template_consumer/package.json index 0faaca6ab8..17af1b0af7 100644 --- a/services/apps/template_consumer/package.json +++ b/services/apps/template_consumer/package.json @@ -15,6 +15,7 @@ }, "dependencies": { "@crowd/archetype-consumer": "file:../../archetypes/consumer", + "@crowd/archetype-standard": "file:../../archetypes/standard", "@types/node": "^20.8.2", "ts-node": "^10.9.1", "tsconfig-paths": "^4.2.0", diff --git a/services/apps/template_consumer/src/main.ts b/services/apps/template_consumer/src/main.ts index 50c6e38302..2a11fc6e76 100644 --- a/services/apps/template_consumer/src/main.ts +++ b/services/apps/template_consumer/src/main.ts @@ -1,5 +1,5 @@ -import { Config } from '@crowd/standard' -import { ServiceConsumer, Options } from '@crowd/consumer' +import { Config } from '@crowd/archetype-standard' +import { ServiceConsumer, Options } from '@crowd/archetype-consumer' const config: Config = { producer: { diff --git a/services/apps/template_consumer/tsconfig.json b/services/apps/template_consumer/tsconfig.json index f34861f26c..47a983b5d5 100644 --- a/services/apps/template_consumer/tsconfig.json +++ b/services/apps/template_consumer/tsconfig.json @@ -10,7 +10,8 @@ "esModuleInterop": true, "baseUrl": "./src", "paths": { - "@crowd/*": ["../../../libs/*/src", "../../../archetypes/*/src"] + "@crowd/*": ["../../../libs/*/src"], + "@crowd/archetype-*": ["../../../archetypes/*/src"] } }, "include": ["src/**/*"] diff --git a/services/apps/template_worker/package-lock.json b/services/apps/template_worker/package-lock.json index fbbb0c4dbf..c6dc0d0dc5 100644 --- a/services/apps/template_worker/package-lock.json +++ b/services/apps/template_worker/package-lock.json @@ -8,6 +8,7 @@ "name": "@crowd/template-worker", "version": "1.0.0", "dependencies": { + "@crowd/archetype-standard": "file:../../archetypes/standard", "@crowd/archetype-worker": "file:../../archetypes/worker", "@temporalio/workflow": "~1.8.6", "@types/node": "^20.8.2", @@ -52,7 +53,6 @@ "../../archetypes/standard": { "name": "@crowd/archetype-standard", "version": "1.0.0", - "extraneous": true, "dependencies": { "@crowd/logging": "file:../../libs/logging", "@crowd/redis": "file:../../libs/redis", @@ -286,6 +286,10 @@ "node": ">=0.10.0" } }, + "node_modules/@crowd/archetype-standard": { + "resolved": "../../archetypes/standard", + "link": true + }, "node_modules/@crowd/archetype-worker": { "resolved": "../../archetypes/worker", "link": true @@ -2942,6 +2946,30 @@ "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", "dev": true }, + "@crowd/archetype-standard": { + "version": "file:../../archetypes/standard", + "requires": { + "@crowd/logging": "file:../../libs/logging", + "@crowd/redis": "file:../../libs/redis", + "@crowd/tracing": "file:../../libs/tracing", + "@temporalio/client": "~1.8.6", + "@types/config": "^3.3.1", + "@types/node": "^20.8.2", + "@typescript-eslint/eslint-plugin": "^6.7.4", + "@typescript-eslint/parser": "^6.7.4", + "asyncapi-validator": "~4.0.0", + "config": "^3.3.9", + "eslint": "^8.50.0", + "eslint-config-prettier": "^9.0.0", + "eslint-plugin-prettier": "^5.0.0", + "kafkajs": "~2.2.4", + "prettier": "^3.0.3", + "ts-node": "^10.9.1", + "tsconfig-paths": "^4.2.0", + "typescript": "^5.2.2", + "unleash-client": "~4.1.1" + } + }, "@crowd/archetype-worker": { "version": "file:../../archetypes/worker", "requires": { diff --git a/services/apps/template_worker/package.json b/services/apps/template_worker/package.json index 03855caf69..60be644429 100644 --- a/services/apps/template_worker/package.json +++ b/services/apps/template_worker/package.json @@ -14,6 +14,7 @@ "tsc-check": "./node_modules/.bin/tsc --noEmit" }, "dependencies": { + "@crowd/archetype-standard": "file:../../archetypes/standard", "@crowd/archetype-worker": "file:../../archetypes/worker", "@temporalio/workflow": "~1.8.6", "@types/node": "^20.8.2", diff --git a/services/apps/template_worker/src/main.ts b/services/apps/template_worker/src/main.ts index 97309b5a2a..de9edd4796 100644 --- a/services/apps/template_worker/src/main.ts +++ b/services/apps/template_worker/src/main.ts @@ -1,5 +1,5 @@ -import { Config } from '@crowd/standard' -import { ServiceWorker, Options } from '@crowd/worker' +import { Config } from '@crowd/archetype-standard' +import { ServiceWorker, Options } from '@crowd/archetype-worker' const config: Config = { producer: { diff --git a/services/apps/template_worker/tsconfig.json b/services/apps/template_worker/tsconfig.json index f34861f26c..47a983b5d5 100644 --- a/services/apps/template_worker/tsconfig.json +++ b/services/apps/template_worker/tsconfig.json @@ -10,7 +10,8 @@ "esModuleInterop": true, "baseUrl": "./src", "paths": { - "@crowd/*": ["../../../libs/*/src", "../../../archetypes/*/src"] + "@crowd/*": ["../../../libs/*/src"], + "@crowd/archetype-*": ["../../../archetypes/*/src"] } }, "include": ["src/**/*"] diff --git a/services/archetypes/consumer/src/index.ts b/services/archetypes/consumer/src/index.ts index f10e9bcb90..037dd1d5c4 100644 --- a/services/archetypes/consumer/src/index.ts +++ b/services/archetypes/consumer/src/index.ts @@ -1,6 +1,6 @@ import { Consumer as KafkaConsumer } from 'kafkajs' -import { Config, Service } from '@crowd/standard' +import { Config, Service } from '@crowd/archetype-standard' // List all required environment variables, grouped per "component". // They are in addition to the ones required by the "standard" archetype. diff --git a/services/archetypes/consumer/tsconfig.json b/services/archetypes/consumer/tsconfig.json index f34861f26c..47a983b5d5 100644 --- a/services/archetypes/consumer/tsconfig.json +++ b/services/archetypes/consumer/tsconfig.json @@ -10,7 +10,8 @@ "esModuleInterop": true, "baseUrl": "./src", "paths": { - "@crowd/*": ["../../../libs/*/src", "../../../archetypes/*/src"] + "@crowd/*": ["../../../libs/*/src"], + "@crowd/archetype-*": ["../../../archetypes/*/src"] } }, "include": ["src/**/*"] diff --git a/services/archetypes/standard/package-lock.json b/services/archetypes/standard/package-lock.json index c4dc3370bf..04f7beb6e0 100644 --- a/services/archetypes/standard/package-lock.json +++ b/services/archetypes/standard/package-lock.json @@ -114,7 +114,8 @@ "@opentelemetry/instrumentation-bunyan": "~0.32.1", "@opentelemetry/instrumentation-express": "~0.33.1", "@opentelemetry/instrumentation-http": "~0.43.0", - "@opentelemetry/instrumentation-redis": "~0.35.1", + "@opentelemetry/instrumentation-pg": "^0.36.2", + "@opentelemetry/instrumentation-redis-4": "^0.35.3", "@opentelemetry/resource-detector-aws": "~1.3.1", "@opentelemetry/resources": "~1.17.0", "@opentelemetry/sdk-node": "~0.43.0", diff --git a/services/archetypes/standard/src/index.ts b/services/archetypes/standard/src/index.ts index c817728c9f..53a9b961c1 100644 --- a/services/archetypes/standard/src/index.ts +++ b/services/archetypes/standard/src/index.ts @@ -13,16 +13,19 @@ const logger = getServiceLogger() // List all required environment variables, grouped per "component". const envvars = { - base: ['CROWD_SERVICE', 'CROWD_UNLEASH_URL', 'CROWD_UNLEASH_API_TOKEN'], + base: ['SERVICE', 'CROWD_UNLEASH_URL', 'CROWD_UNLEASH_BACKEND_API_KEY'], producer: ['CROWD_KAFKA_BROKERS'], temporal: ['CROWD_TEMPORAL_SERVER_URL', 'CROWD_TEMPORAL_NAMESPACE'], - redis: ['CROWD_REDIS_HOST', 'CROWD_REDIS_PORT', 'CROWD_REDIS_USER', 'CROWD_REDIS_PASSWORD'], + redis: ['CROWD_REDIS_HOST', 'CROWD_REDIS_PORT', 'CROWD_REDIS_USERNAME', 'CROWD_REDIS_PASSWORD'], } /* Config is used to configure the service. */ export interface Config { + // Additional environment variables required by the service to properly run. + envvars?: string[] + // Enable and configure the Kafka producer, if needed. producer: { enabled: boolean @@ -64,17 +67,20 @@ export class Service { protected _redisClient: RedisClient | null constructor(config: Config) { - this.name = process.env['CROWD_SERVICE'] + this.name = process.env['SERVICE'] this.tracer = tracer this.log = logger this.config = config this.integrations = INTEGRATION_SERVICES + // TODO: Handle SSL and SASL configuration. if (config.producer.enabled && process.env['CROWD_KAFKA_BROKERS']) { const brokers = process.env['CROWD_KAFKA_BROKERS'] this._kafka = new Kafka({ clientId: this.name, brokers: brokers.split(','), + // sasl + // ssl }) } } @@ -143,6 +149,12 @@ export class Service { } }) + this.config.envvars.forEach((envvar) => { + if (!process.env[envvar]) { + missing.push(envvar) + } + }) + // Only validate Kafka-related environment variables if enabled. if (this.config.producer.enabled) { envvars.producer.forEach((envvar) => { @@ -189,7 +201,7 @@ export class Service { appName: this.name, url: process.env['CROWD_UNLEASH_URL'], customHeaders: { - Authorization: process.env['CROWD_UNLEASH_API_TOKEN'], + Authorization: process.env['CROWD_UNLEASH_BACKEND_API_KEY'], }, }) @@ -224,7 +236,7 @@ export class Service { this._redisClient = await getRedisClient({ host: process.env['CROWD_REDIS_HOST'], port: process.env['CROWD_REDIS_PORT'], - username: process.env['CROWD_REDIS_USER'], + username: process.env['CROWD_REDIS_USERNAME'], password: process.env['CROWD_REDIS_PASSWORD'], }) diff --git a/services/archetypes/standard/tsconfig.json b/services/archetypes/standard/tsconfig.json index f34861f26c..47a983b5d5 100644 --- a/services/archetypes/standard/tsconfig.json +++ b/services/archetypes/standard/tsconfig.json @@ -10,7 +10,8 @@ "esModuleInterop": true, "baseUrl": "./src", "paths": { - "@crowd/*": ["../../../libs/*/src", "../../../archetypes/*/src"] + "@crowd/*": ["../../../libs/*/src"], + "@crowd/archetype-*": ["../../../archetypes/*/src"] } }, "include": ["src/**/*"] diff --git a/services/archetypes/worker/src/index.ts b/services/archetypes/worker/src/index.ts index 36fd6f266c..fc9e376867 100644 --- a/services/archetypes/worker/src/index.ts +++ b/services/archetypes/worker/src/index.ts @@ -2,7 +2,7 @@ import path from 'path' import { NativeConnection, Worker as TemporalWorker, bundleWorkflowCode } from '@temporalio/worker' -import { Config, Service } from '@crowd/standard' +import { Config, Service } from '@crowd/archetype-standard' import { getDbConnection, DbStore } from '@crowd/database' // List all required environment variables, grouped per "component". @@ -10,11 +10,12 @@ import { getDbConnection, DbStore } from '@crowd/database' const envvars = { worker: ['CROWD_TEMPORAL_SERVER_URL', 'CROWD_TEMPORAL_NAMESPACE', 'CROWD_TEMPORAL_TASKQUEUE'], postgres: [ - 'CROWD_POSTGRES_HOST', - 'CROWD_POSTGRES_PORT', - 'CROWD_POSTGRES_USER', - 'CROWD_POSTGRES_PASSWORD', - 'CROWD_POSTGRES_DATABASE', + 'CROWD_DB_READ_HOST', + 'CROWD_DB_WRITE_HOST', + 'CROWD_DB_PORT', + 'CROWD_DB_USERNAME', + 'CROWD_DB_PASSWORD', + 'CROWD_DB_DATABASE', ], } @@ -34,7 +35,8 @@ export class ServiceWorker extends Service { readonly options: Options protected _worker: TemporalWorker - protected _postgres: DbStore + protected _postgresReader: DbStore + protected _postgresWriter: DbStore constructor(config: Config, opts: Options) { super(config) @@ -42,12 +44,15 @@ export class ServiceWorker extends Service { this.options = opts } - get postgres(): DbStore | null { + get postgres(): { reader: DbStore; writer: DbStore } | null { if (!this.options.postgres.enabled) { return null } - return this._postgres + return { + reader: this._postgresReader, + writer: this._postgresWriter, + } } // We first need to ensure a standard service can be initialized given the config @@ -111,14 +116,28 @@ export class ServiceWorker extends Service { if (this.options.postgres.enabled) { try { const dbConnection = await getDbConnection({ - host: process.env['CROWD_POSTGRES_HOST'], - port: Number(process.env['CROWD_POSTGRES_PORT']), - user: process.env['CROWD_POSTGRES_USER'], - password: process.env['CROWD_POSTGRES_PASSWORD'], - database: process.env['CROWD_POSTGRES_DATABASE'], + host: process.env['CROWD_DB_READ_HOST'], + port: Number(process.env['CROWD_DB_PORT']), + user: process.env['CROWD_DB_USERNAME'], + password: process.env['CROWD_DB_PASSWORD'], + database: process.env['CROWD_DB_DATABASE'], + }) + + this._postgresReader = new DbStore(this.log, dbConnection) + } catch (err) { + throw new Error(err) + } + + try { + const dbConnection = await getDbConnection({ + host: process.env['CROWD_DB_WRITE_HOST'], + port: Number(process.env['CROWD_DB_PORT']), + user: process.env['CROWD_DB_USERNAME'], + password: process.env['CROWD_DB_PASSWORD'], + database: process.env['CROWD_DB_DATABASE'], }) - this._postgres = new DbStore(this.log, dbConnection) + this._postgresWriter = new DbStore(this.log, dbConnection) } catch (err) { throw new Error(err) } @@ -137,9 +156,9 @@ export class ServiceWorker extends Service { // Stop allows to gracefully stop the service. Order for closing connections // matters. We need to stop the Temporal worker before closing other connections. protected override async stop() { - this._worker.shutdown() if (this.options.postgres.enabled) { - this._postgres.dbInstance.end() + this._postgresWriter.dbInstance.end() + this._postgresReader.dbInstance.end() } await super.stop() diff --git a/services/archetypes/worker/tsconfig.json b/services/archetypes/worker/tsconfig.json index f34861f26c..47a983b5d5 100644 --- a/services/archetypes/worker/tsconfig.json +++ b/services/archetypes/worker/tsconfig.json @@ -10,7 +10,8 @@ "esModuleInterop": true, "baseUrl": "./src", "paths": { - "@crowd/*": ["../../../libs/*/src", "../../../archetypes/*/src"] + "@crowd/*": ["../../../libs/*/src"], + "@crowd/archetype-*": ["../../../archetypes/*/src"] } }, "include": ["src/**/*"]