Add function to fetch active organizations in the last year (#1630)

Author: skwowet

backend/src/bin/scripts/enrich-organizations-synchronous.ts

@@ -57,7 +57,7 @@ if (parameters.help || !parameters.tenant || !parameters.limit) {
     const limit = parameters.limit
 
     for (const tenantId of tenantIds) {
-      await BulkorganizationEnrichmentWorker(tenantId, limit, true)
+      await BulkorganizationEnrichmentWorker(tenantId, limit, true, true)
       log.info(`Done for tenant ${tenantId}`)
     }
 

backend/src/database/repositories/organizationRepository.ts

@@ -131,6 +131,89 @@ class OrganizationRepository {
     return orgs
   }
 
+  static async filterByActiveLastYear(
+    tenantId: string,
+    limit: number,
+    options: IRepositoryOptions,
+  ): Promise<IEnrichableOrganization[]> {
+    const database = SequelizeRepository.getSequelize(options)
+    const transaction = SequelizeRepository.getTransaction(options)
+    const query = `
+    with org_activities as (select a."organizationId", count(a.id) as "orgActivityCount"
+                        from activities a
+                        where a."tenantId" = :tenantId
+                          and a."deletedAt" is null
+                          and a."isContribution" = true
+                          and a."createdAt" > (CURRENT_DATE - INTERVAL '1 year')
+                        group by a."organizationId"
+                        having count(id) > 0),
+     identities as (select oi."organizationId", jsonb_agg(oi) as "identities"
+                    from "organizationIdentities" oi
+                    where oi."tenantId" = :tenantId
+                    group by oi."organizationId")
+    select org.id,
+          i.identities,
+          org."displayName",
+          org."location",
+          org."website",
+          org."lastEnrichedAt",
+          org."twitter",
+          org."employees",
+          org."size",
+          org."founded",
+          org."industry",
+          org."naics",
+          org."profiles",
+          org."headline",
+          org."ticker",
+          org."type",
+          org."address",
+          org."geoLocation",
+          org."employeeCountByCountry",
+          org."twitter",
+          org."linkedin",
+          org."crunchbase",
+          org."github",
+          org."description",
+          org."revenueRange",
+          org."tags",
+          org."affiliatedProfiles",
+          org."allSubsidiaries",
+          org."alternativeDomains",
+          org."alternativeNames",
+          org."averageEmployeeTenure",
+          org."averageTenureByLevel",
+          org."averageTenureByRole",
+          org."directSubsidiaries",
+          org."employeeChurnRate",
+          org."employeeCountByMonth",
+          org."employeeGrowthRate",
+          org."employeeCountByMonthByLevel",
+          org."employeeCountByMonthByRole",
+          org."gicsSector",
+          org."grossAdditionsByMonth",
+          org."grossDeparturesByMonth",
+          org."ultimateParent",
+          org."immediateParent",
+          activity."orgActivityCount"
+    from "organizations" as org
+            join org_activities activity on activity."organizationId" = org."id"
+            join identities i on i."organizationId" = org.id
+    where :tenantId = org."tenantId"
+    order by org."lastEnrichedAt" asc, org."website", activity."orgActivityCount" desc, org."createdAt" desc
+    limit :limit
+    `
+    const orgs: IEnrichableOrganization[] = await database.query(query, {
+      type: QueryTypes.SELECT,
+      transaction,
+      replacements: {
+        tenantId,
+        limit,
+      },
+    })
+    return orgs
+  }
+
   static async create(data, options: IRepositoryOptions) {
     const currentUser = SequelizeRepository.getCurrentUser(options)
 

backend/src/serverless/microservices/nodejs/bulk-enrichment/bulkOrganizationEnrichmentWorker.ts

@@ -10,6 +10,7 @@ export async function BulkorganizationEnrichmentWorker(
   tenantId: string,
   maxEnrichLimit: number = 0,
   verbose: boolean = false,
+  includeOrgsActiveLastYear: boolean = false,
 ) {
   const userContext = await getUserContext(tenantId)
   const redis = await getRedisClient(REDIS_CONFIG, true)
@@ -39,7 +40,10 @@ export async function BulkorganizationEnrichmentWorker(
       tenantId,
       limit: remainderEnrichmentLimit,
     })
-    enrichedOrgs = await enrichmentService.enrichOrganizationsAndSignalDone(verbose)
+    enrichedOrgs = await enrichmentService.enrichOrganizationsAndSignalDone(
+      includeOrgsActiveLastYear,
+      verbose,
+    )
   }
 
   if (!skipCredits) {

backend/src/services/premium/enrichment/organizationEnrichmentService.ts

@@ -70,6 +70,7 @@ export default class OrganizationEnrichmentService extends LoggerBase {
   }
 
   public async enrichOrganizationsAndSignalDone(
+    includeOrgsActiveLastYear: boolean = false,
     verbose: boolean = false,
   ): Promise<IOrganization[]> {
     const enrichmentPlatformPriority = [
@@ -80,7 +81,12 @@ export default class OrganizationEnrichmentService extends LoggerBase {
     const enrichedOrganizations: IOrganization[] = []
     const enrichedCacheOrganizations: IOrganizationCache[] = []
     let count = 0
-    for (const instance of await OrganizationRepository.filterByPayingTenant(
+
+    const organizationFilterMethod = includeOrgsActiveLastYear
+      ? OrganizationRepository.filterByActiveLastYear
+      : OrganizationRepository.filterByPayingTenant
+
+    for (const instance of await organizationFilterMethod(
       this.tenantId,
       this.maxOrganizationsLimit,
       this.options,