Skip to content

Commit 82bf678

Browse files
joanagmaiajoanagmaia
committed
Merge remote-tracking branch 'origin/main' into improvement/consistent-timestamps
2 parents c634eff + 6cb2ed6 commit 82bf678

File tree

14 files changed

+200
-60
lines changed

14 files changed

+200
-60
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
</a>
77

8-
<h2 align="center">Understand who is engaging with your open source project.</h2>
8+
<h2 align="center">Effortlessly centralize community, product, and customer data in one database</h2>
99

1010
<p align="center">
1111
<br>

‎backend/src/database/repositories/organizationRepository.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,35 @@ class OrganizationRepository {
515515
return record.get({ plain: true })
516516
}
517517

518+
static async findByDomain(domain, options: IRepositoryOptions) {
519+
const transaction = SequelizeRepository.getTransaction(options)
520+
const currentTenant = SequelizeRepository.getCurrentTenant(options)
521+
522+
// Check if organization exists
523+
const organization = await options.database.organization.findOne({
524+
where: {
525+
website: {
526+
[Sequelize.Op.or]: [
527+
// Matches URLs having 'http://' or 'https://'
528+
{ [Sequelize.Op.iLike]: `%://${domain}` },
529+
// Matches URLs having 'www'
530+
{ [Sequelize.Op.iLike]: `%://www.${domain}` },
531+
// Matches URLs that doesn't have 'http://' or 'https://' and 'www'
532+
{ [Sequelize.Op.iLike]: `${domain}` },
533+
],
534+
},
535+
tenantId: currentTenant.id,
536+
},
537+
transaction,
538+
})
539+
540+
if (!organization) {
541+
return null
542+
}
543+
544+
return organization.get({ plain: true })
545+
}
546+
518547
static async filterIdInTenant(id, options: IRepositoryOptions) {
519548
return lodash.get(await this.filterIdsInTenant([id], options), '[0]', null)
520549
}

‎backend/src/services/aws.ts‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import AWS, { SQS } from 'aws-sdk'
2+
import { trimUtf8ToMaxByteLength } from '@crowd/common'
23
import { COMPREHEND_CONFIG, IS_DEV_ENV, KUBE_MODE, S3_CONFIG, SQS_CONFIG } from '../conf'
34

45
let sqsInstance
@@ -100,22 +101,6 @@ if (KUBE_MODE) {
100101
: undefined
101102
}
102103

103-
const trimUtf8ToMaxByteLength = (utf8Str: string, maxByteLength: number): string => {
104-
if (Buffer.byteLength(utf8Str, 'utf8') > maxByteLength) {
105-
// this will get us close but some characters could be multibyte encoded so we might need to trim a bit more
106-
utf8Str = utf8Str.slice(0, maxByteLength)
107-
}
108-
109-
// trim till we get to the requested byte length or lower (if we cut multibyte character)
110-
let byteLength = Buffer.byteLength(utf8Str, 'utf8')
111-
while (byteLength > maxByteLength) {
112-
utf8Str = utf8Str.slice(0, -1)
113-
byteLength = Buffer.byteLength(utf8Str, 'utf8')
114-
}
115-
116-
return utf8Str
117-
}
118-
119104
const ALLOWED_MAX_BYTE_LENGTH = 5000
120105

121106
/**

‎backend/src/services/memberService.ts‎

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -303,10 +303,10 @@ export default class MemberService extends LoggerBase {
303303
}
304304
}
305305

306+
// Collect IDs for relation
307+
const organizations = []
306308
// If organizations are sent
307309
if (data.organizations) {
308-
// Collect IDs for relation
309-
const organizations = []
310310
for (const organization of data.organizations) {
311311
if (typeof organization === 'string' && validator.isUUID(organization)) {
312312
// If an ID was already sent, we simply push it to the list
@@ -329,33 +329,35 @@ export default class MemberService extends LoggerBase {
329329
organizations.push({ id: organizationRecord.id })
330330
}
331331
}
332+
}
332333

333-
// Auto assign member to organization if email domain matches
334-
if (data.emails) {
335-
const emailDomains = new Set()
334+
// Auto assign member to organization if email domain matches
335+
if (data.emails) {
336+
const emailDomains = new Set()
336337

337-
// Collect unique domains
338-
for (const email of data.emails) {
339-
if (!email) {
340-
continue
341-
}
342-
const domain = email.split('@')[1]
343-
emailDomains.add(domain)
338+
// Collect unique domains
339+
for (const email of data.emails) {
340+
if (!email) {
341+
continue
344342
}
343+
const domain = email.split('@')[1]
344+
emailDomains.add(domain)
345+
}
345346

346-
// Fetch organization ids for these domains
347-
const organizationService = new OrganizationService(this.options)
348-
for (const domain of emailDomains) {
349-
if (domain) {
350-
const organizationRecord = await organizationService.findByUrl(domain)
351-
if (organizationRecord) {
352-
organizations.push({ id: organizationRecord.id })
353-
}
347+
// Fetch organization ids for these domains
348+
const organizationService = new OrganizationService(this.options)
349+
for (const domain of emailDomains) {
350+
if (domain) {
351+
const organizationRecord = await organizationService.findByDomain(domain)
352+
if (organizationRecord) {
353+
organizations.push({ id: organizationRecord.id })
354354
}
355355
}
356356
}
357+
}
357358

358-
// Remove dups
359+
// Remove dups
360+
if (organizations.length > 0) {
359361
data.organizations = lodash.uniqBy(organizations, 'id')
360362
}
361363

‎backend/src/services/organizationService.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,10 @@ export default class OrganizationService extends LoggerBase {
212212
return OrganizationRepository.findByUrl(url, this.options)
213213
}
214214

215+
async findByDomain(domain) {
216+
return OrganizationRepository.findByDomain(domain, this.options)
217+
}
218+
215219
async query(data) {
216220
const advancedFilter = data.filter
217221
const orderBy = data.orderBy

‎frontend/src/modules/member/components/member-identities.vue‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
:tooltip-label="platformContent(platform).tooltipLabel"
1818
:as-link="platformContent(platform).asLink"
1919
:show-handles-badge="true"
20-
:backup-url="props.member.attributes.url[platform]"
20+
:backup-url="props.member.attributes.url?.[platform]"
2121
/>
2222
</div>
2323
</div>
@@ -54,7 +54,7 @@ const platformContent = (platform) => {
5454
};
5555
5656
const hasSocialIdentities = computed(
57-
() => Object.keys(props.username).some((k) => !!props.username[k].length),
57+
() => Object.values(props.username).some((k) => k.length > 0),
5858
);
5959
</script>
6060

‎services/apps/data_sink_worker/src/repo/organization.repo.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,61 @@ export class OrganizationRepository extends RepositoryBase<OrganizationRepositor
173173
return result
174174
}
175175

176+
public async findByDomain(
177+
tenantId: string,
178+
segmentId: string,
179+
domain: string,
180+
): Promise<IDbOrganization> {
181+
const result = await this.db().oneOrNone(
182+
`
183+
SELECT
184+
o.id,
185+
o.name,
186+
o.url,
187+
o.description,
188+
o.emails,
189+
o.logo,
190+
o.tags,
191+
o.github,
192+
o.twitter,
193+
o.linkedin,
194+
o.crunchbase,
195+
o.employees,
196+
o.location,
197+
o.website,
198+
o.type,
199+
o.size,
200+
o.headline,
201+
o.industry,
202+
o.founded,
203+
o.attributes
204+
FROM
205+
organizations o
206+
WHERE
207+
o."tenantId" = $(tenantId) AND
208+
(
209+
o.website ILIKE $(protocolDomain) OR
210+
o.website ILIKE $(domainWithWww) OR
211+
o.website ILIKE $(domain)
212+
) AND
213+
o.id IN (
214+
SELECT os."organizationId"
215+
FROM "organizationSegments" os
216+
WHERE os."segmentId" = $(segmentId)
217+
)
218+
`,
219+
{
220+
tenantId,
221+
protocolDomain: `%://${domain}`,
222+
domainWithWww: `%://www.${domain}`,
223+
domain,
224+
segmentId,
225+
},
226+
)
227+
228+
return result
229+
}
230+
176231
public async insert(tenantId: string, data: IDbInsertOrganizationData): Promise<string> {
177232
const id = generateUUIDv1()
178233
const ts = new Date()

‎services/apps/data_sink_worker/src/service/member.service.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,17 @@ export default class MemberService extends LoggerBase {
7979
}
8080
}
8181

82+
if (data.emails) {
83+
const orgIds = await this.assignOrganizationByEmailDomain(
84+
tenantId,
85+
segmentId,
86+
data.emails,
87+
)
88+
if (orgIds.length > 0) {
89+
organizationIds.push(...orgIds)
90+
}
91+
}
92+
8293
return {
8394
id,
8495
organizationIds,
@@ -179,6 +190,17 @@ export default class MemberService extends LoggerBase {
179190
}
180191
}
181192

193+
if (data.emails) {
194+
const orgIds = await this.assignOrganizationByEmailDomain(
195+
tenantId,
196+
segmentId,
197+
data.emails,
198+
)
199+
if (orgIds.length > 0) {
200+
organizationIds.push(...orgIds)
201+
}
202+
}
203+
182204
return { updated, organizationIds }
183205
})
184206

@@ -198,6 +220,32 @@ export default class MemberService extends LoggerBase {
198220
}
199221
}
200222

223+
public async assignOrganizationByEmailDomain(
224+
tenantId: string,
225+
segmentId: string,
226+
emails: string[],
227+
): Promise<string[]> {
228+
const orgService = new OrganizationService(this.store, this.log)
229+
const organizationIds: string[] = []
230+
const emailDomains = new Set<string>()
231+
232+
// Collect unique domains
233+
for (const email of emails) {
234+
const domain = email.split('@')[1]
235+
emailDomains.add(domain)
236+
}
237+
238+
// Assign member to organization based on email domain
239+
for (const domain of emailDomains) {
240+
const org = await orgService.findByDomain(tenantId, segmentId, domain as string)
241+
if (org) {
242+
organizationIds.push(org.id)
243+
}
244+
}
245+
246+
return organizationIds
247+
}
248+
201249
public async processMemberEnrich(
202250
tenantId: string,
203251
integrationId: string,

‎services/apps/data_sink_worker/src/service/organization.service.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,14 @@ export class OrganizationService extends LoggerBase {
189189
await this.repo.addToMember(memberId, orgIds)
190190
}
191191

192+
public async findByDomain(
193+
tenantId: string,
194+
segmentId: string,
195+
domain: string,
196+
): Promise<IOrganization> {
197+
return await this.repo.findByDomain(tenantId, segmentId, domain)
198+
}
199+
192200
public async processOrganizationEnrich(
193201
tenantId: string,
194202
integrationId: string,

‎services/apps/search_sync_worker/src/service/member.sync.service.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { SERVICE_CONFIG } from '@/conf'
22
import { IDbMemberSyncData } from '@/repo/member.data'
33
import { MemberRepository } from '@/repo/member.repo'
44
import { OpenSearchIndex } from '@/types'
5-
import { distinct, distinctBy, groupBy } from '@crowd/common'
5+
import { distinct, distinctBy, groupBy, trimUtf8ToMaxByteLength } from '@crowd/common'
66
import { DbStore } from '@crowd/database'
77
import { Logger, LoggerBase, logExecutionTime } from '@crowd/logging'
88
import { RedisClient } from '@crowd/redis'
@@ -433,21 +433,29 @@ export class MemberSyncService extends LoggerBase {
433433
p.keyword_displayName = data.displayName
434434
const p_attributes = {}
435435

436+
// max byte length that can be indexed in OpenSearch
437+
const maxByteLength = 25000
438+
436439
for (const attribute of attributes) {
437440
// eslint-disable-next-line @typescript-eslint/no-explicit-any
438441
const attData = data.attributes as any
439442

440443
if (attribute.name in attData) {
441444
if (attribute.type === MemberAttributeType.SPECIAL) {
442-
const data = JSON.stringify(attData[attribute.name])
445+
let data = JSON.stringify(attData[attribute.name])
446+
data = trimUtf8ToMaxByteLength(data, maxByteLength)
443447
p_attributes[`string_${attribute.name}`] = data
444448
} else {
445449
const p_data = {}
446450
const defValue = attData[attribute.name].default
447451
const prefix = this.attributeTypeToOpenSearchPrefix(defValue, attribute.type)
448452

449453
for (const key of Object.keys(attData[attribute.name])) {
450-
p_data[`${prefix}_${key}`] = attData[attribute.name][key]
454+
let value = attData[attribute.name][key]
455+
if (attribute.type === MemberAttributeType.STRING) {
456+
value = trimUtf8ToMaxByteLength(value, maxByteLength)
457+
}
458+
p_data[`${prefix}_${key}`] = value
451459
}
452460

453461
p_attributes[`obj_${attribute.name}`] = p_data

0 commit comments

Comments
 (0)