diff --git a/backend/src/database/migrations/U1693120114__trackManuallyCreatedOrgsAndMembers.sql b/backend/src/database/migrations/U1693120114__trackManuallyCreatedOrgsAndMembers.sql new file mode 100644 index 0000000000..52fcaaae59 --- /dev/null +++ b/backend/src/database/migrations/U1693120114__trackManuallyCreatedOrgsAndMembers.sql @@ -0,0 +1,3 @@ +ALTER TABLE public."organizations" DROP COLUMN "manuallyCreated"; +ALTER TABLE public."members" DROP COLUMN "manuallyCreated"; +ALTER TABLE public."organizationCaches" DROP COLUMN "manuallyCreated"; \ No newline at end of file diff --git a/backend/src/database/migrations/V1693120114__trackManuallyCreatedOrgsAndMembers.sql b/backend/src/database/migrations/V1693120114__trackManuallyCreatedOrgsAndMembers.sql new file mode 100644 index 0000000000..23d7ef7670 --- /dev/null +++ b/backend/src/database/migrations/V1693120114__trackManuallyCreatedOrgsAndMembers.sql @@ -0,0 +1,3 @@ +ALTER TABLE public."members" ADD COLUMN "manuallyCreated" boolean NOT NULL DEFAULT false; +ALTER TABLE public."organizations" ADD COLUMN "manuallyCreated" boolean NOT NULL DEFAULT false; +ALTER TABLE public."organizationCaches" ADD COLUMN "manuallyCreated" boolean NOT NULL DEFAULT false; \ No newline at end of file diff --git a/backend/src/database/models/member.ts b/backend/src/database/models/member.ts index f4dd404bca..469bdf0d8e 100644 --- a/backend/src/database/models/member.ts +++ b/backend/src/database/models/member.ts @@ -57,6 +57,11 @@ export default (sequelize) => { enrichedBy: { type: DataTypes.ARRAY(DataTypes.TEXT), }, + manuallyCreated: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + }, }, { indexes: [ diff --git a/backend/src/database/models/organization.ts b/backend/src/database/models/organization.ts index 21cb5146b8..8032c9855c 100644 --- a/backend/src/database/models/organization.ts +++ b/backend/src/database/models/organization.ts @@ -150,6 +150,11 @@ export default (sequelize) => { type: DataTypes.DATE, allowNull: true, }, + manuallyCreated: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + }, attributes: { type: DataTypes.JSONB, defaultValue: {}, diff --git a/backend/src/database/models/organizationCache.ts b/backend/src/database/models/organizationCache.ts index b917752cd4..d48a0e27a8 100644 --- a/backend/src/database/models/organizationCache.ts +++ b/backend/src/database/models/organizationCache.ts @@ -133,6 +133,11 @@ export default (sequelize) => { type: DataTypes.DATE, allowNull: true, }, + manuallyCreated: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + }, }, { indexes: [ diff --git a/backend/src/database/repositories/__tests__/memberRepository.test.ts b/backend/src/database/repositories/__tests__/memberRepository.test.ts index 7cef67543a..ac5d25d209 100644 --- a/backend/src/database/repositories/__tests__/memberRepository.test.ts +++ b/backend/src/database/repositories/__tests__/memberRepository.test.ts @@ -141,6 +141,7 @@ describe('MemberRepository tests', () => { numberOfOpenSourceContributions: 0, lastActivity: null, affiliations: [], + manuallyCreated: false, } expect(memberCreated).toStrictEqual(expectedMemberCreated) }) @@ -210,6 +211,7 @@ describe('MemberRepository tests', () => { organizations: [], joinedAt: new Date('2020-05-27T15:13:30Z'), affiliations: [], + manuallyCreated: false, } expect(memberCreated).toStrictEqual(expectedMemberCreated) }) @@ -272,6 +274,7 @@ describe('MemberRepository tests', () => { lastActive: null, lastActivity: null, affiliations: [], + manuallyCreated: false, } expect(memberCreated).toStrictEqual(expectedMemberCreated) @@ -506,6 +509,7 @@ describe('MemberRepository tests', () => { lastActive: null, lastActivity: null, affiliations: [], + manuallyCreated: false, } const memberById = await MemberRepository.findById(memberCreated.id, mockIRepositoryOptions) @@ -556,6 +560,7 @@ describe('MemberRepository tests', () => { organizations: [], joinedAt: new Date('2020-05-27T15:13:30Z'), affiliations: [], + manuallyCreated: false, } const memberById = await MemberRepository.findById( @@ -726,6 +731,7 @@ describe('MemberRepository tests', () => { delete member1Returned.activeDaysCount delete member1Returned.numberOfOpenSourceContributions delete member1Returned.affiliations + delete member1Returned.manuallyCreated member1Returned.segments = member1Returned.segments.map((s) => s.id) const found = await MemberRepository.memberExists( @@ -2985,6 +2991,7 @@ describe('MemberRepository tests', () => { lastActive: null, lastActivity: null, affiliations: [], + manuallyCreated: false, } expect(updatedMember).toStrictEqual(expectedMemberCreated) @@ -3083,6 +3090,7 @@ describe('MemberRepository tests', () => { reach: { total: -1 }, joinedAt: new Date(updateFields.joinedAt), affiliations: [], + manuallyCreated: false, } expect(updatedMember).toStrictEqual(expectedMemberCreated) @@ -3167,6 +3175,7 @@ describe('MemberRepository tests', () => { lastActive: null, lastActivity: null, affiliations: [], + manuallyCreated: false, } expect(member1).toStrictEqual(expectedMemberCreated) @@ -3283,6 +3292,7 @@ describe('MemberRepository tests', () => { lastActive: null, lastActivity: null, affiliations: [], + manuallyCreated: false, } expect(member1).toStrictEqual(expectedMemberCreated) diff --git a/backend/src/database/repositories/__tests__/organizationCacheRepository.test.ts b/backend/src/database/repositories/__tests__/organizationCacheRepository.test.ts index 0bc3a71e98..b50dfacfab 100644 --- a/backend/src/database/repositories/__tests__/organizationCacheRepository.test.ts +++ b/backend/src/database/repositories/__tests__/organizationCacheRepository.test.ts @@ -50,6 +50,7 @@ const toCreate = { founded: null, employeeCountByCountry: null, address: null, + manuallyCreated: false, } describe('OrganizationCacheCacheRepository tests', () => { diff --git a/backend/src/database/repositories/__tests__/organizationRepository.test.ts b/backend/src/database/repositories/__tests__/organizationRepository.test.ts index 4af2df2862..7e04ddf780 100644 --- a/backend/src/database/repositories/__tests__/organizationRepository.test.ts +++ b/backend/src/database/repositories/__tests__/organizationRepository.test.ts @@ -52,6 +52,7 @@ const toCreate = { employeeCountByCountry: null, address: null, profiles: null, + manuallyCreated: false, } async function createMembers(options) { diff --git a/backend/src/database/repositories/memberRepository.ts b/backend/src/database/repositories/memberRepository.ts index fbfba4d4cf..2ec69b47a1 100644 --- a/backend/src/database/repositories/memberRepository.ts +++ b/backend/src/database/repositories/memberRepository.ts @@ -85,6 +85,7 @@ class MemberRepository { 'score', 'reach', 'joinedAt', + 'manuallyCreated', 'importHash', ]), tenantId: tenant.id, diff --git a/backend/src/database/repositories/organizationCacheRepository.ts b/backend/src/database/repositories/organizationCacheRepository.ts index 5b9757bf50..1d53a8764b 100644 --- a/backend/src/database/repositories/organizationCacheRepository.ts +++ b/backend/src/database/repositories/organizationCacheRepository.ts @@ -40,6 +40,7 @@ class OrganizationCacheRepository { 'address', 'size', 'lastEnrichedAt', + 'manuallyCreated', ]), }, { diff --git a/backend/src/database/repositories/organizationRepository.ts b/backend/src/database/repositories/organizationRepository.ts index e79442203e..808412815e 100644 --- a/backend/src/database/repositories/organizationRepository.ts +++ b/backend/src/database/repositories/organizationRepository.ts @@ -120,6 +120,7 @@ class OrganizationRepository { 'founded', 'size', 'lastEnrichedAt', + 'manuallyCreated', ]), tenantId: tenant.id, @@ -630,9 +631,18 @@ class OrganizationRepository { if (filter.and) { filter.and.push({ - activityCount: { - gt: 0, - }, + or: [ + { + manuallyCreated: { + eq: true, + }, + }, + { + activityCount: { + gt: 0, + }, + }, + ], }) } @@ -685,6 +695,8 @@ class OrganizationRepository { } } + console.log('parsed organization query', JSON.stringify(parsed)) + const response = await options.opensearch.search({ index: OpenSearchIndex.ORGANIZATIONS, body: parsed, @@ -995,6 +1007,7 @@ class OrganizationRepository { 'isTeamOrganization', 'type', 'attributes', + 'manuallyCreated', ], 'organization', ), @@ -1093,6 +1106,7 @@ class OrganizationRepository { 'address', 'profiles', 'attributes', + 'manuallyCreated', ], 'organization', ), diff --git a/backend/src/services/__tests__/memberService.test.ts b/backend/src/services/__tests__/memberService.test.ts index a184c3d023..817373fa38 100644 --- a/backend/src/services/__tests__/memberService.test.ts +++ b/backend/src/services/__tests__/memberService.test.ts @@ -183,6 +183,7 @@ describe('MemberService tests', () => { enrichedBy: [], contributions: null, affiliations: [], + manuallyCreated: false, } expect(memberCreated).toStrictEqual(memberExpected) @@ -276,6 +277,7 @@ describe('MemberService tests', () => { reach: { total: -1 }, joinedAt: new Date('2020-05-28T15:13:30Z'), affiliations: [], + manuallyCreated: false, } expect(memberCreated).toStrictEqual(memberExpected) @@ -328,6 +330,7 @@ describe('MemberService tests', () => { reach: { total: 10, [PlatformType.GITHUB]: 10 }, joinedAt: new Date('2020-05-28T15:13:30Z'), affiliations: [], + manuallyCreated: false, } expect(memberCreated).toStrictEqual(memberExpected) @@ -379,6 +382,7 @@ describe('MemberService tests', () => { reach: { total: 20, [PlatformType.GITHUB]: 10, [PlatformType.TWITTER]: 10 }, joinedAt: new Date('2020-05-28T15:13:30Z'), affiliations: [], + manuallyCreated: false, } expect(memberCreated).toStrictEqual(memberExpected) @@ -430,6 +434,7 @@ describe('MemberService tests', () => { reach: { total: 20, [PlatformType.DISCORD]: 10, [PlatformType.TWITTER]: 10 }, joinedAt: new Date('2020-05-28T15:13:30Z'), affiliations: [], + manuallyCreated: false, } expect(memberCreated).toStrictEqual(memberExpected) @@ -508,6 +513,7 @@ describe('MemberService tests', () => { address: null, profiles: null, attributes: {}, + manuallyCreated: false, }) }) @@ -584,6 +590,7 @@ describe('MemberService tests', () => { address: null, profiles: null, attributes: {}, + manuallyCreated: false, }) }) @@ -664,6 +671,7 @@ describe('MemberService tests', () => { address: null, profiles: null, attributes: {}, + manuallyCreated: false, }) }) @@ -760,6 +768,7 @@ describe('MemberService tests', () => { address: null, profiles: null, attributes: {}, + manuallyCreated: false, }) }) @@ -860,6 +869,7 @@ describe('MemberService tests', () => { joinedAt: new Date('2020-05-28T15:13:30Z'), reach: { total: -1 }, affiliations: [], + manuallyCreated: false, } expect(memberUpdated).toStrictEqual(memberExpected) @@ -971,6 +981,7 @@ describe('MemberService tests', () => { joinedAt: new Date('2020-05-28T15:13:30Z'), reach: { total: -1 }, affiliations: [], + manuallyCreated: false, } expect(memberUpdated).toStrictEqual(memberExpected) @@ -1077,6 +1088,7 @@ describe('MemberService tests', () => { joinedAt: new Date('2020-05-28T15:13:30Z'), reach: { total: -1 }, affiliations: [], + manuallyCreated: false, } expect(memberUpdated).toStrictEqual(memberExpected) @@ -1255,6 +1267,7 @@ describe('MemberService tests', () => { updatedById: mockIServiceOptions.currentUser.id, reach: { total: -1 }, affiliations: [], + manuallyCreated: false, } expect(memberUpdated).toStrictEqual(memberExpected) @@ -1311,6 +1324,7 @@ describe('MemberService tests', () => { emails: [], attributes: {}, affiliations: [], + manuallyCreated: false, } expect(memberUpdated).toStrictEqual(memberExpected) @@ -1368,6 +1382,7 @@ describe('MemberService tests', () => { emails: [], attributes: {}, affiliations: [], + manuallyCreated: false, } expect(memberUpdated).toStrictEqual(memberExpected) @@ -1426,6 +1441,7 @@ describe('MemberService tests', () => { emails: [], attributes: {}, affiliations: [], + manuallyCreated: false, } expect(memberUpdated).toStrictEqual(memberExpected) @@ -1484,6 +1500,7 @@ describe('MemberService tests', () => { emails: [], attributes: {}, affiliations: [], + manuallyCreated: false, } expect(memberUpdated).toStrictEqual(memberExpected) @@ -1888,6 +1905,7 @@ describe('MemberService tests', () => { lastActivity: activityCreated, numberOfOpenSourceContributions: 0, affiliations: [], + manuallyCreated: false, } expect( @@ -2128,6 +2146,7 @@ describe('MemberService tests', () => { delete returnedMember1.activeDaysCount delete returnedMember1.numberOfOpenSourceContributions delete returnedMember1.affiliations + delete returnedMember1.manuallyCreated returnedMember1.segments = returnedMember1.segments.map((s) => s.id) @@ -2228,6 +2247,7 @@ describe('MemberService tests', () => { delete returnedMember1.activeDaysCount delete returnedMember1.numberOfOpenSourceContributions delete returnedMember1.affiliations + delete returnedMember1.manuallyCreated returnedMember1.segments = returnedMember1.segments.map((s) => s.id) diff --git a/frontend/src/modules/member/pages/member-form-page.vue b/frontend/src/modules/member/pages/member-form-page.vue index 82d87d701a..e80a08371a 100644 --- a/frontend/src/modules/member/pages/member-form-page.vue +++ b/frontend/src/modules/member/pages/member-form-page.vue @@ -117,6 +117,7 @@ import { import { onBeforeRouteLeave, useRoute, useRouter } from 'vue-router'; import isEqual from 'lodash/isEqual'; import { useStore } from 'vuex'; +import { storeToRefs } from 'pinia'; import AppMemberFormDetails from '@/modules/member/components/form/member-form-details.vue'; import AppMemberFormIdentities from '@/modules/member/components/form/member-form-identities.vue'; import AppMemberFormAttributes from '@/modules/member/components/form/member-form-attributes.vue'; @@ -129,7 +130,6 @@ import getCustomAttributes from '@/shared/fields/get-custom-attributes'; import getAttributesModel from '@/shared/attributes/get-attributes-model'; import getParsedAttributes from '@/shared/attributes/get-parsed-attributes'; import { useMemberStore } from '@/modules/member/store/pinia'; -import { storeToRefs } from 'pinia'; const LoaderIcon = h( 'i', @@ -383,6 +383,8 @@ async function onSubmit() { ...Object.keys(formModel.value.username).length && { username: formModel.value.username, }, + + manuallyCreated: true, }; let isRequestSuccessful = false; diff --git a/frontend/src/modules/organization/pages/organization-form-page.vue b/frontend/src/modules/organization/pages/organization-form-page.vue index cd5e178b5c..59b210a311 100644 --- a/frontend/src/modules/organization/pages/organization-form-page.vue +++ b/frontend/src/modules/organization/pages/organization-form-page.vue @@ -359,7 +359,7 @@ function platformPayload(platform, value) { async function onSubmit() { isFormSubmitting.value = true; const data = { - + manuallyCreated: true, ...formModel.value, name: isEditPage.value === false ? formModel.value.displayName : undefined, displayName: isEditPage.value === true ? formModel.value.displayName : undefined, diff --git a/services/apps/search_sync_worker/src/repo/member.data.ts b/services/apps/search_sync_worker/src/repo/member.data.ts index 91bcfc8830..a0e65d71e3 100644 --- a/services/apps/search_sync_worker/src/repo/member.data.ts +++ b/services/apps/search_sync_worker/src/repo/member.data.ts @@ -30,6 +30,7 @@ export interface IDbMemberSyncData { lastEnriched: string | null joinedAt: string createdAt: string + manuallyCreated: boolean totalReach: number numberOfOpenSourceContributions: number diff --git a/services/apps/search_sync_worker/src/repo/member.repo.ts b/services/apps/search_sync_worker/src/repo/member.repo.ts index aae5f0d804..7da2e8b013 100644 --- a/services/apps/search_sync_worker/src/repo/member.repo.ts +++ b/services/apps/search_sync_worker/src/repo/member.repo.ts @@ -55,7 +55,10 @@ export class MemberRepository extends RepositoryBase { m."searchSyncedAt" < $(cutoffDate) ) and - exists (select 1 from activities where "memberId" = m.id) and + ( + exists (select 1 from activities where "memberId" = m.id) or + m."manuallyCreated" + ) and exists (select 1 from "memberIdentities" where "memberId" = m.id) limit ${perPage} offset ${(page - 1) * perPage};`, { @@ -159,6 +162,7 @@ export class MemberRepository extends RepositoryBase { m.score, m."lastEnriched", m."joinedAt", + m."manuallyCreated", m."createdAt", (m.reach -> 'total')::integer as "totalReach", coalesce(jsonb_array_length(m.contributions), 0) as "numberOfOpenSourceContributions", @@ -178,13 +182,14 @@ export class MemberRepository extends RepositoryBase { from "memberSegments" ms inner join members m on ms."memberId" = m.id inner join identities i on m.id = i."memberId" - inner join activity_data ad on ms."memberId" = ad."memberId" and ms."segmentId" = ad."segmentId" + left join activity_data ad on ms."memberId" = ad."memberId" and ms."segmentId" = ad."segmentId" left join to_merge_data tmd on m.id = tmd."memberId" left join no_merge_data nmd on m.id = nmd."memberId" left join member_tags mt on ms."memberId" = mt."memberId" left join member_organizations mo on ms."memberId" = mo."memberId" and ms."segmentId" = mo."segmentId" where ms."memberId" in ($(ids:csv)) - and m."deletedAt" is null;`, + and m."deletedAt" is null + and (ad."memberId" is not null or m."manuallyCreated");`, { ids, }, @@ -209,7 +214,10 @@ export class MemberRepository extends RepositoryBase { from members m where m."tenantId" = $(tenantId ) and m.id in ($(memberIds:csv)) and - exists(select 1 from activities a where a."memberId" = m.id) and + ( + exists (select 1 from activities where "memberId" = m.id) or + m."manuallyCreated" + ) and exists(select 1 from "memberIdentities" mi where mi."memberId" = m.id) `, { diff --git a/services/apps/search_sync_worker/src/repo/organization.data.ts b/services/apps/search_sync_worker/src/repo/organization.data.ts index 235b08e8e4..a30ffab6d2 100644 --- a/services/apps/search_sync_worker/src/repo/organization.data.ts +++ b/services/apps/search_sync_worker/src/repo/organization.data.ts @@ -6,6 +6,7 @@ export interface IDbOrganizationSyncData { address: unknown | null attributes: unknown createdAt: string + manuallyCreated: boolean description: string | null displayName: string emails: string[] diff --git a/services/apps/search_sync_worker/src/repo/organization.repo.ts b/services/apps/search_sync_worker/src/repo/organization.repo.ts index 83db795526..73f70f5d42 100644 --- a/services/apps/search_sync_worker/src/repo/organization.repo.ts +++ b/services/apps/search_sync_worker/src/repo/organization.repo.ts @@ -40,6 +40,7 @@ export class OrganizationRepository extends RepositoryBase