diff --git a/services/libs/integrations/src/integrations/discord/api/errorHandler.ts b/services/libs/integrations/src/integrations/discord/api/errorHandler.ts index 3a71046b6d..e30c873e44 100644 --- a/services/libs/integrations/src/integrations/discord/api/errorHandler.ts +++ b/services/libs/integrations/src/integrations/discord/api/errorHandler.ts @@ -32,6 +32,12 @@ export const handleDiscordError = ( return new RateLimitError(rateLimitResetSeconds, url, err) } + + if (err && err.response && err.response.status === 403) { + logger.warn('No access to resourse, ignoring it', { input, err }) + return undefined + } + logger.error(err, { input }, `Error while calling Discord API URL: ${url}`) return err } diff --git a/services/libs/integrations/src/integrations/discord/api/getChannel.ts b/services/libs/integrations/src/integrations/discord/api/getChannel.ts index 2eb29b4fd9..053d2d3d19 100644 --- a/services/libs/integrations/src/integrations/discord/api/getChannel.ts +++ b/services/libs/integrations/src/integrations/discord/api/getChannel.ts @@ -2,12 +2,14 @@ import axios, { AxiosRequestConfig } from 'axios' import { handleDiscordError } from './errorHandler' import { DiscordApiChannel } from '../types' import { IProcessStreamContext } from '../../../types' +import { getRateLimiter } from './handleRateLimit' export const getChannel = async ( channelId: string, token: string, ctx: IProcessStreamContext, ): Promise => { + const rateLimiter = getRateLimiter(ctx) const config: AxiosRequestConfig = { method: 'get', url: `https://discord.com/api/v10/channels/${channelId}`, @@ -17,10 +19,14 @@ export const getChannel = async ( } try { + await rateLimiter.checkRateLimit('getChannel') + await rateLimiter.incrementRateLimit() const response = await axios(config) return response.data } catch (err) { const newErr = handleDiscordError(err, config, { channelId }, ctx) - throw newErr + if (newErr) { + throw newErr + } } } diff --git a/services/libs/integrations/src/integrations/discord/api/getChannels.ts b/services/libs/integrations/src/integrations/discord/api/getChannels.ts index 6a67e8287b..49044548c5 100644 --- a/services/libs/integrations/src/integrations/discord/api/getChannels.ts +++ b/services/libs/integrations/src/integrations/discord/api/getChannels.ts @@ -3,6 +3,8 @@ import { timeout } from '@crowd/common' import { DiscordApiChannel, DiscordGetChannelsInput, DiscordGetMessagesInput } from '../types' import getMessages from './getMessages' import { IProcessStreamContext } from '../../../types' +import { getRateLimiter } from './handleRateLimit' +import { handleDiscordError } from './errorHandler' /** * Try if a channel is readable @@ -31,14 +33,19 @@ async function getChannels( ctx: IProcessStreamContext, tryChannels = true, ): Promise { + const rateLimiter = getRateLimiter(ctx) + + const config = { + method: 'get', + url: `https://discord.com/api/v10/guilds/${input.guildId}/channels?`, + headers: { + Authorization: input.token, + }, + } + try { - const config = { - method: 'get', - url: `https://discord.com/api/v10/guilds/${input.guildId}/channels?`, - headers: { - Authorization: input.token, - }, - } + await rateLimiter.checkRateLimit('getChannels') + await rateLimiter.incrementRateLimit() const response = await axios(config) // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -69,8 +76,10 @@ async function getChannels( return result } catch (err) { - ctx.log.error({ err, input }, 'Error while getting channels from Discord') - throw err + const newErr = handleDiscordError(err, config, { input }, ctx) + if (newErr) { + throw newErr + } } } diff --git a/services/libs/integrations/src/integrations/discord/api/getMember.ts b/services/libs/integrations/src/integrations/discord/api/getMember.ts index 2ac3b866ee..713480f408 100644 --- a/services/libs/integrations/src/integrations/discord/api/getMember.ts +++ b/services/libs/integrations/src/integrations/discord/api/getMember.ts @@ -2,6 +2,7 @@ import axios, { AxiosRequestConfig } from 'axios' import { DiscordApiMember } from '../types' import { handleDiscordError } from './errorHandler' import { IProcessStreamContext } from '../../../types' +import { getRateLimiter } from './handleRateLimit' export const getMember = async ( guildId: string, @@ -9,6 +10,8 @@ export const getMember = async ( token: string, ctx: IProcessStreamContext, ): Promise => { + const rateLimiter = getRateLimiter(ctx) + // eslint-disable-next-line @typescript-eslint/no-explicit-any const config: AxiosRequestConfig = { method: 'get', @@ -19,10 +22,14 @@ export const getMember = async ( } try { + await rateLimiter.checkRateLimit('getMember') + await rateLimiter.incrementRateLimit() const response = await axios(config) return response.data } catch (err) { const newErr = handleDiscordError(err, config, { guildId, userId }, ctx) - throw newErr + if (newErr) { + throw newErr + } } } diff --git a/services/libs/integrations/src/integrations/discord/api/getMembers.ts b/services/libs/integrations/src/integrations/discord/api/getMembers.ts index d2397ebda4..80f62ca44e 100644 --- a/services/libs/integrations/src/integrations/discord/api/getMembers.ts +++ b/services/libs/integrations/src/integrations/discord/api/getMembers.ts @@ -1,11 +1,13 @@ import axios from 'axios' import { DiscordApiMember, DiscordGetMembersInput, DiscordGetMembersOutput } from '../types' import { IProcessStreamContext } from '../../../types' +import { getRateLimiter } from './handleRateLimit' async function getMembers( input: DiscordGetMembersInput, ctx: IProcessStreamContext, ): Promise { + const rateLimiter = getRateLimiter(ctx) try { let url = `https://discord.com/api/v10/guilds/${input.guildId}/members?limit=${input.perPage}` if (input.page !== undefined && input.page !== '') { @@ -19,6 +21,8 @@ async function getMembers( }, } + await rateLimiter.checkRateLimit('getMembers') + await rateLimiter.incrementRateLimit() const response = await axios(config) const records: DiscordApiMember[] = response.data const limit = parseInt(response.headers['x-ratelimit-remaining'], 10) diff --git a/services/libs/integrations/src/integrations/discord/api/getMessage.ts b/services/libs/integrations/src/integrations/discord/api/getMessage.ts index be447caae3..1bdd6e4f90 100644 --- a/services/libs/integrations/src/integrations/discord/api/getMessage.ts +++ b/services/libs/integrations/src/integrations/discord/api/getMessage.ts @@ -1,6 +1,7 @@ import axios, { AxiosRequestConfig } from 'axios' import { handleDiscordError } from './errorHandler' import { IProcessStreamContext } from '../../../types' +import { getRateLimiter } from './handleRateLimit' export const getMessage = async ( channelId: string, @@ -8,6 +9,8 @@ export const getMessage = async ( token: string, ctx: IProcessStreamContext, ) => { + const rateLimiter = getRateLimiter(ctx) + const config: AxiosRequestConfig = { method: 'get', url: `https://discord.com/api/v10/channels/${channelId}/messages/${messageId}`, @@ -17,6 +20,8 @@ export const getMessage = async ( } try { + await rateLimiter.checkRateLimit('getMessage') + await rateLimiter.incrementRateLimit() const response = await axios(config) return response.data } catch (err) { diff --git a/services/libs/integrations/src/integrations/discord/api/getMessages.ts b/services/libs/integrations/src/integrations/discord/api/getMessages.ts index ae5af8739d..19e8bc98d7 100644 --- a/services/libs/integrations/src/integrations/discord/api/getMessages.ts +++ b/services/libs/integrations/src/integrations/discord/api/getMessages.ts @@ -1,12 +1,14 @@ import axios from 'axios' import { DiscordApiMessage, DiscordParsedReponse, DiscordGetMessagesInput } from '../types' import { IProcessStreamContext } from '../../../types' +import { getRateLimiter } from './handleRateLimit' async function getMessages( input: DiscordGetMessagesInput, ctx: IProcessStreamContext, showError = true, ): Promise { + const rateLimiter = getRateLimiter(ctx) try { let url = `https://discord.com/api/v10/channels/${input.channelId}/messages?limit=${input.perPage}` if (input.page !== undefined && input.page !== '') { @@ -20,6 +22,8 @@ async function getMessages( }, } + await rateLimiter.checkRateLimit('getMessages') + await rateLimiter.incrementRateLimit() const response = await axios(config) const records: DiscordApiMessage[] = response.data diff --git a/services/libs/integrations/src/integrations/discord/api/getThreads.ts b/services/libs/integrations/src/integrations/discord/api/getThreads.ts index 6272bf5a39..da0a80e4c1 100644 --- a/services/libs/integrations/src/integrations/discord/api/getThreads.ts +++ b/services/libs/integrations/src/integrations/discord/api/getThreads.ts @@ -1,25 +1,31 @@ import axios from 'axios' import { DiscordApiChannel, DiscordGetChannelsInput } from '../types' import { IProcessStreamContext } from '../../../types' +import { getRateLimiter } from './handleRateLimit' +import { handleDiscordError } from './errorHandler' async function getThreads( input: DiscordGetChannelsInput, ctx: IProcessStreamContext, ): Promise { + const rateLimiter = getRateLimiter(ctx) + const config = { + method: 'get', + url: `https://discord.com/api/v10/guilds/${input.guildId}/threads/active?`, + headers: { + Authorization: input.token, + }, + } try { - const config = { - method: 'get', - url: `https://discord.com/api/v10/guilds/${input.guildId}/threads/active?`, - headers: { - Authorization: input.token, - }, - } - + await rateLimiter.checkRateLimit('getThreads') + await rateLimiter.incrementRateLimit() const response = await axios(config) return response.data.threads } catch (err) { - ctx.log.error({ err, input }, 'Error while getting threads from Discord') - throw err + const newErr = handleDiscordError(err, config, { input }, ctx) + if (newErr) { + throw newErr + } } } diff --git a/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts b/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts new file mode 100644 index 0000000000..9b4ae6f694 --- /dev/null +++ b/services/libs/integrations/src/integrations/discord/api/handleRateLimit.ts @@ -0,0 +1,9 @@ +import { IProcessStreamContext } from '../../../types' + +const DISCORD_RATE_LIMIT = 50 +const DISCORD_RATE_LIMIT_TIME = 1 +const REDIS_KEY = 'discord-request-count' + +export const getRateLimiter = (ctx: IProcessStreamContext) => { + return ctx.getRateLimiter(DISCORD_RATE_LIMIT, DISCORD_RATE_LIMIT_TIME, REDIS_KEY) +}