| 1 | /* eslint-disable ts/no-explicit-any */ |
| 2 | import type { EventStream } from './enum/event-stream.enum' |
| 3 | import type { EventTopic } from './enum/event-topic.enum' |
| 4 | import type { EventPayloadMap } from './event-payload.interface' |
| 5 | import { randomUUID } from 'node:crypto' |
| 6 | import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common' |
| 7 | import { Redis } from 'ioredis' |
| 8 | |
| 9 | export type EventEnvelope<TTopic extends EventTopic = EventTopic> = TTopic extends EventTopic ? { |
| 10 | eventId: string |
| 11 | topic: TTopic |
| 12 | version: number |
| 13 | occurredAt: string |
| 14 | source: string |
| 15 | idempotencyKey: string |
| 16 | payload: EventPayloadMap[TTopic] |
| 17 | } : never |
| 18 | |
| 19 | export interface ConsumerConfig { |
| 20 | group: string |
| 21 | consumer: string |
| 22 | streams: EventStream[] |
| 23 | topics?: EventTopic[] |
| 24 | handler: (envelope: EventEnvelope) => Promise<void> |
| 25 | maxRetries?: number |
| 26 | pollInterval?: number |
| 27 | } |
| 28 | |
| 29 | @Injectable() |
| 30 | export class EventStreamService implements OnModuleInit, OnModuleDestroy { |
| 31 | private readonly logger = new Logger(EventStreamService.name) |
| 32 | private readonly consumers: ConsumerConfig[] = [] |
| 33 | private running = false |
| 34 | |
| 35 | constructor(private readonly redis: Redis) {} |
| 36 | |
| 37 | async onModuleInit() { |
| 38 | this.running = true |
| 39 | for (const config of this.consumers) { |
| 40 | await this.ensureConsumerGroups(config) |
| 41 | this.startPolling(config).catch((err) => { |
| 42 | this.logger.error(err, `Polling failed for group ${config.group}`) |
| 43 | }) |
| 44 | } |
| 45 | } |
| 46 | |
| 47 | onModuleDestroy() { |
| 48 | this.running = false |
| 49 | } |
| 50 | |
| 51 | async emit<TTopic extends EventTopic>( |
| 52 | stream: EventStream, |
| 53 | topic: TTopic, |
| 54 | payload: EventPayloadMap[TTopic], |
| 55 | options?: { source?: string, idempotencyKey?: string }, |
| 56 | ): Promise<string> { |
| 57 | const envelope = { |
| 58 | eventId: randomUUID(), |
| 59 | topic, |
| 60 | version: 1, |
| 61 | occurredAt: new Date().toISOString(), |
| 62 | source: options?.source ?? 'aitoearn-server', |
| 63 | idempotencyKey: options?.idempotencyKey ?? randomUUID(), |
| 64 | payload, |
| 65 | } |
| 66 | |
| 67 | await this.redis.xadd(stream, '*', 'data', JSON.stringify(envelope)) |
| 68 | return envelope.eventId |
| 69 | } |
| 70 | |
| 71 | subscribe(config: ConsumerConfig): void { |
| 72 | this.consumers.push(config) |
| 73 | if (this.running) { |
| 74 | this.ensureConsumerGroups(config) |
| 75 | .then(() => this.startPolling(config)) |
| 76 | .catch((err) => { |
| 77 | this.logger.error(err, `Failed to start polling for group ${config.group}`) |
| 78 | }) |
| 79 | } |
| 80 | } |
| 81 | |
| 82 | private async ensureConsumerGroups(config: ConsumerConfig): Promise<void> { |
| 83 | for (const stream of config.streams) { |
| 84 | try { |
| 85 | await this.redis.xgroup('CREATE', stream, config.group, '$', 'MKSTREAM') |
| 86 | } |
| 87 | catch (err: any) { |
| 88 | if (!err.message?.includes('BUSYGROUP')) { |
| 89 | this.logger.error(err, `Failed to create consumer group ${config.group} on stream ${stream}`) |
| 90 | } |
| 91 | } |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | private async startPolling(config: ConsumerConfig): Promise<void> { |
| 96 | const pollInterval = config.pollInterval ?? 1000 |
| 97 | const maxRetries = config.maxRetries ?? 5 |
| 98 | |
| 99 | while (this.running) { |
| 100 | try { |
| 101 | const streamArgs = Array.from({ length: config.streams.length }, () => '>') |
| 102 | |
| 103 | const results = await this.redis.xreadgroup( |
| 104 | 'GROUP', |
| 105 | config.group, |
| 106 | config.consumer, |
| 107 | 'COUNT', |
| 108 | 10, |
| 109 | 'BLOCK', |
| 110 | pollInterval, |
| 111 | 'STREAMS', |
| 112 | ...config.streams, |
| 113 | ...streamArgs, |
| 114 | ) |
| 115 | |
| 116 | if (results) { |
| 117 | for (const [stream, messages] of results as Array<[string, Array<[string, string[]]>]>) { |
| 118 | for (const [id, fields] of messages) { |
| 119 | await this.processMessage(stream, id, fields, config, maxRetries) |
| 120 | } |
| 121 | } |
| 122 | } |
| 123 | } |
| 124 | catch (err: any) { |
| 125 | if (this.running) { |
| 126 | this.logger.error(err, `Poll error for group ${config.group}`) |
| 127 | await this.sleep(pollInterval) |
| 128 | } |
| 129 | } |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | private async processMessage( |
| 134 | stream: string, |
| 135 | id: string, |
| 136 | fields: string[], |
| 137 | config: ConsumerConfig, |
| 138 | maxRetries: number, |
| 139 | ): Promise<void> { |
| 140 | const dataField = this.extractField(fields, 'data') |
| 141 | if (!dataField) { |
| 142 | await this.redis.xack(stream, config.group, id) |
| 143 | return |
| 144 | } |
| 145 | |
| 146 | const envelope: EventEnvelope = JSON.parse(dataField) |
| 147 | if (config.topics?.length && !config.topics.includes(envelope.topic)) { |
| 148 | await this.redis.xack(stream, config.group, id) |
| 149 | return |
| 150 | } |
| 151 | |
| 152 | let retries = 0 |
| 153 | |
| 154 | while (retries <= maxRetries) { |
| 155 | try { |
| 156 | await config.handler(envelope) |
| 157 | await this.redis.xack(stream, config.group, id) |
| 158 | return |
| 159 | } |
| 160 | catch (err: any) { |
| 161 | retries++ |
| 162 | if (retries > maxRetries) { |
| 163 | this.logger.error(err, `Max retries exceeded for message ${id} in group ${config.group}`) |
| 164 | await this.sendToDlq(stream, envelope, err, retries) |
| 165 | await this.redis.xack(stream, config.group, id) |
| 166 | return |
| 167 | } |
| 168 | await this.sleep(1000 * retries) |
| 169 | } |
| 170 | } |
| 171 | } |
| 172 | |
| 173 | private async sendToDlq(stream: string, envelope: EventEnvelope, error: unknown, retries: number): Promise<void> { |
| 174 | const dlqStream = `${stream}:dlq` |
| 175 | const dlqPayload = { |
| 176 | originalEnvelope: JSON.stringify(envelope), |
| 177 | error: error instanceof Error ? error.message : String(error), |
| 178 | retries, |
| 179 | sentToDlqAt: new Date().toISOString(), |
| 180 | } |
| 181 | await this.redis.xadd(dlqStream, '*', 'data', JSON.stringify(dlqPayload)) |
| 182 | } |
| 183 | |
| 184 | private extractField(fields: string[], key: string): string | undefined { |
| 185 | for (let i = 0; i < fields.length; i += 2) { |
| 186 | if (fields[i] === key) { |
| 187 | return fields[i + 1] |
| 188 | } |
| 189 | } |
| 190 | return undefined |
| 191 | } |
| 192 | |
| 193 | private sleep(ms: number): Promise<void> { |
| 194 | return new Promise(resolve => setTimeout(resolve, ms)) |
| 195 | } |
| 196 | } |
| 197 |