// bufferService.ts import { BehaviorSubject, Observable, Subject, from } from "rxjs"; import mongoose, { Connection, Model } from "mongoose"; import { ConnectionState, Message, MessageLog, } from "../interfaces/general.interface"; import * as fs from 'fs'; export class BufferService { private bufferIdentifier!: string private messageStream: Subject; private connectionState: BehaviorSubject; private messageBuffered: Message[] = []; private messageModel!: Model private readonly dbUrl!: string private bufferLimit!: number constructor( messageFromApp: Subject, connectionStateSubject: BehaviorSubject, dbName: string, bufferLimit?: number ) { if (bufferLimit) { this.bufferLimit = bufferLimit } else { this.bufferLimit = 10000 // default buffer limit } this.bufferIdentifier = dbName this.messageStream = messageFromApp; this.connectionState = connectionStateSubject; this.setupSubscriptions(); // Note: The handle buffer will push the data in local array before pushing to mongo via initial check up model /* Disabled for now due to missing data not during transmision. The issue was suspected to be it's async nature of confusing timing when it was queued into the event queue. Sometimes the messages will be late to be saved */ // this.initializeDatabaseConnection(dbName) // .then(async (connection: mongoose.Connection) => { // const grpcMessageSchema = require("../models/message.schema"); // this.messageModel = connection.model( // "Message", // grpcMessageSchema // ); // await this.transferLocalBufferToMongoDB(); // transfer all data from local array into mongodb after the mongo setup is complete // }) // .catch((error) => { // console.error("Database initialization failed:", error); // // Implement retry logic or additional error handling here. Perhaps retry logic in the future... // }); } // to be exposed to acquire the messages public getMessages(): Observable { return this.messageStream as Observable; } public getStateObservable(): BehaviorSubject { return this.connectionState; } // To subscrcibe for the message stream as well as the connection state private setupSubscriptions(): void { this.messageStream.subscribe({ next: (message: Message) => this.handleIncomingMessage(message), error: (err) => console.error("Error in messageToBePublished subject:", err), complete: () => console.log("messageToBePublished subscription completed"), }); this.connectionState.subscribe({ next: (state: ConnectionState) => this.handleConnectionStateChanges(state), error: (err) => console.error("Error in connectionState subject:", err), complete: () => console.log("connectionState subscription completed"), }); } private async initializeDatabaseConnection( dbName: string ): Promise { try { console.log(`${this.dbUrl}${dbName}`); const connection: mongoose.Connection = await mongoose.createConnection( `${this.dbUrl}${dbName}` ); console.log(`Connected to ${this.dbUrl}${dbName}`); return connection; } catch (error) { console.error("Error connecting to MongoDB:", error); throw error; } } private handleIncomingMessage(message: any): void { if (this.connectionState.getValue().status === `LIMIT_EXCEEDED`) { // do nothing... Let handleConnectionStateChanges deal with this state } if (this.connectionState.getValue().status === "BUFFER") { this.fileManagement(message, 'buffer'); this.bufferMessage(message); } if (this.connectionState.getValue().status === "DIRECT_PUBLISH") { /* Note: Since the main outGoingMessage is being published irregardless of the connection state, so there's no need to do anything aside from releasing buffered messages which will be handled by handleConnectionStateChange */ // additional logic here } } private handleConnectionStateChanges(state: ConnectionState): void { console.log(`${this.bufferIdentifier}: ${this.connectionState.getValue().status}`); if (state.status === `LIMIT_EXCEEDED`) { console.log(`Limit exceed. Clearing buffered messages...`) let message: Message = { id: `test`, message: `Limit exceed. Please take care. Buffer Service Out!` } this.messageStream.next(message) // this.messageStream.unsubscribe() //destroy existing subscription this.messageBuffered = [] } if (state.status === "BUFFER") { if (state.payload && typeof state.payload !== "string") { this.bufferMessage(state.payload); // Buffer the last message immediately } } if (state.status === "DIRECT_PUBLISH") { // Relese the messages by inserting them into the outgoing Messages together. this.releaseBufferedMessages(this.messageStream); } // testing if(state.status === "TARGET_PUBLISH") { this.fileManagement(state, 'release'); this.releaseTargetBufferedMessage(state); } } releaseTargetBufferedMessage(message: ConnectionState) { if(this.bufferIdentifier === "file") { } let target = this.messageBuffered.find((data: Message) => { if(data.id === message.uuid) { return data; } }) if(target) { this.messageBuffered = this.messageBuffered.filter((data: Message) => { if(data.id !== message.uuid) { return data; } }) } console.log(`TESTING : Releasing buffer Message under ${this.bufferIdentifier}: currently there is ${this.messageBuffered.length} messages to be released`); } fileManagement(message: any, type: 'buffer' | 'release') { let filePath = "./_LocalStorage/retransmissionData.json"; // DECLARATION : status chain let checkStatus: boolean = true; let errorMessage: any; let fileContent: any; if(checkStatus === true) { if(this.bufferIdentifier !== "file") { errorMessage = "not file storage, skip."; checkStatus = false; } } // CONDITION : check state log file exists or not if(checkStatus === true) { if(fs.existsSync(filePath)) { fileContent = fs.readFileSync(filePath, 'utf-8'); } else{ fileContent = JSON.stringify({ events: [] }); } } // CONDITION : fileContent is empty if(checkStatus === true) { if(fileContent === undefined && fileContent === null || fileContent === "") { fileContent = JSON.stringify({ events: [] }); } } // TASK : json parse fileContent if(checkStatus === true) { fileContent = JSON.parse(fileContent); } // TASK : add new message into fileContent if(checkStatus === true) { if(type === "buffer") { fileContent.events.push(message); } } // TASK : add new message into fileContent if(checkStatus === true) { if(type === "release") { fileContent.events = fileContent.events.filter((data) => { if(data.id !== message.uuid) { return data } }) } } // TASK : write file if(checkStatus === true) { try { fs.writeFileSync(filePath, JSON.stringify(fileContent, null, 4), 'utf-8'); console.log('\u001b[' + 32 + 'm' + "RE-TRANSMISSION FILE : ADDED NEW DATA..." + '\u001b[0m'); }catch(error) { errorMessage = error; checkStatus = false; } } // CONDITION : ERROR if(checkStatus === false) { console.log('\u001b[' + 31 + 'm' + "RE-TRANSMISSION FILE : ERROR -> " + JSON.stringify(errorMessage,null,4) + '\u001b[0m'); } } private async bufferMessage(message: any): Promise { if (this.messageModel) { try { // const newMessage = new this.messageModel(message); await this.messageModel.create(message); this.messageModel.countDocuments({}).then((count) => { console.log(`Message${(message.message as MessageLog).appData.msgId} saved to MongoDB buffer. There is ${count} messages in datatbase under ${this.bufferIdentifier} at the moment.`); // if connection status okay // if(this.connectionState.getValue().status == "DIRECT_PUBLISH") // { // console.log("Message count release " + count); // // Then release back to message stream // this.releaseBufferedMessages(this.messageStream); // } }); } catch (error) { console.error("Error saving message to MongoDB:", error); // Implement retry logic or additional error handling here } } else { if (this.bufferLimit > this.messageBuffered.length) { this.messageBuffered.push(message); // console.log(this.messageBuffered) // Fallback to local buffer if model is not defined console.log(`pushing into local array buffer under ${this.bufferIdentifier}.... There is now ${this.messageBuffered.length} messages`); } else { let reportState: ConnectionState = { status: `LIMIT_EXCEEDED`, reason: `${this.bufferLimit} Limit exceeded. Buffer Service will be terminated...` } this.connectionState.next(reportState) } } } private releaseBufferedMessages( messageFromBuffer: Subject ): Promise { return new Promise(async (resolve, reject) => { if (this.messageModel) { try { // use then let countPromise = checkMessageCount(this.messageModel, this.bufferIdentifier); countPromise.then(async (amount) => { console.log("Amount1:" + amount); // let countPromise = checkMessageCount(this.messageModel, this.bufferIdentifier); // countPromise.then(async (amount)=>{ // console.log("Amount2:"+amount); // }) while (amount > 0) { console.log("AmountInLoop1:" + amount) try { await extractData(messageFromBuffer, this.messageModel); // New function to process a batch } catch (processError) { console.error('Error processing batch:', processError); } console.log('Checking again...'); amount = await checkMessageCount(this.messageModel, this.bufferIdentifier); console.log("AmountInLoop:" + amount) } console.log('All messages extracted.'); }) let amount: number = await countPromise resolve(true); } catch (error) { console.error('Error in releaseBufferedMessages:', error); reject(false); } async function checkMessageCount(messageModel: Model, bufferIdentifier: string): Promise { return new Promise((resolve, reject) => { messageModel.countDocuments({}).then((count) => { console.log(`There is ${count} messages in database under ${bufferIdentifier} at the moment. Releasing them....`); resolve(count) }).catch((error) => { console.error(error) reject(error) }) }) } // Stream all the data inside the database out and deleting them async function extractData(messageFromBuffer: Subject, messageModel: Model): Promise { return new Promise((resolve, reject) => { const stream = messageModel.find().cursor(); stream.on("data", async (message) => { // Process each message individually` messageFromBuffer.next(message); }); stream.on("error", (error) => { console.error("Error streaming messages from MongoDB:", error); reject(error); }); stream.on("end", async () => { // Delete the data once it has been streamed try { if (messageModel) { await messageModel.deleteMany({}); console.log("Data in Mongo deleted successfully."); } else { console.log(`Message Mongoose Model is not intiated properly...`); } } catch (err) { console.error("Error deleting data:", err); } resolve(true); }); }) } } if (!this.messageModel) { // If MongoDB model is not defined, use the local buffer console.log(`Releasing buffer Message under ${this.bufferIdentifier}: currently there is ${this.messageBuffered.length} messages to be released`); this.messageBuffered.forEach((message) => this.messageStream.next(message) ); this.messageBuffered.length = 0; // Clear the local buffer after transferring if (this.messageBuffered.length < 1) { resolve(true); } else { reject(`Somehow the array is not emptied. This should not happen`); } } }); } private async transferLocalBufferToMongoDB(): Promise { return new Promise((resolve, reject) => { console.log(`Transferring local array buffered Message under ${this.bufferIdentifier}: currently there is ${this.messageBuffered.length}. Transferring to database...`); if (this.messageModel) { let locallyBufferedMessage: Observable = from(this.messageBuffered); locallyBufferedMessage.subscribe({ next: async (message: Message) => { try { if (this.messageModel) { await this.messageModel.create(message); console.log(`Transferring ${(message.message as MessageLog).appData.msgId} into database.`); } } catch (error) { console.error("Error transferring message to MongoDB:", error); } }, error: (err) => console.error(err), complete: () => { if (this.messageModel) { this.messageModel.countDocuments({}).then((count) => { console.log(`Local buffered message transfer completed. There is a total of ${count} messages in database ${this.bufferIdentifier} at the moment.`) this.messageBuffered = [] // Clear local buffer after transferring }); } }, }); } }); } // Additional methods as required... }