Merge branch 'master' of https://swopt.com:3000/enzo/Fis-AuditMessage

.env

@@ -0,0 +1 @@
+CheckAudit = Yes

documentations/notificationSynchronization.plantuml

@@ -0,0 +1,51 @@
+@startuml NotificationSynchronizatoin
+header Notificaton Synchronization Diagram
+title Notification Synchronization 
+left to right direction
+skinparam backgroundColor #EEEBDC
+
+(CentralRedist) as (centralredis) << REDIST >>
+usecase "**CentralRedist**" as centralredis #fa9dd0;line:Black;line.bold;text:Black
+
+package Primary {
+    usecase "**BackEnd**" as be1 <<BE>> #fdff80;line:Black;line.dashed;text:Black
+    usecase "**ContentDeliveryManagementServer**" as cdms1 <<cdms>> #a4fa91;line:Black;line.dotted:Black
+    database MongoDB as mongo1 #aaeefa;line:Black;line.bold;text:Black
+    database FISdatabase as fisdb1 #d0b0ff;line:Black;text:Black
+    card "**UCP**" as ucp1 #d0b0ff;line:Black;line.bold;text:Black 
+    frame "FingerPrint UI" as fp1
+    card "**ProgramWorkflow**" as pw1 <<FisObserver>> #91adfa;line:Blue;line.bold;text:Black;
+}
+
+package Secondary {
+    usecase "**BackEnd**" as be2 <<BE>> #fdff80;line:Black;line.dashed;text:Black
+    usecase "**ContentDeliveryManagementServer**" as cdms2 <<cdms>> #a4fa91;line:Black;line.dotted:Black
+    database MongoDB as mongo2 #aaeefa;line:Black;line.bold;text:Black
+    database FISdatabase as fisdb2 #d0b0ff;line:Black;text:Black
+    card "**UCP**" as ucp2 #d0b0ff;line:Black;line.bold;text:Black
+    frame "FingerPrint UI" as fp2
+    card "**ProgramWorkflow**" as pw2 <<FisObserver>> #91adfa;line:Blue;line.bold;text:Black
+} 
+
+' Primary Side
+fp1 ..> pw1 :User entry
+ucp1 <- pw1 : publish \nNotifications
+cdms1 <-- pw1 
+be1 <-- ucp1 
+be1 -> fisdb1
+mongo1 <- cdms1
+
+ucp1 <==> centralredis #DarkRed;line.bold;text:green
+
+' Secondary Side
+fp2 <.. pw2 :User view changes 
+pw2 <- ucp2 : receive subscribed \nNotifications
+cdms2 <-- pw2 
+be2 <-- ucp2  
+fisdb2 <- be2
+cdms2 -> mongo2 
+ 
+centralredis <==> ucp2 #DarkRed;line.bold;text:green 
+  
+@enduml
+  

package-lock.json

@@ -10,8 +10,9 @@
       "license": "ISC",
       "dependencies": {
         "@types/node": "^18.11.18",
-        "dotenv": "^16.0.3",
+        "dotenv": "^16.3.1",
         "jsonschema": "^1.4.1",
+        "lodash": "^4.17.21",
         "mongoose": "^6.9.0",
         "rfdc": "^1.3.0",
         "rxjs": "^7.8.1",
@@ -1192,11 +1193,14 @@
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
     "node_modules/dotenv": {
-      "version": "16.0.3",
-      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz",
-      "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==",
+      "version": "16.3.1",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz",
+      "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==",
       "engines": {
         "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/motdotla/dotenv?sponsor=1"
       }
     },
     "node_modules/fast-xml-parser": {
@@ -1255,6 +1259,11 @@
         "node": ">=12.0.0"
       }
     },
+    "node_modules/lodash": {
+      "version": "4.17.21",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
+    },
     "node_modules/memory-pager": {
       "version": "1.5.0",
       "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
@@ -2440,9 +2449,9 @@
       }
     },
     "dotenv": {
-      "version": "16.0.3",
-      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz",
-      "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ=="
+      "version": "16.3.1",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz",
+      "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ=="
     },
     "fast-xml-parser": {
       "version": "4.0.11",
@@ -2473,6 +2482,11 @@
       "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.5.1.tgz",
       "integrity": "sha512-7jFxRVm+jD+rkq3kY0iZDJfsO2/t4BBPeEb2qKn2lR/9KhuksYk5hxzfRYWMPV8P/x2d0kHD306YyWLzjjH+uA=="
     },
+    "lodash": {
+      "version": "4.17.21",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
+    },
     "memory-pager": {
       "version": "1.5.0",
       "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",

package.json

@@ -15,6 +15,7 @@
     "start3b": "node test/test3b.js",
     "start3c": "node test/test3c.js",
     "start4": "node test/test4.js",
+    "start5": "node test/test5.js",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "repository": {
@@ -25,8 +26,9 @@
   "license": "ISC",
   "dependencies": {
     "@types/node": "^18.11.18",
-    "dotenv": "^16.0.3",
+    "dotenv": "^16.3.1",
     "jsonschema": "^1.4.1",
+    "lodash": "^4.17.21",
     "mongoose": "^6.9.0",
     "rfdc": "^1.3.0",
     "rxjs": "^7.8.1",

services/acknowledgement.service.ts

@@ -1,9 +1,8 @@
-import { AnyObject } from "mongoose";
 import { map, Observable, of, tap } from "rxjs";
 import { Acknowledgemeent, AcknowledgementLogSetting } from "../type/acknowledgement.interface";
-import { BaseMessage, Command, FisCreateMessageUtility, ResponseMessage, Uuid } from "../dependencies/msgutil/interface/export";
 import { LogSetting, MessageLog } from "../dependencies/log/type/datatype";
 import { LoggingService } from "../dependencies/log/interface/export";
+import { BaseMessage, Command, FisCreateMessageUtility, ResponseMessage, Uuid } from "../dependencies/log/dependencies/msgutil/interface/export";
 
 
 /**

services/incomingMessage.service.ts

@@ -1,8 +1,8 @@
 import { map, Observable, of, tap } from "rxjs";
 import { IncomingMessageServiceInterface } from "../type/datatype";
-import { LoggingService } from "../dependencies/log/interface/export";
 import { LogSetting, MessageLog } from "../dependencies/log/type/datatype";
-import { BaseMessage, Uuid } from "../dependencies/msgutil/interface/export";
+import { BaseMessage, Uuid } from "../dependencies/log/dependencies/msgutil/interface/export";
+import { LoggingService } from "../dependencies/log/interface/export";
 
 /**
  * @deprecated The logging is now supported by the Fis-Logging library.

services/message-auditor.service.ts

@@ -1,8 +1,11 @@
 import { map, Observable, of, Subject } from "rxjs";
 import { ErrorTrigger, MessageAuditorServiceInterface, MessageSynchronisationServiceSetting } from "../type/datatype";
-import { LoggingService } from "../dependencies/log/interface/export";
-import { BaseMessage } from "../dependencies/msgutil/interface/export";
 import { MessageLog } from "../dependencies/log/type/datatype";
+import * as _ from 'lodash'
+import { LoggingService } from "../dependencies/log/interface/export";
+import { BaseMessage } from "../dependencies/log/dependencies/msgutil/interface/export";
+let processedMsgIds = new Set();
+require('dotenv').config();
 
 export class MessageAuditorService implements MessageAuditorServiceInterface {
     private settings: MessageSynchronisationServiceSetting
@@ -14,6 +17,9 @@ export class MessageAuditorService implements MessageAuditorServiceInterface {
     the data to be synchronized */
     public init(settings: MessageSynchronisationServiceSetting): void {
         this.settings = settings;
+        if (settings.filters) {
+            console.log(`Integrating filters: ${Object.keys(this.settings.filters)} in AuditMessage service`)
+        }
     }
 
     /* This is the main interface of the message sync service. The argument will take in an observable stream of 
@@ -26,6 +32,11 @@ export class MessageAuditorService implements MessageAuditorServiceInterface {
         obsTrigger.subscribe({
             next: obsTrigger => {
                 console.log(obsTrigger.message)// just checking the message
+                if (!this.settings.filters) {
+                    console.log(`No filters applies`)
+                } else {
+                    console.log(`Synchronizating with filters: '${Object.keys(this.settings.filters)}': '${Object.values(this.settings.filters)}'`)
+                }
                 let missingMsg: Observable<MessageLog> = this.synchronize()
                 missingMsg.subscribe({
                     next: element => {
@@ -38,14 +49,69 @@ export class MessageAuditorService implements MessageAuditorServiceInterface {
         return this.missingMessageSubject
     }
 
+
+    /* ________________ Private Functions _________________ */
+    // Filtering functions to filters out messages
+    private filterData(filters: any, message: MessageLog): boolean {
+        let response: boolean = true //Just using this like a statemanagement
+        let payload: BaseMessage = JSON.parse(message.appData.msgPayload as string) // Extract the payload from the messageLog first
+        this.checkIfIsInPayloadDataFormat(payload) // Convert stringified nested payload if there's any
+        // Making a separate function to cater to different multi filters conditions are coded below
+        if (filters) { // if filters is not null
+            if (Object.keys(filters).length > 1) {
+                let totalCount = Object.keys(filters).length
+                let matchedCount = 0
+                Object.entries(filters).forEach(([key, value]) => {
+                    let filters = { [key]: value }
+                    // console.log(filters)
+                    if (this.checkValues(payload, filters) == true) matchedCount++
+                })
+                if (totalCount == matchedCount) { // check if all the criterias are met
+                    response = true
+                } else {
+                    response = false
+                }
+            } else {
+                if (this.checkValues(payload, filters) == true) {
+                    response = true
+                } else {
+                    response = false
+                }
+            }
+        } else { // if not filters is provided. Then the just let response be true so that the data can be further processed
+            response = true
+        }
+        return response
+    }
+
     /* This is where the 'synching' operation takes place. */
     private synchronize(): Subject<MessageLog> {
         let subjectOutput: Subject<MessageLog> = new Subject()
         // Acquire the data from both location and return them as an array respectively.
         this.acquireData().then((data: { arr1: MessageLog[], arr2: MessageLog[] }) => {
             // In the case where there are differences in the array length, then extensive comparison
-            // will be carried out to filter out the differences. Differences are the missing data.
+            // will be carried out to filters out the differences. Differences are the missing data.
+
+            if(process.env.CheckAudit)
+            {
+                console.log("Record set 1: ", _.keys(_.countBy(data.arr1,function(data:MessageLog){return data.appData['msgId']})).length);
+                console.log("Record set 2: ", _.keys(_.countBy(data.arr2,function(data:MessageLog){return data.appData['msgId']})).length);
+            }
+
+
+            if(process.env.CheckAudit)
+            {
+                console.log("[CheckAudit] Record set 1: ", _.keys(_.countBy(data.arr1,function(data:MessageLog){return data.appData['msgId']})).length);
+                console.log("[CheckAudit] Record set 2: ", _.keys(_.countBy(data.arr2,function(data:MessageLog){return data.appData['msgId']})).length);
+            }
+
             this.checkArrayDifferences(data).then((data: MessageLog[]) => {
+
+                if(process.env.CheckAudit)
+                {
+                    console.log("Difference: ",data.length);
+                }
+
                 data.forEach(msgElement => {
                     let refined = JSON.parse(JSON.stringify(msgElement))
                     // Once the missing data has been weeded out, it is then passed into the Subject 
@@ -61,25 +127,28 @@ export class MessageAuditorService implements MessageAuditorServiceInterface {
     private async acquireData(): Promise<any> {
         const promiseQuery: Promise<any> = new Promise((resolve, reject) => {
             // declare what to expect.
-            let allSets: {
-                arr1: MessageLog[],
-                arr2: MessageLog[]
-            } = {
+            let allSets: { arr1: MessageLog[], arr2: MessageLog[] } = {
                 arr1: [],
                 arr2: []
             }
-            let set1: MessageLog[]
-            let set2: MessageLog[]
+            let set1: MessageLog[] = []
+            let set2: MessageLog[] = []
 
             // Initiate the source to find the location of the targeted data to be synched.
             this.sourceSrc.init(this.settings.incomingSource).then(() => {
                 this.targetSrc.init(this.settings.target).then(() => {
                     // Filter also carries out the query aspect of the operation, allowing it to acquire all the relevant data.
                     this.sourceSrc.filter({ msgTag: this.settings.incomingSource.tags[0] }).then((data: MessageLog[]) => {
-                        set1 = data
+                        data.forEach((message: MessageLog) => {
+                            if (this.filterData(this.settings.filters, message)) set1.push(message)
+                        })
+                    }).catch((err) => {
+                        console.error(err.message)
                     }).then(() => {
                         this.targetSrc.filter({ msgTag: this.settings.target.tags[0] }).then((data: MessageLog[]) => {
-                            set2 = data
+                            data.forEach(message => {
+                                if (this.filterData(this.settings.filters, message)) set2.push(message)
+                            })
                             allSets.arr1 = set1
                             allSets.arr2 = set2
                             resolve(allSets)
@@ -92,7 +161,7 @@ export class MessageAuditorService implements MessageAuditorServiceInterface {
     }
 
     // compare results and return differences
-    private async checkArrayDifferences(args: { arr1?: any[], arr2?: any[] }): Promise<MessageLog[]> {
+    private async checkArrayDifferences(args: { arr1: MessageLog[], arr2: MessageLog[] }): Promise<MessageLog[]> {
         return new Promise((resolve, reject) => {
             let missingMsg: MessageLog[] = []
             args.arr1.forEach((msgElement: MessageLog) => {
@@ -110,5 +179,48 @@ export class MessageAuditorService implements MessageAuditorServiceInterface {
         })
     }
 
+    // To be used by the filterData function to check between payload values and filter conditions
+    private checkValues(payload, filters): boolean { //FYI, all parameters are string
+        let key = Object.keys(filters)
+        // console.log(Object.values(filters))
+        let value = Object.values(filters)[0]
+        let res = _.get(payload, key[0])
+        // Check first if the payload has the filtering properties/path
+        if (_.has(payload, key[0])) {
+            let strarray: string[]
+            // check array 
+            if (Array.isArray(value)) {
+                strarray = value as string[]
+            }
+            else {
+                strarray = [value as string]
+            }
+            // compare array with that string 
+            if (strarray.includes(res)) {
+                return true
+            } else {
+                return false
+            }
+        } else {
+            console.log(`${key} does not exists in payload`)
+            return false
+        }
+    }
+
+    // Check in the case of notification messages, for the nested data properties
+    // Notification message may have multiple nested data properties that maybe in string format
+    private checkIfIsInPayloadDataFormat(payload: BaseMessage | any) {
+        let parsedData: any
+        if (payload.data
+            && payload.data.data
+            && payload.data.data.data && typeof payload.data.data.data === 'string') {
+            parsedData = JSON.parse(payload.data.data.data)
+            // console.log(parsedData)
+            payload.data.data.data = parsedData
+            return payload
+        } else {
+            return payload
+        }
+    }
 
 }

test/test-streamOBS.ts

@@ -1,12 +1,12 @@
 /* ----------------------        Simulate a stream of messages to be inserted or used by the test        ---------------------- */
 
 
-import { from, map, Observable, of, Subject } from "rxjs";
+import { Subject } from "rxjs";
 import * as fs from "fs"
-import { BaseMessage } from "../dependencies/msgutil/interface/export";
+import { BaseMessage } from "../dependencies/log/dependencies/msgutil/interface/export";
 
 export class StreamingService {
-    private messagesJSON: any = fs.readFileSync("testRequest.json")
+    private messagesJSON: any = fs.readFileSync("log.json")
     private messages = JSON.parse(this.messagesJSON)
 
     public stream(): Subject<BaseMessage> {

test/test1a.ts

@@ -1,13 +1,9 @@
-import { AcknowledgementService } from "../services/acknowledgement.service";
 import { StreamingService } from "./test-streamOBS";
-import { MessageAuditorService } from "../services/message-auditor.service";
 import { IncomingMessageService } from "../services/incomingMessage.service";
-import { map, Observable } from "rxjs";
-import { BaseMessage } from "../dependencies/msgutil/interface/export";
+import { Observable } from "rxjs";
 import { LogSetting } from "../dependencies/log/type/datatype";
+import { BaseMessage } from "../dependencies/log/dependencies/msgutil/interface/export";
 const incoming = new IncomingMessageService()
-const acknowledge = new AcknowledgementService()
-const syncrhonize = new MessageAuditorService()
 const streamService = new StreamingService()
 
 /* --------------  TEST -------------------- */

test/test1b.ts

@@ -3,12 +3,10 @@ import { StreamingService } from "./test-streamOBS";
 import { MessageAuditorService } from "../services/message-auditor.service";
 import { IncomingMessageService } from "../services/incomingMessage.service";
 import { map, Observable } from "rxjs";
-import { BaseMessage } from "../dependencies/msgutil/interface/export";
 import { LogSetting } from "../dependencies/log/type/datatype";
+import { BaseMessage } from "../dependencies/log/dependencies/msgutil/interface/export";
 
-const incoming = new IncomingMessageService()
 const acknowledge = new AcknowledgementService()
-const syncrhonize = new MessageAuditorService()
 const streamService = new StreamingService()
 
 /* --------------  TEST -------------------- */

test/test2a.ts

@@ -1,13 +1,9 @@
-import { AcknowledgementService } from "../services/acknowledgement.service";
 import { StreamingService } from "./test-streamOBS";
-import { MessageAuditorService } from "../services/message-auditor.service";
 import { IncomingMessageService } from "../services/incomingMessage.service";
-import { map, Observable, take } from "rxjs";
-import { BaseMessage } from "../dependencies/msgutil/interface/export";
+import { Observable, take } from "rxjs";
 import { LogSetting } from "../dependencies/log/type/datatype";
+import { BaseMessage } from "../dependencies/log/dependencies/msgutil/interface/export";
 const incoming = new IncomingMessageService()
-const syncrhonize = new MessageAuditorService()
-const acknowledge = new AcknowledgementService()
 const streamService = new StreamingService()
 
 /* --------------  TEST -------------------- */
@@ -29,7 +25,7 @@ let storage: LogSetting = {
         password: "h1nt1OyXw6QeUnzS",
         server: "cluster0.29sklte.mongodb.net",
         database: "log",
-      }
+    }
 }
 let dataSet: LogSetting & { incomingObservable: Observable<BaseMessage> } = {
     storage: storage.storage,

test/test2b.ts

@@ -1,12 +1,8 @@
 import { AcknowledgementService } from "../services/acknowledgement.service";
 import { StreamingService } from "./test-streamOBS";
-import { MessageAuditorService } from "../services/message-auditor.service";
-import { IncomingMessageService } from "../services/incomingMessage.service";
 import { map, Observable, take } from "rxjs";
-import { BaseMessage } from "../dependencies/msgutil/interface/export";
 import { LogSetting } from "../dependencies/log/type/datatype";
-const incoming = new IncomingMessageService()
-const syncrhonize = new MessageAuditorService()
+import { BaseMessage } from "../dependencies/log/dependencies/msgutil/interface/export";
 const acknowledge = new AcknowledgementService()
 const streamService = new StreamingService()
 

test/test3a.ts

@@ -8,8 +8,8 @@ import { IncomingMessageService } from "../services/incomingMessage.service";
 import { MessageAuditorService } from "../services/message-auditor.service";
 import { ErrorTrigger, MessageSynchronisationServiceSetting } from "../type/datatype";
 import { StreamingService } from "./test-streamOBS";
-import { BaseMessage } from "../dependencies/msgutil/interface/export";
 import { LogSetting } from "../dependencies/log/type/datatype";
+import { BaseMessage } from "../dependencies/log/dependencies/msgutil/interface/export";
 
 /* Pre - Defined Data && Settings */
 const stream = new StreamingService()

test/test3b.ts

@@ -9,8 +9,8 @@ import { IncomingMessageService } from "../services/incomingMessage.service";
 import { MessageAuditorService } from "../services/message-auditor.service";
 import { ErrorTrigger, MessageSynchronisationServiceSetting } from "../type/datatype";
 import { StreamingService } from "./test-streamOBS";
-import { BaseMessage } from "../dependencies/msgutil/interface/export";
 import { LogSetting, MessageLog } from "../dependencies/log/type/datatype";
+import { BaseMessage } from "../dependencies/log/dependencies/msgutil/interface/export";
 
 /* Pre - Defined Data && Settings */
 const stream = new StreamingService()

test/test3c.ts

@@ -3,13 +3,13 @@
 Which is local file mongo as the control/source, and then comparing the data from cloud mongoDB
 server data, and then synchronizing them */
 
-import { Observable, map, Subject, takeUntil, take, of, timer, from } from "rxjs";
+import { Observable, Subject, take } from "rxjs";
 import { ErrorTrigger, MessageSynchronisationServiceSetting } from "../type/datatype";
 import { StreamingService } from "./test-streamOBS";
 import { MessageAuditorService } from "../services/message-auditor.service";
-import { BaseMessage, ResponseMessage } from "../dependencies/msgutil/interface/export";
-import { LoggingService } from "../dependencies/log/interface/export";
 import { LogSetting, MessageLog } from "../dependencies/log/type/datatype";
+import { BaseMessage, ResponseMessage } from "../dependencies/log/dependencies/msgutil/interface/export";
+import { LoggingService } from "../dependencies/log/interface/export";
 
 /* Pre - Defined Data && Settings */
 // This service will stream the messages from the local testRequest.json messages
@@ -62,8 +62,11 @@ let publisher_storage: LogSetting = {
         logLocName: 'To be generated in client',
     },
     customSetting: {
-        server: "192.168.100.59:27017",
-        database: "test"
+        srv: true,
+        user: "testDB",
+        password: "h1nt1OyXw6QeUnzS",
+        server: "cluster0.29sklte.mongodb.net",
+        database: "test2",
     }
 }
 
@@ -128,7 +131,7 @@ sync.subscribe({
         console.log(`passing missing message: ${msgToBeSynchronized.appData.msgId} into target/secondary subject.`)
         // the missing data returned will be pushed (next(message)) into the target payload.
         let raw = msgToBeSynchronized.appData.msgPayload
-        let data : BaseMessage = JSON.parse(<string>raw)
+        let data: BaseMessage = JSON.parse(<string>raw)
         subscriber.next(data)
     }
 })
@@ -185,9 +188,9 @@ setTimeout(function () {
 function countdown() {
     let seconds = 0;
     const countUpInterval = setInterval(() => {
-      console.log(`Elapsed seconds: ${seconds}`);
-      seconds++;
+        console.log(`Elapsed seconds: ${seconds}`);
+        seconds++;
     }, 1000); // Update every second (1000 milliseconds)
-  }
+}
 
 countdown()

test/test4.ts

@@ -6,12 +6,10 @@ different types of data aside from messageLogs. */
 /* Note: MessageAudit will not work if storage is FIle. Search does not work at logging service
 does not cater for File system storage */
 import * as mongoose from 'mongoose'
-import { Observable, map, Subject, takeUntil, take, of, timer, from } from "rxjs";
+import { Subject } from "rxjs";
 import { ErrorTrigger, MessageSynchronisationServiceSetting } from "../type/datatype";
-import { StreamingService } from "./test-streamOBS";
 import { MessageAuditorService } from "../services/message-auditor.service";
 import { LoggingService } from '../dependencies/log/interface/export';
-import { BaseMessage } from '../dependencies/msgutil/interface/export';
 import { LogSetting, MessageLog } from '../dependencies/log/type/datatype';
 import * as fs from "fs"
 

test/test5.ts

@@ -0,0 +1,108 @@
+/* ---------------------    TEST 5 : Filtering     -------------------------- */
+/* This test is specifically design for testing the audit with additional fitlers. When the primary source want to perform
+audit on the designated target, they will impose one or many condition, in that only the data that meets the criteria
+will be taken into consideratoin for auditng. */
+import { Observable, Subject, take } from "rxjs"
+import { MessageAuditorService } from "../services/message-auditor.service"
+import { MessageAuditorServiceInterface, MessageSynchronisationServiceSetting } from "../type/datatype"
+import { LogSetting, MessageLog } from "../dependencies/log/type/datatype"
+import * as _ from 'lodash'
+import { StreamingService } from "./test-streamOBS"
+import { LoggingService } from "../dependencies/log/services/logging-service"
+import { ResponseMessage } from "../dependencies/log/dependencies/msgutil/interface/export"
+const auditService: MessageAuditorServiceInterface = new MessageAuditorService()
+const publisherloggingService: LoggingService = new LoggingService()
+const subscriberloggingService: LoggingService = new LoggingService()
+const stream = new StreamingService()
+
+let triggerSyncSubject: Subject<any> = new Subject()
+const publisher_take_four_messages: Observable<any> = stream.stream().pipe(take(4))
+const publisher: Subject<any> = new Subject()
+publisher_take_four_messages.subscribe({
+    next: (data) => {
+        publisher.next(data)
+    }
+})
+const subscriber_take_two_messagse: Observable<any> = stream.stream().pipe(take(2))
+const subscriber: Subject<any> = new Subject()
+subscriber_take_two_messagse.subscribe({
+    next: (data) => {
+        subscriber.next(<ResponseMessage>data)
+    }
+})
+
+let source: LogSetting = {
+    cacheMessageLimit: 0,
+    storage: "MongoDB",
+    setting: {
+        appName: 'Deafult from source',
+        appLocName: 'To be generated in source',
+        logLocName: 'To be generated in source',
+    },
+    customSetting: {
+        url: 'mongodb+srv://testDB:h1nt1OyXw6QeUnzS@cluster0.29sklte.mongodb.net/test1'
+    }
+}
+let target: LogSetting = {
+    cacheMessageLimit: 0,
+    storage: "MongoDB",
+    setting: {
+        appName: 'Default from target',
+        appLocName: 'To be generated in target',
+        logLocName: 'To be generated in target',
+    },
+    customSetting: {
+        url: 'mongodb+srv://testDB:h1nt1OyXw6QeUnzS@cluster0.29sklte.mongodb.net/test2'
+    }
+}
+
+// Combine source and target storage to form MessageSynchronisationServiceSetting. This is required in messageAudit initialization
+let settings: MessageSynchronisationServiceSetting = {
+    incomingSource: {
+        //all of the settings to be combined here
+        ...source,
+        tags: ['default'],
+    },
+    target: {
+        ...target,
+        tags: ['default'],
+    },
+    //  Set Filters here, since it's part of the settings
+    filters: {
+        'data.data.data.personCode': 'w002',
+        'header.messageProducerInformation.origin.userApplication.userAppId': 'Content Delivery Management Server'
+    }
+}
+
+/* -------  Calling the functions to be tested ----------- */
+intializeLogging(source, target)
+initializeAuditService(settings)
+setTimeout(() => {
+    performSync({ status: 1, message: "GO! GO! GO!" })
+}, 5000)
+
+
+// Basically start up all the functions and relevant subscription service in Audit Service.
+async function initializeAuditService(configuration: MessageSynchronisationServiceSetting) {
+    auditService.init(configuration) // Configure two points of audit and also adding filter
+    auditService.subscribe(triggerSyncSubject).subscribe((missingElements: MessageLog) => {
+        let message = JSON.parse(missingElements.appData.msgPayload as any)
+        subscriber.next(message)
+    })
+}
+
+// Emit an args into the synchronization trigger stream to perform a sync
+async function performSync(args: any) {
+    triggerSyncSubject.next(args)
+}
+
+// Set up logging point
+async function intializeLogging(source: LogSetting, target: LogSetting) {
+    publisherloggingService.init(source).then(() => {
+        publisherloggingService.subscribe(publisher)
+    })
+    subscriberloggingService.init(target).then(() => {
+        subscriberloggingService.subscribe(subscriber)
+    })
+}
+

testRequest.json

@@ -41,8 +41,7 @@
                     "msgLogDateTime": "2022-12-06T15:01:46.987Z",
                     "msgDateTime": "2022-12-06T08:50:33.809Z",
                     "msgTag": [
-                        "oval",
-                        "likable"
+                        "oval"
                     ],
                     "msgPayload": "Molestias facilis iusto similique iste voluptas facere. Alias est sequi. Quos consequatur temporibus blanditiis numquam vel. Eos repellat eaque. Voluptatibus optio optio magni eveniet. Quidem architecto esse aut sint neque error magnam perspiciatis."
                 }
@@ -91,7 +90,6 @@
                     "msgLogDateTime": "2022-12-06T15:01:46.987Z",
                     "msgDateTime": "2022-12-06T08:50:33.809Z",
                     "msgTag": [
-                        "oval",
                         "likable"
                     ],
                     "msgPayload": "Molestias facilis iusto similique iste voluptas facere. Alias est sequi. Quos consequatur temporibus blanditiis numquam vel. Eos repellat eaque. Voluptatibus optio optio magni eveniet. Quidem architecto esse aut sint neque error magnam perspiciatis."
@@ -141,7 +139,6 @@
                     "msgLogDateTime": "2022-12-06T15:01:46.987Z",
                     "msgDateTime": "2022-12-06T08:50:33.809Z",
                     "msgTag": [
-                        "oval",
                         "likable"
                     ],
                     "msgPayload": "Molestias facilis iusto similique iste voluptas facere. Alias est sequi. Quos consequatur temporibus blanditiis numquam vel. Eos repellat eaque. Voluptatibus optio optio magni eveniet. Quidem architecto esse aut sint neque error magnam perspiciatis."
@@ -191,8 +188,7 @@
                     "msgLogDateTime": "2022-12-06T15:01:46.987Z",
                     "msgDateTime": "2022-12-06T08:50:33.809Z",
                     "msgTag": [
-                        "oval",
-                        "likable"
+                        "oval"
                     ],
                     "msgPayload": "Molestias facilis iusto similique iste voluptas facere. Alias est sequi. Quos consequatur temporibus blanditiis numquam vel. Eos repellat eaque. Voluptatibus optio optio magni eveniet. Quidem architecto esse aut sint neque error magnam perspiciatis."
                 }
@@ -241,7 +237,6 @@
                     "msgLogDateTime": "2022-12-06T15:01:46.987Z",
                     "msgDateTime": "2022-12-06T08:50:33.809Z",
                     "msgTag": [
-                        "oval",
                         "likable"
                     ],
                     "msgPayload": "Molestias facilis iusto similique iste voluptas facere. Alias est sequi. Quos consequatur temporibus blanditiis numquam vel. Eos repellat eaque. Voluptatibus optio optio magni eveniet. Quidem architecto esse aut sint neque error magnam perspiciatis."

type/acknowledgement.interface.ts

@@ -1,6 +1,6 @@
 import { Observable } from "rxjs";
-import { BaseMessage, ResponseMessage } from "../dependencies/msgutil/interface/export";
 import { LogSetting } from "../dependencies/log/type/datatype";
+import { BaseMessage, ResponseMessage } from "../dependencies/log/dependencies/msgutil/interface/export";
 
 /**
  * @deprecated The acknowledgement will be covered by MessageAuditorService.

type/datatype.ts

@@ -1,6 +1,6 @@
 import { Observable } from "rxjs";
-import { BaseMessage } from "../dependencies/msgutil/interface/export";
 import { LogSetting } from "../dependencies/log/type/datatype";
+import { BaseMessage } from "../dependencies/log/dependencies/msgutil/interface/export";
 
 type IncomingMessageSetting = LogSetting & {
     incomingObservable: Observable<BaseMessage>
@@ -16,7 +16,8 @@ export interface IncomingMessageServiceInterface {
 
 export interface MessageSynchronisationServiceSetting {
     incomingSource: LogSettingwTags,
-    target: LogSettingwTags
+    target: LogSettingwTags,
+    filters?: any
 }
 
 // Renew Structure To fix undefined issue at test3a.ts init()