Fixed lodash.

services/message-auditor.service.ts

@@ -1,10 +1,10 @@
 import { map, Observable, of, Subject } from "rxjs";
 import { ErrorTrigger, MessageAuditorServiceInterface, MessageSynchronisationServiceSetting } from "../type/datatype";
 import { MessageLog } from "../dependencies/log/type/datatype";
-import { _ } from 'lodash'
+import * as _ from 'lodash'
 import { LoggingService } from "../dependencies/log/interface/export";
 import { BaseMessage } from "../dependencies/log/dependencies/msgutil/interface/export";
-
+let processedMsgIds = new Set();
 export class MessageAuditorService implements MessageAuditorServiceInterface {
     private settings: MessageSynchronisationServiceSetting
     private sourceSrc: LoggingService = new LoggingService()
@@ -89,7 +89,28 @@ export class MessageAuditorService implements MessageAuditorServiceInterface {
         this.acquireData().then((data: { arr1: MessageLog[], arr2: MessageLog[] }) => {
             // In the case where there are differences in the array length, then extensive comparison
             // will be carried out to filters out the differences. Differences are the missing data.
+
+            if(process.env.CheckAudit)
+            {
+                console.log("[CheckAudit] Record set 1: ", _.keys(_.countBy(data.arr1,function(data:MessageLog){return data.appData['msgId']})).length);
+                console.log("[CheckAudit] Record set 2: ", _.keys(_.countBy(data.arr2,function(data:MessageLog){return data.appData['msgId']})).length);
+            }
+
             this.checkArrayDifferences(data).then((data: MessageLog[]) => {
+
+                if(process.env.CheckAudit)
+                {
+                    console.log("[CheckAudit] Difference: ", _.keys(_.countBy(data,function(msg:MessageLog){return msg.appData['msgId']})).length);
+                    console.log("[CheckAudit] Missing msgId: ", _.keys(_.countBy(data, function (msg: MessageLog) { return msg.appData['msgId'] })));
+                }
+
+                // filter out the duplicate record and only emit the first record
+                let missingMsgIdArray = _.keys(_.countBy(data, function (msg: MessageLog) { return msg.appData['msgId'] }));
+                let uniqueRecords = missingMsgIdArray.map(msgId => {
+                    let dataForMsgId = data.find(msg => msg.appData['msgId'] === msgId);
+                    return dataForMsgId;
+                });
+                data = uniqueRecords;
                 data.forEach(msgElement => {
                     let refined = JSON.parse(JSON.stringify(msgElement))
                     // Once the missing data has been weeded out, it is then passed into the Subject 
@@ -189,7 +210,9 @@ export class MessageAuditorService implements MessageAuditorServiceInterface {
     // Notification message may have multiple nested data properties that maybe in string format
     private checkIfIsInPayloadDataFormat(payload: BaseMessage | any) {
         let parsedData: any
-        if (payload.data.data.data && typeof payload.data.data.data === 'string') {
+        if (payload.data
+            && payload.data.data
+            && payload.data.data.data && typeof payload.data.data.data === 'string') {
             parsedData = JSON.parse(payload.data.data.data)
             // console.log(parsedData)
             payload.data.data.data = parsedData

test/test5.ts

@@ -6,7 +6,7 @@ import { Observable, Subject, take } from "rxjs"
 import { MessageAuditorService } from "../services/message-auditor.service"
 import { MessageAuditorServiceInterface, MessageSynchronisationServiceSetting } from "../type/datatype"
 import { LogSetting, MessageLog } from "../dependencies/log/type/datatype"
-import { _ } from 'lodash'
+import * as _ from 'lodash'
 import { StreamingService } from "./test-streamOBS"
 import { LoggingService } from "../dependencies/log/services/logging-service"
 import { ResponseMessage } from "../dependencies/log/dependencies/msgutil/interface/export"