test case fix and audit

.env

@@ -1,5 +1,4 @@
-CheckAudit = Yes
-CheckAudit = Yes
+CheckAudit = "Yes"
 domain = "swopt.com"
 primary_database = "primaryDB"
 primary_user = "primaryUser"

package.json

@@ -6,9 +6,9 @@
   "scripts": {
     "build": "tsc -p tsconfig.json",
     "temp": "node test/temptest.js",
-    "test1": "node dist/est/test1.js",
-    "test2": "node dist/est/test2.js",
-    "test3": "node dist/est/test3.js",
+    "test1": "node dist/test/test1.js",
+    "test2": "node dist/test/test2.js",
+    "test3": "node dist/test/test3.js",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "repository": {

src/dependencies/log

@@ -1 +1 @@
-Subproject commit 3005fd32f1e7f0094af9bd5c2423035448a2338e
+Subproject commit 7ccde530e1fc802fb390ceeae09e53e546d9168e

src/services/message-auditor.service.ts

@@ -4,7 +4,6 @@ import { ErrorTrigger, MessageAuditorServiceInterface, MessageSynchronisationSer
 import { LoggingService } from "../dependencies/log/src/export";
 import { MessageLog } from "../dependencies/log/src/type/interface";
 import { BaseMessage } from "../dependencies/log/src/dependencies/msgutil/interface/export";
-let processedMsgIds = new Set();
 require('dotenv').config();
 
 export class MessageAuditorService implements MessageAuditorServiceInterface {
@@ -52,7 +51,7 @@ export class MessageAuditorService implements MessageAuditorServiceInterface {
 
     /* ________________ Private Functions _________________ */
     // Filtering functions to filters out messages
-    private filterData(filters: any, message: MessageLog): boolean {
+    private filterData(message: MessageLog, filters?: any): boolean {
         let response: boolean = true //Just using this like a statemanagement
         let payload: BaseMessage = JSON.parse(message.appData.msgPayload as string) // Extract the payload from the messageLog first
         this.checkIfIsInPayloadDataFormat(payload) // Convert stringified nested payload if there's any
@@ -91,25 +90,22 @@ export class MessageAuditorService implements MessageAuditorServiceInterface {
         this.acquireData().then((data: { arr1: MessageLog[], arr2: MessageLog[] }) => {
             // In the case where there are differences in the array length, then extensive comparison
             // will be carried out to filters out the differences. Differences are the missing data.
-
-            if(process.env.CheckAudit)
-            {
-                console.log("Record set 1: ", _.keys(_.countBy(data.arr1,function(data:MessageLog){return data.appData['msgId']})).length);
-                console.log("Record set 2: ", _.keys(_.countBy(data.arr2,function(data:MessageLog){return data.appData['msgId']})).length);
+            console.log(`Checking Audit ${process.env.CheckAudit}`)
+            if (process.env.CheckAudit == `Yes`) {
+                console.log("Record set 1: ", _.keys(_.countBy(data.arr1, function (data: MessageLog) { return data.appData['msgId'] })).length);
+                console.log("Record set 2: ", _.keys(_.countBy(data.arr2, function (data: MessageLog) { return data.appData['msgId'] })).length);
             }
 
 
-            if(process.env.CheckAudit)
-            {
-                console.log("[CheckAudit] Record set 1: ", _.keys(_.countBy(data.arr1,function(data:MessageLog){return data.appData['msgId']})).length);
-                console.log("[CheckAudit] Record set 2: ", _.keys(_.countBy(data.arr2,function(data:MessageLog){return data.appData['msgId']})).length);
+            if (process.env.CheckAudit == `Yes`) {
+                console.log("[CheckAudit] Record set 1: ", _.keys(_.countBy(data.arr1, function (data: MessageLog) { return data.appData['msgId'] })).length);
+                console.log("[CheckAudit] Record set 2: ", _.keys(_.countBy(data.arr2, function (data: MessageLog) { return data.appData['msgId'] })).length);
             }
 
             this.checkArrayDifferences(data).then((data: MessageLog[]) => {
 
-                if(process.env.CheckAudit)
-                {
-                    console.log("Difference: ",data.length);
+                if (process.env.CheckAudit) {
+                    console.log("Difference: ", data.length);
                 }
 
                 data.forEach(msgElement => {
@@ -140,14 +136,15 @@ export class MessageAuditorService implements MessageAuditorServiceInterface {
                     // Filter also carries out the query aspect of the operation, allowing it to acquire all the relevant data.
                     this.sourceSrc.filter({ msgTag: this.settings.incomingSource.tags[0] }).then((data: MessageLog[]) => {
                         data.forEach((message: MessageLog) => {
-                            if (this.filterData(this.settings.filters, message)) set1.push(message)
+                            if (this.filterData(message, this.settings.filters)) set1.push(message)
                         })
                     }).catch((err) => {
                         console.error(err.message)
                     }).then(() => {
+                        console.log(set1.length)
                         this.targetSrc.filter({ msgTag: this.settings.target.tags[0] }).then((data: MessageLog[]) => {
                             data.forEach(message => {
-                                if (this.filterData(this.settings.filters, message)) set2.push(message)
+                                if (this.filterData(message, this.settings.filters)) set2.push(message)
                             })
                             allSets.arr1 = set1
                             allSets.arr2 = set2

src/test/test1.ts

@@ -62,11 +62,13 @@ let publisher_storage: LogSetting = {
         logLocName: 'To be generated in client',
     },
     mongoSetting: {
-        srv: true,
-        user: "testDB",
-        password: "h1nt1OyXw6QeUnzS",
-        server: "cluster0.29sklte.mongodb.net",
-        database: "test2",
+        server: process.env.domain,
+        user: process.env.primary_user,
+        password: process.env.password,
+        database: process.env.primary_database,
+        tls: true,
+        caFile: process.env.CAfilePath,
+        clientPemFile: process.env.clientPEMpath
     }
 }
 
@@ -83,11 +85,13 @@ let subscriber_storage: LogSetting = {
         logLocName: 'To be generated in client',
     },
     mongoSetting: {
-        srv: true,
-        user: "testDB",
-        password: "h1nt1OyXw6QeUnzS",
-        server: "cluster0.29sklte.mongodb.net",
-        database: "test",
+        server: process.env.domain,
+        user: process.env.secondary_user,
+        password: process.env.password,
+        database: process.env.secondary_database,
+        tls: true,
+        caFile: process.env.CAfilePath,
+        clientPemFile: process.env.clientPEMpath
     }
 }
 
@@ -96,11 +100,11 @@ let settings: MessageSynchronisationServiceSetting = {
     incomingSource: {
         //all of the settings to be combined here
         ...publisher_storage,
-        tags: ['Incoming']
+        tags: [`default`]
     }, //LogSetting & {tags:string[] },   
     target: {
         ...subscriber_storage,
-        tags: ['Incoming']
+        tags: [`default`]
     }  //LogSetting & {tags:string[] }  
 }
 
@@ -148,7 +152,7 @@ setTimeout(() => {
         message: "NO. I dont want to work"
     }
     errorSubject.next(sampleError)
-}, 10000)
+}, 6000)
 
 /* THis is testing for generating error message to be fed into the error subject
 to act as additional trigger to exectute the synchronization when there's no internet

src/test/test2.ts

@@ -131,8 +131,8 @@ setTimeout(() => {
 countdown()
 
 // Convert all the existing cdms into message log
-// convertDataInMongo(primary_storage.mongoSetting.url)
-// convertDataInMongo(secondary_storage.mongoSetting.url)
+convertDataInMongo(primary_storage.mongoSetting.url)
+convertDataInMongo(secondary_storage.mongoSetting.url)
 
 // These declaration are for the secondary to log the converted missing data back in it's own collection at their corresponding servers
 const dbConnection = mongoose.createConnection("mongodb+srv://testDB:h1nt1OyXw6QeUnzS@cluster0.29sklte.mongodb.net/secondary")

src/test/test3.ts

@@ -35,24 +35,36 @@ let source: LogSetting = {
     cacheMessageLimit: 0,
     storage: "MongoDB",
     setting: {
-        appName: 'Deafult from source',
-        appLocName: 'To be generated in source',
-        logLocName: 'To be generated in source',
+        appName: 'Default from client',
+        appLocName: 'To be generated in client',
+        logLocName: 'To be generated in client',
     },
     mongoSetting: {
-        url: 'mongodb+srv://testDB:h1nt1OyXw6QeUnzS@cluster0.29sklte.mongodb.net/test1'
+        server: process.env.domain,
+        user: process.env.primary_user,
+        password: process.env.password,
+        database: process.env.primary_database,
+        tls: true,
+        caFile: process.env.CAfilePath,
+        clientPemFile: process.env.clientPEMpath
     }
 }
 let target: LogSetting = {
     cacheMessageLimit: 0,
     storage: "MongoDB",
     setting: {
-        appName: 'Default from target',
-        appLocName: 'To be generated in target',
-        logLocName: 'To be generated in target',
+        appName: 'Default from client',
+        appLocName: 'To be generated in client',
+        logLocName: 'To be generated in client',
     },
     mongoSetting: {
-        url: 'mongodb+srv://testDB:h1nt1OyXw6QeUnzS@cluster0.29sklte.mongodb.net/test2'
+        server: process.env.domain,
+        user: process.env.secondary_user,
+        password: process.env.password,
+        database: process.env.secondary_database,
+        tls: true,
+        caFile: process.env.CAfilePath,
+        clientPemFile: process.env.clientPEMpath
     }
 }
 
@@ -65,13 +77,13 @@ let settings: MessageSynchronisationServiceSetting = {
     },
     target: {
         ...target,
-        tags: ['default'],
+        tags: [`default`],
     },
     //  Set Filters here, since it's part of the settings
-    filters: {
-        'data.data.data.personCode': 'w002',
-        'header.messageProducerInformation.origin.userApplication.userAppId': 'Content Delivery Management Server'
-    }
+    // filters: {
+    //     'data.data.data.personCode': 'w002',
+    //     'header.messageProducerInformation.origin.userApplication.userAppId': 'Content Delivery Management Server'
+    // }
 }
 
 /* -------  Calling the functions to be tested ----------- */