query.service.ts 7.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184
  1. import * as fs from 'fs'
  2. import { _, isObject, get } from 'lodash'
  3. import { Observable, Subject, interval, map, of } from 'rxjs'
  4. import { DataPrepService } from './dataprep.service'
  5. export class queryService {
  6. private dataPrepService : DataPrepService
  7. constructor(){
  8. this.dataPrepService = new DataPrepService()
  9. }
  10. public callFromOtherClass(){
  11. const t0 = performance.now()
  12. let i
  13. for (i = 0; i <= 6000000000; i++) {
  14. }
  15. const t1 = performance.now()
  16. const timeTakenInSeconds = (t1 - t0) / 1000;
  17. console.log(`Time taken: ${timeTakenInSeconds} seconds to run this function`);
  18. }
  19. public query(storageAddress: Storage, ...conditions: Conditions[]): Observable<any> {
  20. let dataFromStorage: Subject<any> = new Subject()
  21. let filteredResult: Subject<any> = new Subject()
  22. this.dataPrepService.loadObsData(storageAddress, dataFromStorage)
  23. this.filterFromObs(dataFromStorage, filteredResult, ...conditions)
  24. return filteredResult.pipe()
  25. }
  26. // Search and Filter: Pure Observables
  27. private filterFromObs(dataFromStorage: Subject<any>, filteredResult: Subject<any>, ...conditions: Conditions[]) {
  28. dataFromStorage.subscribe({
  29. next: element => {
  30. if (this.filterByKeyValue(element, ...conditions)) {
  31. filteredResult.next(element)
  32. } else {
  33. console.log(`${element.appData.msgId} does not match search criteria`)
  34. }
  35. }
  36. })
  37. }
  38. // Logic 1: Success. But argument must specifies header.messageID.... to search
  39. private hasMatchingProps(data, condition): boolean {
  40. // Merge all condtions into searchObj
  41. let result = _.every(condition, (val, key) => {
  42. const propKeys = key.split('.');
  43. let nestedObj = data;
  44. _.forEach(propKeys, propKey => {
  45. nestedObj = nestedObj[propKey];
  46. });
  47. if (_.isObject(val)) {
  48. return this.hasMatchingProps(nestedObj, val);
  49. }
  50. return nestedObj === val;
  51. });
  52. return result
  53. }
  54. // Logic 2: Success: More superior version than Logic 1 since it can perform flat searches like {messageID : 1234}
  55. // without specifying its parent property's name. eg: {header.messageID: 1234}
  56. private filterByKeyValue(data, ...conditions): boolean {
  57. try {
  58. // Merge all conditions into searchObj
  59. let searchObj = Object.assign({}, ...conditions)
  60. let recordFound = true
  61. // Check for data type. Can actually remove this code if dont want. Not that important anyways
  62. if (typeof data !== 'object' || typeof searchObj !== 'object') {
  63. return false;
  64. }
  65. // Check data to see if the given data is within the date range of the specified column
  66. if (recordFound == true) {
  67. if (searchObj.hasOwnProperty("$dateRange")) {
  68. recordFound = this.filterByDateRange(data, searchObj.$dateRange)
  69. delete searchObj.$dateRange
  70. }
  71. }
  72. // Check if the regular expression value matches any of the data string
  73. if (recordFound == true) {
  74. if (searchObj.hasOwnProperty("$regex")) {
  75. recordFound = this.filterViaRegex(data, searchObj.$regex)
  76. delete searchObj.$regex
  77. }
  78. }
  79. // Check if the key has parent key notation and then perform matching sequences. Eg : "header.appdata. etc etc"
  80. if (recordFound == true) {
  81. // check if key is header.is like 'propertyName1.propertyName2'
  82. let searchkey = Object.keys(searchObj)
  83. searchkey.every((key) => {
  84. if (key.includes('.')) {
  85. let condition = {
  86. key: searchObj[key]
  87. }
  88. this.hasMatchingProps(data, condition)
  89. delete searchObj[key]
  90. }
  91. })
  92. }
  93. // Check the rest of the key value pairs to see if the conditions are fulfilled(entries must matched)
  94. if (recordFound == true) {
  95. recordFound = this.matchValues(data, searchObj)
  96. }
  97. return recordFound
  98. }
  99. catch (e) {
  100. console.error(e.message)
  101. }
  102. }
  103. // Match the key values pair between conditions and the given data
  104. private matchValues(data, searchObj): boolean {
  105. let matchKeys = Object.keys(searchObj);
  106. let isMatchingObject = (object) => {
  107. return matchKeys.every((key) => {
  108. let lodashPath = key.replace(/\[(\w+)\]/g, '.$1').replace(/^\./, '');
  109. let objectValue = _.get(object, lodashPath);
  110. let searchValue = searchObj[key];
  111. if (Array.isArray(searchValue)) {
  112. // Check if any of the search values are included in the object value
  113. return searchValue.some((value) => {
  114. return Array.isArray(objectValue) ? objectValue.includes(value) : objectValue === value;
  115. });
  116. } else if (typeof searchValue === 'object' && typeof objectValue === 'object') {
  117. return isMatchingObject(objectValue);
  118. } else {
  119. return objectValue === searchValue;
  120. }
  121. });
  122. };
  123. let isObjectMatching = (object) => {
  124. if (typeof object !== 'object') {
  125. return false;
  126. }
  127. return isMatchingObject(object) || Object.values(object).some(isObjectMatching);
  128. };
  129. return isObjectMatching(data);
  130. }
  131. // Matching the regex args to see if it matches the data that is now converted to string. As long as partial match, it will return true
  132. private filterViaRegex(element: any, inquiry: any): boolean {
  133. // create a new regular expression to use regex.test
  134. const regex = new RegExp(inquiry);
  135. const hasMatchingSubstring = regex.test(JSON.stringify(element));
  136. return hasMatchingSubstring;
  137. }
  138. // Check if the data's date is within the date range provided and also the column in which the data is to be compared with
  139. private filterByDateRange(data: any, dateRange: DateRange): boolean {
  140. // Lodash implemetation to get the specific property of data
  141. let msgDate: string = get(data, dateRange.column)
  142. let date = new Date(msgDate)
  143. const start = new Date(dateRange.startDate);
  144. const end = new Date(dateRange.endDate);
  145. return date >= start && date <= end;
  146. }
  147. }
  148. // Entries that client will use. Subject to be improved later on
  149. export interface Conditions {
  150. $regex?: string,
  151. $dateRange?: DateRange,
  152. [key: string]: string | Date | DateRange | string[]
  153. }
  154. export interface DateRange {
  155. startDate: string | Date,
  156. endDate: string | Date,
  157. column: string
  158. }
  159. export interface Storage {
  160. type: string,
  161. url: string
  162. }