pre-filtering works now

AgentQueryPlan.json

@@ -1,78 +1,25 @@
 {
   "description": "MongoDB Query Planner for FFB Production",
-  "instructions": "You are an intelligent MongoDB query planner for FFBProduction.\n\nYour job is to:\n1. Understand the user's question and extract intent (AGGREGATE or SEARCH).\n2. Generate a minimal preFilter ($match) for efficiency.\n3. Decide if a vector search is needed; if yes, set vectorQuery and vectorOptions.\n4. Build postPipeline for aggregation or projection; include only fields necessary for computation.\n5. Parse natural language dates into ISO format (YYYY-MM-DD).\n6. Map user terms like \"Total output\" to actual fields (weight, quantity).\n7. Use only allowed fields: [\n        \"site\",\n        \"phase\",\n        \"block\",\n        \"productionDate\",\n        \"weight\",\n        \"quantity\"\n    ].\n8. Use only allowed operators: [\n        \"$eq\",\n        \"$in\",\n        \"$gte\",\n        \"$lte\"\n    ].\n9. Output **valid JSON only**, no extra text.",
+  "instructions": "You are an intelligent MongoDB query planner for FFBProduction.\n\nYour job is to:\n1. Understand the user's question and extract intent (AGGREGATE or SEARCH).\n2. Generate a minimal preFilter ($match) for efficiency.\n3. Output **only the preFilter object** for MongoDB. Do not include postPipeline or fields.\n4. Parse natural language dates into ISO format (YYYY-MM-DD).\n5. Use only allowed fields: [\"site\",\"phase\",\"block\",\"productionDate\",\"weight\",\"quantity\"].\n6. Use only allowed operators: [\"$eq\",\"$in\",\"$gte\",\"$lte\"].\n7. Output valid JSON only, no extra text.",
   "examples": [
     {
       "question": "Total output in Site A for Nov-Dec",
       "plan": {
-        "intent": "AGGREGATE",
         "preFilter": {
           "site": "Site A",
           "productionDate": {
             "$gte": "2025-11-01",
             "$lte": "2025-12-31"
           }
-        },
-        "vectorQuery": null,
-        "vectorOptions": {
-          "limit": 5,
-          "numCandidates": 50
-        },
-        "postPipeline": [
-          {
-            "$group": {
-              "_id": "$site",
-              "totalWeight": {
-                "$sum": "$weight"
-              }
-            }
-          },
-          {
-            "$project": {
-              "site": "$_id",
-              "totalWeight": 1,
-              "_id": 0
-            }
-          }
-        ],
-        "fields": [
-          "site",
-          "weight",
-          "productionDate"
-        ]
+        }
       }
     },
     {
-      "question": "Top 5 most similar records to 'highest producing block in Site B'",
+      "question": "FFB data for Site B",
       "plan": {
-        "intent": "SEARCH",
         "preFilter": {
           "site": "Site B"
-        },
-        "vectorQuery": "highest producing block in Site B",
-        "vectorOptions": {
-          "limit": 5,
-          "numCandidates": 50
-        },
-        "postPipeline": [
-          {
-            "$project": {
-              "site": 1,
-              "phase": 1,
-              "block": 1,
-              "weight": 1,
-              "quantity": 1,
-              "_id": 0
-            }
-          }
-        ],
-        "fields": [
-          "site",
-          "phase",
-          "block",
-          "weight",
-          "quantity"
-        ]
+        }
       }
     }
   ]

src/FFB/ffb-production.controller.ts

@@ -18,12 +18,13 @@ export class FFBProductionController {
   async query(@Body('message') message: string) {
     // 1. Planner generates AgentQueryPlan
     const plan = await this.planner.plan(message);
-
+    console.log(plan)
     // 2. Executor runs query against MongoDB / vector store
     const rawResults = await this.executor.execute(plan);
-
+    console.log(rawResults)
     // 3. Compiler formats results into natural language
     const answer = await this.compiler.compile(plan, rawResults);
+    console.log(answer)
 
     return { answer }; // Return final human-readable response
   }

src/FFB/ffb-query-executor.service.ts

@@ -22,20 +22,38 @@ export class FFBQueryExecutorService {
         return this.repo;
     }
 
+    /** Convert any string dates in preFilter to Date objects */
+    private convertDates(preFilter?: Record<string, any>) {
+        if (!preFilter) return;
+        const pd = preFilter.productionDate;
+        if (pd) {
+            if (pd.$gte) pd.$gte = new Date(pd.$gte);
+            if (pd.$lte) pd.$lte = new Date(pd.$lte);
+        }
+    }
+
     async execute(plan: AgentQueryPlan) {
         const repo = await this.getRepo();
 
+        // Convert string dates to Date objects
+        this.convertDates(plan.preFilter);
+
         // Build $project if fields are specified
-        let postPipeline = plan.postPipeline ?? [];
+        let postPipeline: Array<Record<string, any>> = plan.postPipeline ?? [];
         if (plan.fields && plan.fields.length > 0) {
-            postPipeline = [{ $project: plan.fields.reduce((acc, f) => ({ ...acc, [f]: 1 }), { _id: 0 }) }, ...postPipeline];
+            postPipeline = [
+                { $project: plan.fields.reduce((acc, f) => ({ ...acc, [f]: 1 }), { _id: 0 }) },
+                ...postPipeline,
+            ];
         }
 
-        // CASE 1: Vector search
+        // Construct the aggregation pipeline
+        let pipeline: Array<Record<string, any>> = [];
+
         if (plan.vectorQuery) {
             const vector = await this.vectorService['embedText'](plan.vectorQuery);
 
-            return repo.aggregate([
+            pipeline = [
                 {
                     $vectorSearch: {
                         index: 'vector_index',
@@ -47,13 +65,20 @@ export class FFBQueryExecutorService {
                     },
                 },
                 ...postPipeline,
-            ]);
+            ];
+        } else {
+            // Pure aggregation
+            pipeline = [
+                ...(plan.preFilter ? [{ $match: plan.preFilter }] : []),
+                ...postPipeline,
+            ];
         }
 
-        // CASE 2: Pure aggregation (no vectors)
-        return repo.aggregate([
-            ...(plan.preFilter ? [{ $match: plan.preFilter }] : []),
-            ...postPipeline,
-        ]);
+        console.log('--- Aggregation pipeline ---\n', JSON.stringify(pipeline, null, 2));
+
+        const results = await repo.aggregate(pipeline);
+        console.log('--- Raw results ---\n', results);
+
+        return results;
     }
 }

src/mongo/mongo-ffb-production.repository.ts

@@ -74,7 +74,7 @@ export class FFBProductionRepository {
       .toArray();
   }
 
-  async aggregate(pipeline: any[]): Promise<any[]> {
+  async aggregate(pipeline: Array<Record<string, any>>): Promise<any[]> {
     // Optional: log the pipeline for debugging
     // console.log('Executing aggregation pipeline:', JSON.stringify(pipeline, null, 2));