feat: implement FastAPI backend with OpenAI-powered receipt extraction and template filling services

ai-data-entry-ui/src/app/components/claim-form/claim-form.component.html

@@ -24,12 +24,13 @@
            (dragover)="onDragOver($event)" 
            (dragleave)="onDragLeave($event)" 
            (drop)="onDrop($event)">
-        <img *ngIf="previewUrl" [src]="previewUrl" class="receipt-preview" alt="Receipt Preview">
+        <img *ngIf="previewUrl && !isPdf" [src]="previewUrl" class="receipt-preview" alt="Receipt Preview">
+        <iframe *ngIf="previewUrl && isPdf" [src]="safePdfUrl" width="100%" height="600px" style="border: none;"></iframe>
         <div *ngIf="!previewUrl" class="placeholder">
           <p>Drag and drop a medical receipt here, or use the button below.</p>
-          <input type="file" #fileInput (change)="onFileSelected($event)" accept="image/*" hidden>
+          <input type="file" #fileInput (change)="onFileSelected($event)" accept="image/*,.pdf" hidden>
           <button class="upload-trigger" (click)="fileInput.click()">
-            Select Receipt Image
+            Select Receipt Image or PDF
           </button>
         </div>
         

ai-data-entry-ui/src/app/components/claim-form/claim-form.component.ts

@@ -1,4 +1,5 @@
 import { Component, OnInit, OnDestroy } from '@angular/core';
+import { DomSanitizer, SafeResourceUrl } from '@angular/platform-browser';
 import { FormBuilder, FormGroup, ReactiveFormsModule, Validators } from '@angular/forms';
 import { CommonModule } from '@angular/common';
 import { Router, RouterModule } from '@angular/router';
@@ -19,6 +20,8 @@ export class ClaimFormComponent implements OnInit, OnDestroy {
   claimForm: FormGroup;
   currentUser: User | null = null;
   previewUrl: string | null = null;
+  safePdfUrl: SafeResourceUrl | null = null;
+  isPdf = false;
   selectedFile: File | null = null;
   extractionResponse: ExtractionResponse | null = null;
   v2Response: V2TemplateResponse | null = null;
@@ -51,7 +54,8 @@ export class ClaimFormComponent implements OnInit, OnDestroy {
     private fb: FormBuilder,
     private extractionService: ExtractionService,
     private sessionService: SessionService,
-    private router: Router
+    private router: Router,
+    private sanitizer: DomSanitizer
   ) {
     this.claimForm = this.fb.group({
       provider_name: ['', Validators.required],
@@ -131,8 +135,17 @@ export class ClaimFormComponent implements OnInit, OnDestroy {
 
   private handleFileSelection(file: File): void {
     if (!this.currentUser) return;
+    this.isPdf = file.type === 'application/pdf';
     this.selectedFile = file;
-    this.previewUrl = URL.createObjectURL(file);
+    const url = URL.createObjectURL(file);
+    this.previewUrl = url;
+
+    if (this.isPdf) {
+      this.safePdfUrl = this.sanitizer.bypassSecurityTrustResourceUrl(url);
+    } else {
+      this.safePdfUrl = null;
+    }
+
     this.extractionResponse = null; 
     this.errorMessage = null;
     this.autoFilledFields.clear();

backend/main.py

@@ -77,15 +77,16 @@ async def extract_receipt(
     user_name: str = Form("Demo User"),
     department: str = Form("Operations")
 ):
-    if not file.content_type.startswith("image/"):
+    allowed_types = ["image/jpeg", "image/png", "application/pdf"]
+    if file.content_type not in allowed_types:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail="File provided is not an image."
+            detail="Unsupported file type. Please upload a JPEG, PNG, or PDF."
         )
     
     try:
         content = await file.read()
-        extraction_result = await extract_receipt_data(content, user_name, department)
+        extraction_result = await extract_receipt_data(content, file.content_type, user_name, department)
         
         if extraction_result is None:
              raise HTTPException(
@@ -107,10 +108,11 @@ async def fill_template(
     user_name: str = Form("Demo User"),
     department: str = Form("Operations")
 ):
-    if not file.content_type.startswith("image/"):
+    allowed_types = ["image/jpeg", "image/png", "application/pdf"]
+    if file.content_type not in allowed_types:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail="File provided is not an image."
+            detail="Unsupported file type. Please upload a JPEG, PNG, or PDF."
         )
     
     try:
@@ -119,6 +121,7 @@ async def fill_template(
         
         result = await fill_form_with_template_v2(
             content, 
+            file.content_type,
             template_dict, 
             user_name, 
             department

backend/services/openai_service.py

@@ -6,6 +6,7 @@ import json
 from openai import AsyncOpenAI
 from dotenv import load_dotenv
 from PIL import Image
+import fitz  # PyMuPDF
 from backend.schemas import ExtractionResponse, V2TemplateResponse
 
 load_dotenv()
@@ -35,9 +36,37 @@ def compress_image(image_content: bytes) -> bytes:
     img.save(output_buffer, format="JPEG", quality=IMAGE_QUALITY, optimize=True)
     return output_buffer.getvalue()
 
-async def extract_receipt_data(image_content: bytes, user_name: str, department: str) -> ExtractionResponse:
-    # 1. Compress Image
-    compressed_content = compress_image(image_content)
+def convert_to_image_bytes(content: bytes, content_type: str) -> bytes:
+    """If PDF, convert first page to image using PyMuPDF. Otherwise return original."""
+    if content_type == "application/pdf":
+        try:
+            # Open PDF from bytes
+            doc = fitz.open(stream=content, filetype="pdf")
+            if len(doc) == 0:
+                return content
+            
+            # Get the first page
+            page = doc[0]
+            
+            # Render page to a pixmap (300 DPI: scale=300/72 = 4.166...)
+            # For high quality OCR, a scale of 2.0 or 3.0 is usually sufficient
+            matrix = fitz.Matrix(2.0, 2.0) 
+            pix = page.get_pixmap(matrix=matrix)
+            
+            # Convert pixmap to JPEG bytes
+            img_data = pix.tobytes("jpeg")
+            doc.close()
+            return img_data
+        except Exception as e:
+            logger.error(f"PDF conversion failed: {str(e)}")
+            return content
+    return content
+
+async def extract_receipt_data(image_content: bytes, content_type: str, user_name: str, department: str) -> ExtractionResponse:
+    # 1. Convert if PDF
+    raw_image = convert_to_image_bytes(image_content, content_type)
+    # 2. Compress Image
+    compressed_content = compress_image(raw_image)
     base64_image = base64.b64encode(compressed_content).decode("utf-8")
     
     # 2. Refined Prompt
@@ -89,9 +118,11 @@ async def extract_receipt_data(image_content: bytes, user_name: str, department:
             
     return result
 
-async def fill_form_with_template_v2(image_content: bytes, template_fields: dict, user_name: str, department: str) -> V2TemplateResponse:
-    # 1. Compress Image
-    compressed_content = compress_image(image_content)
+async def fill_form_with_template_v2(image_content: bytes, content_type: str, template_fields: dict, user_name: str, department: str) -> V2TemplateResponse:
+    # 1. Convert if PDF
+    raw_image = convert_to_image_bytes(image_content, content_type)
+    # 2. Compress Image
+    compressed_content = compress_image(raw_image)
     base64_image = base64.b64encode(compressed_content).decode("utf-8")
     
     # 2. V2 Prompt

requirements.txt

@@ -6,3 +6,4 @@ python-multipart
 pydantic>=2.0
 Pillow
 sqlalchemy
+pymupdf

sample_medical_receipts/fake_medical_receipt.pdf

@@ -0,0 +1,38 @@
+KLINIK SERI PERTIWI
+
+(Owned by Pertiwi Medical Group Sdn Bhd)
+Company Reg No: 202301045678 (1523456-X)
+No. 45, Jalan Telawi 3, Bangsar Baru, 59100 Kuala Lumpur
+Tel: 03-2094 8888 | Email: admin@klinikpertiwi.com.my
+
+[Body - Left Aligned]
+OFFICIAL RECEIPT / TAX INVOICE
+Receipt No: OR-88923-2026
+Date: 31 March 2026
+Time: 10:45 AM
+Doctor: Dr. Siti Aminah (MMC: 44123)
+Patient Name: Demo User
+Patient ID: P-9901
+NRIC/Passport: 900101-14-5555
+
+Description                           Qty Unit Price (RM) Total (RM)
+
+Professional Consultation (Level 2)   1 50.00             50.00
+
+Paracetamol 500mg (Strip of 10)       2 8.50              17.00
+Amoxicillin 250mg (Course)  1 25.00                              25.00
+
+Medical Certificate (MC) Processing 1 5.00                       5.00
+
+Subtotal: RM 97.00
+SST (0%): RM 0.00
+Rounding: RM 0.00
+
+TOTAL PAID: RM 97.00
+
+Payment Mode: Credit Card (Visa ****1234)
+
+[Footer - Center Aligned]
+This is a computer-generated receipt. No signature is required.
+GET WELL SOON
+