import os import base64 import io import logging from openai import AsyncOpenAI from dotenv import load_dotenv from PIL import Image from backend.schemas import ExtractionResponse load_dotenv() # Setup logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) client = AsyncOpenAI(api_key=os.getenv("OPENAI_API_KEY")) MAX_IMAGE_SIZE = (2000, 2000) IMAGE_QUALITY = 85 def compress_image(image_content: bytes) -> bytes: """Resizes and compresses the image to reduce upload size.""" img = Image.open(io.BytesIO(image_content)) # Convert to RGB if necessary (to save as JPEG) if img.mode in ("RGBA", "P"): img = img.convert("RGB") # Resize if larger than max dimensions img.thumbnail(MAX_IMAGE_SIZE, Image.Resampling.LANCZOS) # Save to bytes output_buffer = io.BytesIO() img.save(output_buffer, format="JPEG", quality=IMAGE_QUALITY, optimize=True) return output_buffer.getvalue() async def extract_receipt_data(image_content: bytes, user_name: str, department: str) -> ExtractionResponse: # 1. Compress Image compressed_content = compress_image(image_content) base64_image = base64.b64encode(compressed_content).decode("utf-8") # 2. Refined Prompt prompt = ( f"You are a cautious auditor helping an HR department in Malaysia. " f"Extract the requested fields from the provided medical receipt image. " f"The employee submitting this is {user_name} from {department}. " f"IMPORTANT: The context is Malaysia (MYR). " f"For the fields `receipt_ref_no` and `clinic_reg_no`, only provide a value if you can read it clearly without any guessing or inference. If the text is smudged, handwritten, or ambiguous, return `null`. " f"Map the clinic/services to a `claim_category` from: [General, Dental, Optical, Specialist] based on the clinic name or invoice items. " f"Provide a 1-sentence `diagnosis_brief` summarizing the services seen (e.g. 'Fever consultation and medicine'). " f"Set `needs_manual_review` to `true` and provide a low `confidence_score` if: " f"1. The 'Total' does not match the sum of the individual items. " f"2. The receipt looks hand-written and lacks an official stamp. " f"3. The provider name is missing or the amount looks altered. " f"4. The user's name ({user_name}) is not clearly visible on the receipt. " f"IMPORTANT: Fill the `ai_reasoning` field with a 1-sentence explanation of how you identified the clinic and category." ) # 3. Async Extraction completion = await client.beta.chat.completions.parse( model="gpt-4o-mini", messages=[ { "role": "system", "content": "You are an HR data entry assistant. Extract medical receipt data accurately into structured JSON." }, { "role": "user", "content": [ {"type": "text", "text": prompt}, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}, }, ], } ], response_format=ExtractionResponse, ) result = completion.choices[0].message.parsed # 4. Logging for Demo if result: logger.info(f"Extraction complete for {user_name}. Confidence Score: {result.confidence_score}") if result.needs_manual_review: logger.warning(f"Manual review required for receipt submitted by {user_name}") return result