live_analysis_screen.dart 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503
  1. import 'dart:io';
  2. import 'dart:async';
  3. import 'package:flutter/material.dart';
  4. import 'package:camera/camera.dart';
  5. import 'package:permission_handler/permission_handler.dart';
  6. import '../services/tflite_service.dart';
  7. import '../widgets/palm_bounding_box.dart';
  8. import '../constants/mpob_colors.dart';
  9. import '../services/app_strings.dart';
  10. import '../theme/locale_controller.dart';
  11. enum DetectionState { searching, locking, capturing, cooldown }
  12. class LiveAnalysisScreen extends StatefulWidget {
  13. const LiveAnalysisScreen({super.key});
  14. @override
  15. State<LiveAnalysisScreen> createState() => _LiveAnalysisScreenState();
  16. }
  17. class _LiveAnalysisScreenState extends State<LiveAnalysisScreen> {
  18. CameraController? _controller;
  19. final TfliteService _tfliteService = TfliteService();
  20. bool _isInitialized = false;
  21. bool _isProcessing = false;
  22. List<DetectionResult>? _detections;
  23. // Detection Lock Logic
  24. DetectionState _state = DetectionState.searching;
  25. static const double _lockThreshold = 0.60;
  26. // Frame Throttle removed: Atomic Idle Lock active.
  27. final List<bool> _detectionHistory = List.filled(15, false, growable: true); // 15 frames buffer
  28. static const int _requiredHits = 5; // Target hits for momentum lock
  29. int _currentHits = 0; // Track hits for the timer
  30. Timer? _lockTimer;
  31. Timer? _cooldownTimer;
  32. double _lockProgress = 0.0;
  33. bool _showFlash = false;
  34. // Raw CameraImage stream frames arrive in the sensor's native (landscape)
  35. // orientation, never pre-rotated for display — unlike CameraPreview's own
  36. // texture, which is rotated internally to appear upright. Detection boxes
  37. // decoded from the stream must be rotated by this same amount before
  38. // rendering, or they land in the wrong place/orientation relative to what's
  39. // shown on screen. Defaults to 90 (the standard rear-camera value on
  40. // Android) until the actual camera's sensorOrientation is known.
  41. int _sensorOrientation = 90;
  42. @override
  43. void initState() {
  44. super.initState();
  45. _initializeCamera();
  46. }
  47. Future<void> _initializeCamera() async {
  48. final status = await Permission.camera.request();
  49. if (status.isDenied) return;
  50. final cameras = await availableCameras();
  51. if (cameras.isEmpty) return;
  52. _sensorOrientation = cameras[0].sensorOrientation;
  53. _controller = CameraController(
  54. cameras[0],
  55. ResolutionPreset.medium, // Upgraded resolution for YOLO26 performance
  56. enableAudio: false,
  57. imageFormatGroup: Platform.isAndroid ? ImageFormatGroup.yuv420 : ImageFormatGroup.bgra8888,
  58. );
  59. try {
  60. await _controller!.initialize();
  61. await _tfliteService.initModel();
  62. _controller!.startImageStream(_handleImageStream);
  63. if (mounted) {
  64. setState(() {
  65. _isInitialized = true;
  66. });
  67. }
  68. } catch (e) {
  69. debugPrint("Camera init error: $e");
  70. }
  71. }
  72. void _handleImageStream(CameraImage image) {
  73. // Atomic Idle Lock: Only process if the Persistent Isolate is truly idle
  74. if (_isProcessing || _tfliteService.isIsolateBusy || _state == DetectionState.capturing || _state == DetectionState.cooldown) {
  75. return;
  76. }
  77. _processStreamFrame(image);
  78. }
  79. Future<void> _processStreamFrame(CameraImage image) async {
  80. setState(() => _isProcessing = true);
  81. try {
  82. final detections = _rotateDetections(await _tfliteService.runInferenceOnStream(image));
  83. bool currentFrameHasFruit = false;
  84. if (detections.isNotEmpty) {
  85. currentFrameHasFruit = detections.any((d) => d.confidence > _lockThreshold);
  86. }
  87. // Update Sliding Window Buffer
  88. _detectionHistory.removeAt(0);
  89. _detectionHistory.add(currentFrameHasFruit);
  90. _currentHits = _detectionHistory.where((h) => h).length;
  91. if (!mounted) return;
  92. setState(() {
  93. _detections = detections;
  94. });
  95. if (_state == DetectionState.searching) {
  96. if (_currentHits >= _requiredHits) {
  97. setState(() {
  98. _state = DetectionState.locking;
  99. _lockProgress = 0.0;
  100. });
  101. _startLockTimer();
  102. }
  103. }
  104. // Removed the old strict cancel logic.
  105. // _startLockTimer now safely handles momentum drain.
  106. } catch (e) {
  107. debugPrint("Stream processing error: $e");
  108. } finally {
  109. if (mounted) {
  110. setState(() => _isProcessing = false);
  111. }
  112. }
  113. }
  114. /// Rotates each detection's normalized box from raw sensor-space into
  115. /// display-space, matching the rotation CameraPreview's own texture
  116. /// undergoes internally. Confidence/class are untouched.
  117. List<DetectionResult> _rotateDetections(List<DetectionResult> detections) {
  118. final quarterTurns = (_sensorOrientation ~/ 90) % 4;
  119. if (quarterTurns == 0) return detections;
  120. return detections
  121. .map((d) => DetectionResult(
  122. className: d.className,
  123. classIndex: d.classIndex,
  124. confidence: d.confidence,
  125. normalizedBox: _rotateNormalizedRect(d.normalizedBox, quarterTurns),
  126. ))
  127. .toList();
  128. }
  129. /// Rotates an axis-aligned rect in normalized [0,1]x[0,1] space by
  130. /// [quarterTurns] * 90° clockwise.
  131. Rect _rotateNormalizedRect(Rect r, int quarterTurns) {
  132. switch (quarterTurns) {
  133. case 1: // 90° clockwise
  134. return Rect.fromLTRB(1 - r.bottom, r.left, 1 - r.top, r.right);
  135. case 2: // 180°
  136. return Rect.fromLTRB(1 - r.right, 1 - r.bottom, 1 - r.left, 1 - r.top);
  137. case 3: // 270° clockwise (90° counter-clockwise)
  138. return Rect.fromLTRB(r.top, 1 - r.right, r.bottom, 1 - r.left);
  139. default:
  140. return r;
  141. }
  142. }
  143. void _startLockTimer() {
  144. _lockTimer?.cancel();
  145. const duration = Duration(milliseconds: 100);
  146. int momentumTicks = 0;
  147. _lockTimer = Timer.periodic(duration, (timer) {
  148. if (!mounted) {
  149. timer.cancel();
  150. return;
  151. }
  152. // Momentum logic: add or subtract
  153. if (_currentHits >= _requiredHits) {
  154. momentumTicks++;
  155. } else {
  156. momentumTicks--;
  157. }
  158. if (momentumTicks < 0) momentumTicks = 0;
  159. setState(() {
  160. _lockProgress = (momentumTicks / 2.0).clamp(0.0, 1.0); // 2 ticks target
  161. });
  162. if (momentumTicks >= 2) {
  163. timer.cancel();
  164. if (_state == DetectionState.locking) {
  165. _triggerCapture();
  166. }
  167. } else if (momentumTicks <= 0 && _state == DetectionState.locking) {
  168. // Complete momentum loss -> Cancel lock
  169. timer.cancel();
  170. setState(() {
  171. _state = DetectionState.searching;
  172. _lockProgress = 0.0;
  173. });
  174. }
  175. });
  176. }
  177. Future<void> _triggerCapture() async {
  178. setState(() {
  179. _state = DetectionState.capturing;
  180. _lockProgress = 1.0;
  181. _showFlash = true;
  182. });
  183. // Quick 200ms white flash without blocking
  184. Future.delayed(const Duration(milliseconds: 200), () {
  185. if (mounted) setState(() => _showFlash = false);
  186. });
  187. await _captureAndAnalyze();
  188. }
  189. /// Live Inference is a quick verification tool, not a capture mode — the
  190. /// result is shown once and then discarded. Nothing is written to history;
  191. /// use Snap & Analyze if you want the result kept.
  192. Future<void> _captureAndAnalyze() async {
  193. if (_controller == null || !_controller!.value.isInitialized) return;
  194. // 1. Stop stream to avoid resource conflict
  195. await _controller!.stopImageStream();
  196. if (!mounted) return;
  197. try {
  198. // 2. Take high-res picture
  199. final XFile photo = await _controller!.takePicture();
  200. // 3. Run final inference on high-res
  201. final detections = await _tfliteService.runInference(photo.path);
  202. // 4. Discard the temp capture — nothing persists past this quick check.
  203. unawaited(File(photo.path).delete().catchError((_) => File(photo.path)));
  204. if (detections.isNotEmpty) {
  205. final best = detections.first;
  206. if (mounted) {
  207. await _showResultSheet(best.className, best.confidence);
  208. _startCooldown();
  209. }
  210. } else {
  211. if (mounted) {
  212. ScaffoldMessenger.of(context).showSnackBar(
  213. SnackBar(content: Text(AppStrings.t('live_no_detections_snackbar')))
  214. );
  215. _startCooldown();
  216. }
  217. }
  218. } catch (e) {
  219. debugPrint("Capture error: $e");
  220. if (mounted) _startCooldown();
  221. }
  222. }
  223. void _startCooldown() {
  224. if (!mounted) return;
  225. setState(() {
  226. _state = DetectionState.cooldown;
  227. _detections = null; // Clear boxes
  228. });
  229. // Clear detection history to ignore old hits
  230. _detectionHistory.fillRange(0, _detectionHistory.length, false);
  231. _cooldownTimer?.cancel();
  232. _cooldownTimer = Timer(const Duration(seconds: 3), () {
  233. if (!mounted) return;
  234. setState(() {
  235. _state = DetectionState.searching;
  236. });
  237. _controller?.startImageStream(_handleImageStream);
  238. });
  239. }
  240. Future<void> _showResultSheet(String ripenessClass, double confidence) async {
  241. final Color statusColor = getStatusColor(ripenessClass);
  242. final colorScheme = Theme.of(context).colorScheme;
  243. await showModalBottomSheet(
  244. context: context,
  245. isScrollControlled: true,
  246. isDismissible: false,
  247. enableDrag: false,
  248. shape: const RoundedRectangleBorder(borderRadius: BorderRadius.vertical(top: Radius.circular(20))),
  249. builder: (context) => Container(
  250. padding: const EdgeInsets.all(24),
  251. decoration: BoxDecoration(
  252. color: colorScheme.surfaceContainerHighest,
  253. borderRadius: const BorderRadius.vertical(top: Radius.circular(20)),
  254. ),
  255. child: Column(
  256. mainAxisSize: MainAxisSize.min,
  257. children: [
  258. Icon(Icons.check_circle, color: statusColor, size: 64),
  259. const SizedBox(height: 16),
  260. Text(AppStrings.className(ripenessClass), style: TextStyle(fontSize: 28, fontWeight: FontWeight.bold, color: colorScheme.onSurface)),
  261. Text(
  262. AppStrings.tp('confidence_label', {'percent': (confidence * 100).toStringAsFixed(1)}),
  263. style: TextStyle(color: colorScheme.onSurface.withValues(alpha: 0.6)),
  264. ),
  265. const SizedBox(height: 24),
  266. if (ripenessClass == 'Empty_Bunch' || ripenessClass == 'Abnormal')
  267. Container(
  268. padding: const EdgeInsets.all(12),
  269. decoration: BoxDecoration(color: Colors.red.shade50, borderRadius: BorderRadius.circular(8)),
  270. child: Text(AppStrings.t('health_alert_abnormal'), style: const TextStyle(color: Colors.red, fontWeight: FontWeight.bold)),
  271. ),
  272. const SizedBox(height: 24),
  273. SizedBox(
  274. width: double.infinity,
  275. child: ElevatedButton(
  276. onPressed: () => Navigator.pop(context),
  277. child: Text(AppStrings.t('done_button')),
  278. ),
  279. ),
  280. ],
  281. ),
  282. ),
  283. );
  284. }
  285. @override
  286. Widget build(BuildContext context) {
  287. return ValueListenableBuilder<String>(
  288. valueListenable: LocaleController.localeCode,
  289. builder: (context, _, _) => _buildScaffold(context),
  290. );
  291. }
  292. Widget _buildScaffold(BuildContext context) {
  293. if (!_isInitialized || _controller == null) {
  294. return const Scaffold(body: Center(child: CircularProgressIndicator()));
  295. }
  296. final isLockedVisual = _state == DetectionState.locking || _state == DetectionState.capturing;
  297. return Scaffold(
  298. backgroundColor: Colors.black,
  299. body: Stack(
  300. children: [
  301. // Camera Preview + Bounding Box Overlays. The overlay is passed as
  302. // CameraPreview's own `child` slot (not a separate sibling) so it's
  303. // measured against the exact same orientation-aware AspectRatio box
  304. // CameraPreview computes internally for the texture itself —
  305. // otherwise the overlay measures the full screen while the preview
  306. // is letterboxed/pillarboxed to the camera's real aspect ratio,
  307. // causing misaligned/stretched boxes.
  308. Center(
  309. child: CameraPreview(
  310. _controller!,
  311. child: _detections != null && _state != DetectionState.capturing && _state != DetectionState.cooldown
  312. ? LayoutBuilder(
  313. builder: (context, constraints) {
  314. return Stack(
  315. children: _detections!
  316. .map((d) => PalmBoundingBox(
  317. normalizedRect: d.normalizedBox,
  318. label: d.className,
  319. confidence: d.confidence,
  320. constraints: constraints,
  321. isLocked: (_state == DetectionState.locking || _state == DetectionState.capturing) && d.confidence > _lockThreshold,
  322. ))
  323. .toList(),
  324. );
  325. },
  326. )
  327. : null,
  328. ),
  329. ),
  330. // Top Info Bar
  331. Positioned(
  332. top: 40,
  333. left: 20,
  334. right: 20,
  335. child: Container(
  336. padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 8),
  337. decoration: BoxDecoration(
  338. color: Colors.black54,
  339. borderRadius: BorderRadius.circular(20),
  340. ),
  341. child: Row(
  342. children: [
  343. Icon(
  344. _state == DetectionState.cooldown ? Icons.pause_circle_filled :
  345. isLockedVisual ? Icons.lock : Icons.center_focus_weak,
  346. color: _state == DetectionState.cooldown ? Colors.blue :
  347. isLockedVisual ? Colors.green : Colors.yellow,
  348. ),
  349. const SizedBox(width: 8),
  350. Text(
  351. _state == DetectionState.cooldown ? AppStrings.t('live_state_cooldown') :
  352. isLockedVisual ? AppStrings.t('live_state_locking') : AppStrings.t('live_state_searching'),
  353. style: TextStyle(
  354. color: _state == DetectionState.cooldown ? Colors.blue :
  355. isLockedVisual ? Colors.green : Colors.yellow,
  356. fontWeight: FontWeight.bold,
  357. ),
  358. ),
  359. const Spacer(),
  360. IconButton(
  361. icon: const Icon(Icons.close, color: Colors.white),
  362. onPressed: () => Navigator.pop(context),
  363. ),
  364. ],
  365. ),
  366. ),
  367. ),
  368. // Progress Overlay for Locking
  369. if (_state == DetectionState.locking)
  370. Positioned.fill(
  371. child: Center(
  372. child: SizedBox(
  373. width: 120,
  374. height: 120,
  375. child: TweenAnimationBuilder<double>(
  376. tween: Tween<double>(begin: 0.0, end: _lockProgress),
  377. duration: const Duration(milliseconds: 100),
  378. builder: (context, value, _) => CircularProgressIndicator(
  379. value: value,
  380. strokeWidth: 8,
  381. color: Colors.greenAccent,
  382. backgroundColor: Colors.white24,
  383. ),
  384. ),
  385. ),
  386. ),
  387. ),
  388. // White flash overlay
  389. Positioned.fill(
  390. child: IgnorePointer(
  391. child: AnimatedOpacity(
  392. opacity: _showFlash ? 1.0 : 0.0,
  393. duration: const Duration(milliseconds: 200),
  394. child: Container(color: Colors.white),
  395. ),
  396. ),
  397. ),
  398. if (_state == DetectionState.capturing && !_showFlash)
  399. Positioned.fill(
  400. child: Container(
  401. color: Colors.black45,
  402. child: const Center(
  403. child: CircularProgressIndicator(color: Colors.white),
  404. ),
  405. ),
  406. ),
  407. if (_state == DetectionState.cooldown)
  408. Positioned.fill(
  409. child: Container(
  410. color: Colors.black45,
  411. child: Center(
  412. child: Text(
  413. AppStrings.t('live_resuming_scan'),
  414. style: const TextStyle(color: Colors.white, fontSize: 18, fontWeight: FontWeight.bold),
  415. ),
  416. ),
  417. ),
  418. ),
  419. // Bottom Hint
  420. if (_state == DetectionState.searching)
  421. Positioned(
  422. bottom: 40,
  423. left: 0,
  424. right: 0,
  425. child: Center(
  426. child: Text(
  427. AppStrings.t('live_hold_steady_hint'),
  428. style: const TextStyle(color: Colors.white, fontWeight: FontWeight.w500),
  429. ),
  430. ),
  431. ),
  432. ],
  433. ),
  434. );
  435. }
  436. @override
  437. void dispose() {
  438. _lockTimer?.cancel();
  439. _cooldownTimer?.cancel();
  440. _controller?.dispose();
  441. _tfliteService.dispose();
  442. super.dispose();
  443. }
  444. }