|
|
@@ -1,6 +1,8 @@
|
|
|
import 'dart:io';
|
|
|
import 'dart:math';
|
|
|
import 'dart:ui';
|
|
|
+import 'dart:isolate';
|
|
|
+import 'dart:async';
|
|
|
import 'package:flutter/services.dart';
|
|
|
import 'package:flutter/foundation.dart';
|
|
|
import 'package:image/image.dart' as img;
|
|
|
@@ -8,12 +10,10 @@ import 'package:image_picker/image_picker.dart';
|
|
|
import 'package:tflite_flutter/tflite_flutter.dart';
|
|
|
import 'package:camera/camera.dart';
|
|
|
|
|
|
-/// A detection result parsed from the model's end-to-end output.
|
|
|
class DetectionResult {
|
|
|
final String className;
|
|
|
final int classIndex;
|
|
|
final double confidence;
|
|
|
- /// Normalized bounding box (0.0 - 1.0)
|
|
|
final Rect normalizedBox;
|
|
|
|
|
|
const DetectionResult({
|
|
|
@@ -24,44 +24,73 @@ class DetectionResult {
|
|
|
});
|
|
|
|
|
|
Color getStatusColor() {
|
|
|
- if (className == 'Empty_Bunch' || className == 'Abnormal') return const Color(0xFFF44336); // Colors.red
|
|
|
- if (className == 'Ripe' || className == 'Overripe') return const Color(0xFF4CAF50); // Colors.green
|
|
|
- return const Color(0xFFFF9800); // Colors.orange
|
|
|
+ if (className == 'Empty_Bunch' || className == 'Abnormal') return const Color(0xFFF44336);
|
|
|
+ if (className == 'Ripe' || className == 'Overripe') return const Color(0xFF4CAF50);
|
|
|
+ return const Color(0xFFFF9800);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-/// Custom TFLite inference service that correctly decodes the end-to-end
|
|
|
-/// YOLO model output format [1, N, 6] = [batch, detections, (x1,y1,x2,y2,conf,class_id)].
|
|
|
class TfliteService {
|
|
|
static const _modelAsset = 'best.tflite';
|
|
|
static const _labelsAsset = 'labels.txt';
|
|
|
static const int _inputSize = 640;
|
|
|
static const double _confidenceThreshold = 0.25;
|
|
|
|
|
|
- Interpreter? _interpreter;
|
|
|
+ Isolate? _isolate;
|
|
|
+ SendPort? _sendPort;
|
|
|
+ ReceivePort? _receivePort;
|
|
|
+
|
|
|
List<String> _labels = [];
|
|
|
final ImagePicker _picker = ImagePicker();
|
|
|
bool _isInitialized = false;
|
|
|
+ bool _isIsolateBusy = false;
|
|
|
|
|
|
bool get isInitialized => _isInitialized;
|
|
|
|
|
|
Future<void> initModel() async {
|
|
|
try {
|
|
|
- // Load labels
|
|
|
final labelData = await rootBundle.loadString('assets/$_labelsAsset');
|
|
|
_labels = labelData.split('\n').where((l) => l.trim().isNotEmpty).map((l) => l.trim()).toList();
|
|
|
|
|
|
- // Load model
|
|
|
- final interpreterOptions = InterpreterOptions()..threads = 4;
|
|
|
- _interpreter = await Interpreter.fromAsset(
|
|
|
- 'assets/$_modelAsset',
|
|
|
- options: interpreterOptions,
|
|
|
- );
|
|
|
+ final modelData = await rootBundle.load('assets/$_modelAsset');
|
|
|
+ final modelBytes = modelData.buffer.asUint8List();
|
|
|
+
|
|
|
+ _receivePort = ReceivePort();
|
|
|
+ _isolate = await Isolate.spawn(_isolateEntry, _receivePort!.sendPort);
|
|
|
+
|
|
|
+ final completer = Completer<SendPort>();
|
|
|
+ StreamSubscription? sub;
|
|
|
+ sub = _receivePort!.listen((message) {
|
|
|
+ if (message is SendPort) {
|
|
|
+ completer.complete(message);
|
|
|
+ sub?.cancel();
|
|
|
+ }
|
|
|
+ });
|
|
|
+ _sendPort = await completer.future;
|
|
|
+
|
|
|
+ final initCompleter = Completer<void>();
|
|
|
+ final initReplyPort = ReceivePort();
|
|
|
+
|
|
|
+ _sendPort!.send({
|
|
|
+ 'command': 'init',
|
|
|
+ 'modelBytes': modelBytes,
|
|
|
+ 'labelData': labelData,
|
|
|
+ 'replyPort': initReplyPort.sendPort,
|
|
|
+ });
|
|
|
+
|
|
|
+ StreamSubscription? initSub;
|
|
|
+ initSub = initReplyPort.listen((message) {
|
|
|
+ if (message == 'init_done') {
|
|
|
+ initCompleter.complete();
|
|
|
+ initSub?.cancel();
|
|
|
+ initReplyPort.close();
|
|
|
+ }
|
|
|
+ });
|
|
|
+
|
|
|
+ await initCompleter.future;
|
|
|
|
|
|
_isInitialized = true;
|
|
|
- print('TfliteService: Model loaded. Labels: $_labels');
|
|
|
- print('TfliteService: Input: ${_interpreter!.getInputTensors().map((t) => t.shape)}');
|
|
|
- print('TfliteService: Output: ${_interpreter!.getOutputTensors().map((t) => t.shape)}');
|
|
|
+ print('TfliteService: Model loaded via persistent isolate.');
|
|
|
} catch (e) {
|
|
|
print('TfliteService init error: $e');
|
|
|
rethrow;
|
|
|
@@ -76,26 +105,31 @@ class TfliteService {
|
|
|
);
|
|
|
}
|
|
|
|
|
|
- /// Run inference on the image at [imagePath].
|
|
|
- /// Returns a list of [DetectionResult] sorted by confidence descending.
|
|
|
- /// Offloaded to a background isolate to keep UI smooth.
|
|
|
Future<List<DetectionResult>> runInference(String imagePath) async {
|
|
|
if (!_isInitialized) await initModel();
|
|
|
|
|
|
final imageBytes = await File(imagePath).readAsBytes();
|
|
|
|
|
|
- // We pass the raw bytes and asset paths to the isolate.
|
|
|
- // The isolate will handle decoding, resizing, and inference.
|
|
|
- return await _runInferenceInIsolate(imageBytes);
|
|
|
+ final replyPort = ReceivePort();
|
|
|
+ _sendPort!.send({
|
|
|
+ 'command': 'inference_static',
|
|
|
+ 'imageBytes': imageBytes,
|
|
|
+ 'replyPort': replyPort.sendPort,
|
|
|
+ });
|
|
|
+
|
|
|
+ final detections = await replyPort.first;
|
|
|
+ replyPort.close();
|
|
|
+ return detections as List<DetectionResult>;
|
|
|
}
|
|
|
|
|
|
- /// Run inference on a [CameraImage] from the stream.
|
|
|
- /// Throttled by the caller.
|
|
|
Future<List<DetectionResult>> runInferenceOnStream(CameraImage image) async {
|
|
|
if (!_isInitialized) await initModel();
|
|
|
+ if (_isIsolateBusy) return <DetectionResult>[];
|
|
|
|
|
|
- // We pass the CameraImage planes to the isolate for conversion and inference.
|
|
|
- return await compute(_inferenceStreamTaskWrapper, {
|
|
|
+ _isIsolateBusy = true;
|
|
|
+ final replyPort = ReceivePort();
|
|
|
+ _sendPort!.send({
|
|
|
+ 'command': 'inference_stream',
|
|
|
'planes': image.planes.map((p) => {
|
|
|
'bytes': p.bytes,
|
|
|
'bytesPerRow': p.bytesPerRow,
|
|
|
@@ -104,37 +138,119 @@ class TfliteService {
|
|
|
'width': image.width,
|
|
|
'height': image.height,
|
|
|
'format': image.format.group,
|
|
|
- 'modelBytes': (await rootBundle.load('assets/$_modelAsset')).buffer.asUint8List(),
|
|
|
- 'labelData': await rootBundle.loadString('assets/$_labelsAsset'),
|
|
|
+ 'replyPort': replyPort.sendPort,
|
|
|
});
|
|
|
+
|
|
|
+ final detections = await replyPort.first;
|
|
|
+ replyPort.close();
|
|
|
+ _isIsolateBusy = false;
|
|
|
+ return detections as List<DetectionResult>;
|
|
|
}
|
|
|
|
|
|
- static List<DetectionResult> _inferenceStreamTaskWrapper(Map<String, dynamic> args) {
|
|
|
- final modelBytes = args['modelBytes'] as Uint8List;
|
|
|
- final labelData = args['labelData'] as String;
|
|
|
- final planes = args['planes'] as List<dynamic>;
|
|
|
- final width = args['width'] as int;
|
|
|
- final height = args['height'] as int;
|
|
|
-
|
|
|
- final interpreter = Interpreter.fromBuffer(modelBytes);
|
|
|
- final labels = labelData.split('\n').where((l) => l.trim().isNotEmpty).map((l) => l.trim()).toList();
|
|
|
+ static void _isolateEntry(SendPort sendPort) {
|
|
|
+ final receivePort = ReceivePort();
|
|
|
+ sendPort.send(receivePort.sendPort);
|
|
|
+
|
|
|
+ Interpreter? interpreter;
|
|
|
+ List<String> labels = [];
|
|
|
+
|
|
|
+ receivePort.listen((message) {
|
|
|
+ if (message is Map) {
|
|
|
+ final command = message['command'];
|
|
|
+ final replyPort = message['replyPort'] as SendPort;
|
|
|
+
|
|
|
+ if (command == 'init') {
|
|
|
+ final modelBytes = message['modelBytes'] as Uint8List;
|
|
|
+ final labelData = message['labelData'] as String;
|
|
|
+
|
|
|
+ final interpreterOptions = InterpreterOptions()..threads = 4;
|
|
|
+ interpreter = Interpreter.fromBuffer(modelBytes, options: interpreterOptions);
|
|
|
+ labels = labelData.split('\n').where((l) => l.trim().isNotEmpty).map((l) => l.trim()).toList();
|
|
|
+
|
|
|
+ replyPort.send('init_done');
|
|
|
+ } else if (command == 'inference_static') {
|
|
|
+ if (interpreter == null) {
|
|
|
+ replyPort.send(<DetectionResult>[]);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ final imageBytes = message['imageBytes'] as Uint8List;
|
|
|
+ final results = _inferenceStaticTask(imageBytes, interpreter!, labels);
|
|
|
+ replyPort.send(results);
|
|
|
+ } else if (command == 'inference_stream') {
|
|
|
+ if (interpreter == null) {
|
|
|
+ replyPort.send(<DetectionResult>[]);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ final planes = message['planes'] as List<dynamic>;
|
|
|
+ final width = message['width'] as int;
|
|
|
+ final height = message['height'] as int;
|
|
|
+ final format = message['format'];
|
|
|
+
|
|
|
+ final results = _inferenceStreamTask(planes, width, height, format, interpreter!, labels);
|
|
|
+ replyPort.send(results);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ static List<DetectionResult> _inferenceStaticTask(Uint8List imageBytes, Interpreter interpreter, List<String> labels) {
|
|
|
+ try {
|
|
|
+ final decoded = img.decodeImage(imageBytes);
|
|
|
+ if (decoded == null) throw Exception('Could not decode image');
|
|
|
+
|
|
|
+ final int width = decoded.width;
|
|
|
+ final int height = decoded.height;
|
|
|
+ final int size = width < height ? width : height;
|
|
|
+ final int offsetX = (width - size) ~/ 2;
|
|
|
+ final int offsetY = (height - size) ~/ 2;
|
|
|
+
|
|
|
+ final cropped = img.copyCrop(decoded, x: offsetX, y: offsetY, width: size, height: size);
|
|
|
+ final resized = img.copyResize(cropped, width: _inputSize, height: _inputSize, interpolation: img.Interpolation.linear);
|
|
|
+
|
|
|
+ final inputTensor = List.generate(1, (_) =>
|
|
|
+ List.generate(_inputSize, (y) =>
|
|
|
+ List.generate(_inputSize, (x) {
|
|
|
+ final pixel = resized.getPixel(x, y);
|
|
|
+ return [pixel.r / 255.0, pixel.g / 255.0, pixel.b / 255.0];
|
|
|
+ })
|
|
|
+ )
|
|
|
+ );
|
|
|
+
|
|
|
+ final outputShape = interpreter.getOutputTensors()[0].shape;
|
|
|
+ final outputTensor = List.generate(1, (_) =>
|
|
|
+ List.generate(outputShape[1], (_) =>
|
|
|
+ List<double>.filled(outputShape[2], 0.0)
|
|
|
+ )
|
|
|
+ );
|
|
|
|
|
|
+ interpreter.run(inputTensor, outputTensor);
|
|
|
+
|
|
|
+ return _decodeDetections(
|
|
|
+ outputTensor[0],
|
|
|
+ labels,
|
|
|
+ cropSize: size,
|
|
|
+ offsetX: offsetX,
|
|
|
+ offsetY: offsetY,
|
|
|
+ fullWidth: width,
|
|
|
+ fullHeight: height
|
|
|
+ );
|
|
|
+ } catch (e) {
|
|
|
+ print('Isolate static inference error: $e');
|
|
|
+ return <DetectionResult>[];
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ static List<DetectionResult> _inferenceStreamTask(
|
|
|
+ List<dynamic> planes, int width, int height, dynamic format,
|
|
|
+ Interpreter interpreter, List<String> labels
|
|
|
+ ) {
|
|
|
try {
|
|
|
final size = width < height ? width : height;
|
|
|
final offsetX = (width - size) ~/ 2;
|
|
|
final offsetY = (height - size) ~/ 2;
|
|
|
|
|
|
img.Image? image;
|
|
|
- if (args['format'] == ImageFormatGroup.yuv420) {
|
|
|
- image = _convertYUV420ToImage(
|
|
|
- planes: planes,
|
|
|
- width: width,
|
|
|
- height: height,
|
|
|
- cropSize: size,
|
|
|
- offsetX: offsetX,
|
|
|
- offsetY: offsetY,
|
|
|
- );
|
|
|
- } else if (args['format'] == ImageFormatGroup.bgra8888) {
|
|
|
+ if (format == ImageFormatGroup.bgra8888) {
|
|
|
final fullImage = img.Image.fromBytes(
|
|
|
width: width,
|
|
|
height: height,
|
|
|
@@ -144,11 +260,20 @@ class TfliteService {
|
|
|
order: img.ChannelOrder.bgra,
|
|
|
);
|
|
|
image = img.copyCrop(fullImage, x: offsetX, y: offsetY, width: size, height: size);
|
|
|
+ } else if (format == ImageFormatGroup.yuv420) {
|
|
|
+ image = _convertYUV420ToImage(
|
|
|
+ planes: planes,
|
|
|
+ width: width,
|
|
|
+ height: height,
|
|
|
+ cropSize: size,
|
|
|
+ offsetX: offsetX,
|
|
|
+ offsetY: offsetY,
|
|
|
+ );
|
|
|
+ } else {
|
|
|
+ print("TfliteService: Unsupported format: $format. Ensure platform correctly requests YUV420 or BGRA.");
|
|
|
+ return <DetectionResult>[];
|
|
|
}
|
|
|
|
|
|
- if (image == null) return [];
|
|
|
-
|
|
|
- // Resize and Run
|
|
|
final resized = img.copyResize(image, width: _inputSize, height: _inputSize);
|
|
|
|
|
|
final inputTensor = List.generate(1, (_) =>
|
|
|
@@ -169,7 +294,6 @@ class TfliteService {
|
|
|
|
|
|
interpreter.run(inputTensor, outputTensor);
|
|
|
|
|
|
- // Map detections back to full frame
|
|
|
return _decodeDetections(
|
|
|
outputTensor[0],
|
|
|
labels,
|
|
|
@@ -179,8 +303,9 @@ class TfliteService {
|
|
|
fullWidth: width,
|
|
|
fullHeight: height
|
|
|
);
|
|
|
- } finally {
|
|
|
- interpreter.close();
|
|
|
+ } catch (e) {
|
|
|
+ print('Isolate stream inference error: $e');
|
|
|
+ return <DetectionResult>[];
|
|
|
}
|
|
|
}
|
|
|
|
|
|
@@ -204,32 +329,52 @@ class TfliteService {
|
|
|
final uvRowStride = uPlane['bytesPerRow'] as int;
|
|
|
final uvPixelStride = uPlane['bytesPerPixel'] as int;
|
|
|
|
|
|
- final image = img.Image(width: cropSize, height: cropSize);
|
|
|
+ // Use a flat Uint8List buffer for fast native-style memory writing
|
|
|
+ // 3 channels: R, G, B
|
|
|
+ final Uint8List rgbBytes = Uint8List(cropSize * cropSize * 3);
|
|
|
+ int bufferIndex = 0;
|
|
|
|
|
|
for (int y = 0; y < cropSize; y++) {
|
|
|
for (int x = 0; x < cropSize; x++) {
|
|
|
final int actualX = x + offsetX;
|
|
|
final int actualY = y + offsetY;
|
|
|
|
|
|
- final int uvIndex = (uvRowStride * (actualY / 2).floor()) + (uvPixelStride * (actualX / 2).floor());
|
|
|
+ // Mathematical offset matching
|
|
|
+ final int uvIndex = (uvRowStride * (actualY >> 1)) + (uvPixelStride * (actualX >> 1));
|
|
|
final int yIndex = (actualY * yRowStride) + actualX;
|
|
|
|
|
|
- // Ensure we don't go out of bounds
|
|
|
- if (yIndex >= yBytes.length || uvIndex >= uBytes.length || uvIndex >= vBytes.length) continue;
|
|
|
+ // Skip if out of bounds (should not happen mathematically if offsets are valid,
|
|
|
+ // but kept as safety check for corrupted frames)
|
|
|
+ if (yIndex >= yBytes.length || uvIndex >= uBytes.length || uvIndex >= vBytes.length) {
|
|
|
+ bufferIndex += 3;
|
|
|
+ continue;
|
|
|
+ }
|
|
|
|
|
|
final int yp = yBytes[yIndex];
|
|
|
final int up = uBytes[uvIndex];
|
|
|
final int vp = vBytes[uvIndex];
|
|
|
|
|
|
// Standard YUV to RGB conversion
|
|
|
- int r = (yp + (1.370705 * (vp - 128))).toInt().clamp(0, 255);
|
|
|
- int g = (yp - (0.337633 * (up - 128)) - (0.698001 * (vp - 128))).toInt().clamp(0, 255);
|
|
|
- int b = (yp + (1.732446 * (up - 128))).toInt().clamp(0, 255);
|
|
|
-
|
|
|
- image.setPixelRgb(x, y, r, g, b);
|
|
|
+ int r = (yp + (1.370705 * (vp - 128))).toInt();
|
|
|
+ int g = (yp - (0.337633 * (up - 128)) - (0.698001 * (vp - 128))).toInt();
|
|
|
+ int b = (yp + (1.732446 * (up - 128))).toInt();
|
|
|
+
|
|
|
+ // Write directly to sequential memory with inline clamping
|
|
|
+ rgbBytes[bufferIndex++] = r < 0 ? 0 : (r > 255 ? 255 : r);
|
|
|
+ rgbBytes[bufferIndex++] = g < 0 ? 0 : (g > 255 ? 255 : g);
|
|
|
+ rgbBytes[bufferIndex++] = b < 0 ? 0 : (b > 255 ? 255 : b);
|
|
|
}
|
|
|
}
|
|
|
- return image;
|
|
|
+
|
|
|
+ // Construct image mapping directly from the fast buffer
|
|
|
+ return img.Image.fromBytes(
|
|
|
+ width: cropSize,
|
|
|
+ height: cropSize,
|
|
|
+ bytes: rgbBytes.buffer,
|
|
|
+ format: img.Format.uint8,
|
|
|
+ numChannels: 3,
|
|
|
+ order: img.ChannelOrder.rgb,
|
|
|
+ );
|
|
|
}
|
|
|
|
|
|
static List<DetectionResult> _decodeDetections(
|
|
|
@@ -277,86 +422,12 @@ class TfliteService {
|
|
|
return detections;
|
|
|
}
|
|
|
|
|
|
- Future<List<DetectionResult>> _runInferenceInIsolate(Uint8List imageBytes) async {
|
|
|
- // We need the model and labels passed as data
|
|
|
- final modelData = await rootBundle.load('assets/$_modelAsset');
|
|
|
- final labelData = await rootBundle.loadString('assets/$_labelsAsset');
|
|
|
-
|
|
|
- // Use compute to run in a real isolate
|
|
|
- return await compute(_inferenceTaskWrapper, {
|
|
|
- 'imageBytes': imageBytes,
|
|
|
- 'modelBytes': modelData.buffer.asUint8List(),
|
|
|
- 'labelData': labelData,
|
|
|
- });
|
|
|
- }
|
|
|
-
|
|
|
- static List<DetectionResult> _inferenceTaskWrapper(Map<String, dynamic> args) {
|
|
|
- return _inferenceTask(
|
|
|
- args['imageBytes'] as Uint8List,
|
|
|
- args['modelBytes'] as Uint8List,
|
|
|
- args['labelData'] as String,
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
- /// The static task that runs in the background isolate
|
|
|
- static List<DetectionResult> _inferenceTask(Uint8List imageBytes, Uint8List modelBytes, String labelData) {
|
|
|
- // 1. Initialize Interpreter inside the isolate
|
|
|
- final interpreter = Interpreter.fromBuffer(modelBytes);
|
|
|
- final labels = labelData.split('\n').where((l) => l.trim().isNotEmpty).map((l) => l.trim()).toList();
|
|
|
-
|
|
|
- try {
|
|
|
- // 2. Preprocess image
|
|
|
- final decoded = img.decodeImage(imageBytes);
|
|
|
- if (decoded == null) throw Exception('Could not decode image');
|
|
|
-
|
|
|
- // Center-Square Crop
|
|
|
- final int width = decoded.width;
|
|
|
- final int height = decoded.height;
|
|
|
- final int size = width < height ? width : height;
|
|
|
- final int offsetX = (width - size) ~/ 2;
|
|
|
- final int offsetY = (height - size) ~/ 2;
|
|
|
-
|
|
|
- final cropped = img.copyCrop(decoded, x: offsetX, y: offsetY, width: size, height: size);
|
|
|
- final resized = img.copyResize(cropped, width: _inputSize, height: _inputSize, interpolation: img.Interpolation.linear);
|
|
|
-
|
|
|
- final inputTensor = List.generate(1, (_) =>
|
|
|
- List.generate(_inputSize, (y) =>
|
|
|
- List.generate(_inputSize, (x) {
|
|
|
- final pixel = resized.getPixel(x, y);
|
|
|
- return [pixel.r / 255.0, pixel.g / 255.0, pixel.b / 255.0];
|
|
|
- })
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- // 3. Prepare output
|
|
|
- final outputShape = interpreter.getOutputTensors()[0].shape;
|
|
|
- final outputTensor = List.generate(1, (_) =>
|
|
|
- List.generate(outputShape[1], (_) =>
|
|
|
- List<double>.filled(outputShape[2], 0.0)
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- // 4. Run
|
|
|
- interpreter.run(inputTensor, outputTensor);
|
|
|
-
|
|
|
- // Map detections back to full frame
|
|
|
- return _decodeDetections(
|
|
|
- outputTensor[0],
|
|
|
- labels,
|
|
|
- cropSize: size,
|
|
|
- offsetX: offsetX,
|
|
|
- offsetY: offsetY,
|
|
|
- fullWidth: width,
|
|
|
- fullHeight: height
|
|
|
- );
|
|
|
- } finally {
|
|
|
- interpreter.close();
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
void dispose() {
|
|
|
- _interpreter?.close();
|
|
|
- _interpreter = null;
|
|
|
+ _receivePort?.close();
|
|
|
+ if (_isolate != null) {
|
|
|
+ _isolate!.kill(priority: Isolate.immediate);
|
|
|
+ _isolate = null;
|
|
|
+ }
|
|
|
_isInitialized = false;
|
|
|
}
|
|
|
}
|
Dr-Swopt