feat:UvcCamera从mjpeg改为yuv;增加推理时的日志;修改索引库默认阈值为0.77;

app/build.gradle

@@ -3,8 +3,8 @@ plugins {
     id 'org.jetbrains.kotlin.android'
 }
 
-def APP_VERSION_CODE = 1000201
-def APP_VERSION_NAME = "1.0.2.1"
+def APP_VERSION_CODE = 1000202
+def APP_VERSION_NAME = "1.0.2.2"
 
 android {
     namespace 'com.wmdigit.cateringdetect'

camera/build.gradle

@@ -36,6 +36,7 @@ dependencies {
 
     implementation project(path: ":common")
     implementation project(path: ":data-local")
+    implementation project(path: ":opencv")
 
     // CameraX core library using the camera2 implementation
     def camerax_version = "1.3.0-beta01"

camera/src/main/java/com/jiangdg/uvc/USBCameraHelper.java

@@ -4,7 +4,10 @@ package com.jiangdg.uvc;
 import android.content.Context;
 import android.graphics.Bitmap;
 import android.graphics.BitmapFactory;
+import android.graphics.ImageFormat;
+import android.graphics.Rect;
 import android.graphics.SurfaceTexture;
+import android.graphics.YuvImage;
 import android.hardware.usb.UsbDevice;
 import android.opengl.GLES11Ext;
 import android.os.Handler;
@@ -20,6 +23,7 @@ import com.wmdigit.common.utils.NV21ToBitmap;
 import com.wmdigit.common.utils.YuvToRgbConverter;
 import com.wmdigit.data.mmkv.repository.CameraLocalRepository;
 
+import java.io.ByteArrayOutputStream;
 import java.nio.ByteBuffer;
 
 /**
@@ -62,16 +66,6 @@ public class USBCameraHelper  {
         @Override
         public void onFrame(ByteBuffer buffer) {
             synchronized (mSyncFrame) {
-                /*try {
-                    if (isFirstFrame) {
-                        isFirstFrame = false;
-                        if (onCameraOpenListener != null) {
-                            onCameraOpenListener.onSuccess();
-                        }
-                    }
-                } catch (Exception e) {
-                    onCameraOpenListener = null;
-                }*/
                 count ++;
                 int len = buffer.capacity();
                 frame = new byte[len];
@@ -79,14 +73,34 @@ public class USBCameraHelper  {
                 if (count % frameInterval == 0){
                     count = 0;
                     if (onImageAnalyzeListener != null){
+                        onImageAnalyzeListener.onAnalyzed(encodeYuvToJpeg(frame, 640, 480, ImageFormat.NV21));
 //                        onImageAnalyzeListener.onAnalyzed(nv21ToBitmap.nv21ToBitmap(frame, 640, 480));
-                        onImageAnalyzeListener.onAnalyzed(BitmapFactory.decodeByteArray(frame, 0, frame.length));
+//                        onImageAnalyzeListener.onAnalyzed(BitmapFactory.decodeByteArray(frame, 0, frame.length));
                     }
                 }
             }
         }
     };
 
+    private Bitmap encodeYuvToJpeg(byte[] yuvData, int width, int height, int format){
+        try {
+            // 将YUV数据转换为Bitmap
+            YuvImage yuvImage = new YuvImage(yuvData, format, width, height, null);
+            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            yuvImage.compressToJpeg(new Rect(0, 0, width, height), 75, out);
+            byte[] imageBytes = out.toByteArray();
+            return BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.length);
+//            // 使用OpenCV进一步压缩
+//            Mat mat = Imgcodecs.imdecode(new MatOfByte(imageBytes), Imgcodecs.IMREAD_UNCHANGED);
+//            MatOfByte mob = new MatOfByte();
+//            Imgcodecs.imencode(".jpg", mat, mob);
+//            return mob.toArray();
+        } catch (Exception e) {
+            XLog.e(e);
+            return null;
+        }
+    }
+
     public static USBCameraHelper getInstance(Context context) {
         if (instance == null) {
             synchronized (USBCameraHelper.class) {
@@ -98,10 +112,6 @@ public class USBCameraHelper  {
         return instance;
     }
 
-/*    public void setOnCameraOpenListener(OnCameraOpenListener onListener) {
-        onCameraOpenListener = onListener;
-    }*/
-
     public USBCameraHelper(Context context) {
         mContext = context;
         if (mWorkerHandler == null) {
@@ -148,17 +158,17 @@ public class USBCameraHelper  {
                     final UVCCamera camera = new UVCCamera();
                     camera.open(ctrlBlock);
                     try {
-                        camera.setPreviewSize(640, 480, UVCCamera.FRAME_FORMAT_MJPEG);
+                        camera.setPreviewSize(640, 480, UVCCamera.PIXEL_FORMAT_YUV420SP);
                     } catch (final IllegalArgumentException e) {
                         // fallback to YUV mode
                         try {
-                            camera.setPreviewSize(640, 480, UVCCamera.FRAME_FORMAT_MJPEG);
+                            camera.setPreviewSize(640, 480,UVCCamera.PIXEL_FORMAT_YUV420SP);
                         } catch (final IllegalArgumentException e1) {
                             camera.destroy();
                             return;
                         }
                     }
-                    camera.setFrameCallback(iFrameCallback, UVCCamera.FRAME_FORMAT_MJPEG);
+                    camera.setFrameCallback(iFrameCallback, UVCCamera.PIXEL_FORMAT_YUV420SP);
                     if (mSurfaceTexture == null) {
                         mSurfaceTexture = new SurfaceTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES);
                     }

common/src/main/java/com/wmdigit/common/utils/StopWatch.java

+package com.wmdigit.common.utils;
+
+import java.text.NumberFormat;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Simple stop watch, allowing for timing of a number of tasks,
+ * exposing total running time and running time for each named task.
+ *
+ * <p>Conceals use of {@code System.currentTimeMillis()}, improving the
+ * readability of application code and reducing the likelihood of calculation errors.
+ *
+ * <p>Note that this object is not designed to be thread-safe and does not
+ * use synchronization.
+ *
+ * <p>This class is normally used to verify performance during proof-of-concepts
+ * and in development, rather than as part of production applications.
+ *
+ * @author Rod Johnson
+ * @author Juergen Hoeller
+ * @author Sam Brannen
+ * @since May 2, 2001
+ */
+public class StopWatch {
+
+    /**
+     * Identifier of this stop watch.
+     * Handy when we have output from multiple stop watches
+     * and need to distinguish between them in log or console output.
+     */
+    private final String id;
+
+    private boolean keepTaskList = true;
+
+    private final List<TaskInfo> taskList = new LinkedList<TaskInfo>();
+
+    /**
+     * Start time of the current task
+     */
+    private long startTimeMillis;
+
+    /**
+     * Is the stop watch currently running?
+     */
+    private boolean running;
+
+    /**
+     * Name of the current task
+     */
+    private String currentTaskName;
+
+    private TaskInfo lastTaskInfo;
+
+    private int taskCount;
+
+    /**
+     * Total running time
+     */
+    private long totalTimeMillis;
+
+
+    /**
+     * Construct a new stop watch. Does not start any task.
+     */
+    public StopWatch() {
+        this.id = "";
+    }
+
+    /**
+     * Construct a new stop watch with the given id.
+     * Does not start any task.
+     *
+     * @param id identifier for this stop watch.
+     *           Handy when we have output from multiple stop watches
+     *           and need to distinguish between them.
+     */
+    public StopWatch(String id) {
+        this.id = id;
+    }
+
+
+    /**
+     * Determine whether the TaskInfo array is built over time. Set this to
+     * "false" when using a StopWatch for millions of intervals, or the task
+     * info structure will consume excessive memory. Default is "true".
+     */
+    public void setKeepTaskList(boolean keepTaskList) {
+        this.keepTaskList = keepTaskList;
+    }
+
+
+    /**
+     * Start an unnamed task. The results are undefined if {@link #stop()}
+     * or timing methods are called without invoking this method.
+     *
+     * @see #stop()
+     */
+    public void start() throws IllegalStateException {
+        start("");
+    }
+
+    /**
+     * Start a named task. The results are undefined if {@link #stop()}
+     * or timing methods are called without invoking this method.
+     *
+     * @param taskName the name of the task to start
+     * @see #stop()
+     */
+    public void start(String taskName) throws IllegalStateException {
+        if (this.running) {
+            throw new IllegalStateException("Can't start StopWatch: it's already running");
+        }
+        this.startTimeMillis = System.currentTimeMillis();
+        this.running = true;
+        this.currentTaskName = taskName;
+    }
+
+    /**
+     * Stop the current task. The results are undefined if timing
+     * methods are called without invoking at least one pair
+     * {@link #start()} / {@link #stop()} methods.
+     *
+     * @see #start()
+     */
+    public void stop() throws IllegalStateException {
+        if (!this.running) {
+            throw new IllegalStateException("Can't stop StopWatch: it's not running");
+        }
+        long lastTime = System.currentTimeMillis() - this.startTimeMillis;
+        this.totalTimeMillis += lastTime;
+        this.lastTaskInfo = new TaskInfo(this.currentTaskName, lastTime);
+        if (this.keepTaskList) {
+            this.taskList.add(lastTaskInfo);
+        }
+        ++this.taskCount;
+        this.running = false;
+        this.currentTaskName = null;
+    }
+
+    /**
+     * Return whether the stop watch is currently running.
+     */
+    public boolean isRunning() {
+        return this.running;
+    }
+
+
+    /**
+     * Return the time taken by the last task.
+     */
+    public long getLastTaskTimeMillis() throws IllegalStateException {
+        if (this.lastTaskInfo == null) {
+            throw new IllegalStateException("No tasks run: can't get last task interval");
+        }
+        return this.lastTaskInfo.getTimeMillis();
+    }
+
+    /**
+     * Return the name of the last task.
+     */
+    public String getLastTaskName() throws IllegalStateException {
+        if (this.lastTaskInfo == null) {
+            throw new IllegalStateException("No tasks run: can't get last task name");
+        }
+        return this.lastTaskInfo.getTaskName();
+    }
+
+    /**
+     * Return the last task as a TaskInfo object.
+     */
+    public TaskInfo getLastTaskInfo() throws IllegalStateException {
+        if (this.lastTaskInfo == null) {
+            throw new IllegalStateException("No tasks run: can't get last task info");
+        }
+        return this.lastTaskInfo;
+    }
+
+
+    /**
+     * Return the total time in milliseconds for all tasks.
+     */
+    public long getTotalTimeMillis() {
+        return this.totalTimeMillis;
+    }
+
+    /**
+     * Return the total time in seconds for all tasks.
+     */
+    public double getTotalTimeSeconds() {
+        return this.totalTimeMillis / 1000.0;
+    }
+
+    /**
+     * Return the number of tasks timed.
+     */
+    public int getTaskCount() {
+        return this.taskCount;
+    }
+
+    /**
+     * Return an array of the data for tasks performed.
+     */
+    public TaskInfo[] getTaskInfo() {
+        if (!this.keepTaskList) {
+            throw new UnsupportedOperationException("Task info is not being kept!");
+        }
+        return this.taskList.toArray(new TaskInfo[this.taskList.size()]);
+    }
+
+
+    /**
+     * Return a short description of the total running time.
+     */
+    public String shortSummary() {
+        return "StopWatch '" + this.id + "': running time (millis) = " + getTotalTimeMillis();
+    }
+
+    /**
+     * Return a string with a table describing all tasks performed.
+     * For custom reporting, call getTaskInfo() and use the task info directly.
+     */
+    public String prettyPrint() {
+        StringBuilder sb = new StringBuilder(shortSummary());
+        sb.append('\n');
+        if (!this.keepTaskList) {
+            sb.append("No task info kept");
+        } else {
+            sb.append("-----------------------------------------\n");
+            sb.append("ms     %     Task name\n");
+            sb.append("-----------------------------------------\n");
+            NumberFormat nf = NumberFormat.getNumberInstance();
+            nf.setMinimumIntegerDigits(5);
+            nf.setGroupingUsed(false);
+            NumberFormat pf = NumberFormat.getPercentInstance();
+            pf.setMinimumIntegerDigits(3);
+            pf.setGroupingUsed(false);
+            for (TaskInfo task : getTaskInfo()) {
+                sb.append(nf.format(task.getTimeMillis())).append("  ");
+                sb.append(pf.format(task.getTimeSeconds() / getTotalTimeSeconds())).append("  ");
+                sb.append(task.getTaskName()).append("\n");
+            }
+        }
+        return sb.toString();
+    }
+
+    /**
+     * Return an informative string describing all tasks performed
+     * For custom reporting, call {@code getTaskInfo()} and use the task info directly.
+     */
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder(shortSummary());
+        if (this.keepTaskList) {
+            for (TaskInfo task : getTaskInfo()) {
+                sb.append("; [").append(task.getTaskName()).append("] took ").append(task.getTimeMillis());
+                long percent = Math.round((100.0 * task.getTimeSeconds()) / getTotalTimeSeconds());
+                sb.append(" = ").append(percent).append("%");
+            }
+        } else {
+            sb.append("; no task info kept");
+        }
+        return sb.toString();
+    }
+
+
+    /**
+     * Inner class to hold data about one task executed within the stop watch.
+     */
+    public static final class TaskInfo {
+
+        private final String taskName;
+
+        private final long timeMillis;
+
+        TaskInfo(String taskName, long timeMillis) {
+            this.taskName = taskName;
+            this.timeMillis = timeMillis;
+        }
+
+        /**
+         * Return the name of this task.
+         */
+        public String getTaskName() {
+            return this.taskName;
+        }
+
+        /**
+         * Return the time in milliseconds this task took.
+         */
+        public long getTimeMillis() {
+            return this.timeMillis;
+        }
+
+        /**
+         * Return the time in seconds this task took.
+         */
+        public double getTimeSeconds() {
+            return this.timeMillis / 1000.0;
+        }
+    }
+
+}
\ No newline at end of file

core/src/main/java/com/wmdigit/core/catering/TargetDetectionRepository.java

@@ -9,9 +9,12 @@ import com.wmdigit.common.model.CropValueDTO;
 import com.wmdigit.core.catering.dish.DishDetection;
 import com.wmdigit.core.catering.model.TargetDetectResult;
 import com.wmdigit.core.catering.plate.PlateDetection;
+import com.wmdigit.data.database.entity.ProductsPO;
+import com.wmdigit.data.disk.repository.DiskRepository;
 import com.wmdigit.data.mmkv.repository.AiLocalRepository;
 import com.wmdigit.data.mmkv.repository.CropLocalRepository;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import io.reactivex.Observable;

core/src/main/java/com/wmdigit/core/catering/dish/DishDetection.java

@@ -59,12 +59,10 @@ public class DishDetection implements TargetDetection {
     @Override
     public TargetDetectResult processImage(Bitmap bitmap) {
         synchronized (syncLock) {
-            long startTime = System.currentTimeMillis();
             TargetDetectResult result = process(bitmap);
             if (result == null){
                 return null;
             }
-            XLog.i("推理耗时：" + (System.currentTimeMillis() - startTime) + "ms");
             return result;
         }
     }

core/src/main/java/com/wmdigit/core/catering/plate/PlateDetection.java

@@ -63,11 +63,8 @@ public class PlateDetection implements TargetDetection {
     @Override
     public TargetDetectResult processImage(Bitmap bitmap) {
         synchronized (syncLock) {
-            long startTime = System.currentTimeMillis();
-            TargetDetectResult result = process(bitmap);
 
-            XLog.i("推理耗时：" + (System.currentTimeMillis() - startTime) + "ms" );
-            return result;
+            return process(bitmap);
         }
     }
 

core/src/main/java/com/wmdigit/core/hnsw/HnswRepository.java

@@ -115,7 +115,7 @@ public class HnswRepository {
             XLog.i("索引库未完成初始化");
             return null;
         }
-        String productCode = hnsw.retrieve(feature, 0.8f);
+        String productCode = hnsw.retrieve(feature, 0.77f);
         if (TextUtils.isEmpty(productCode)){
             return null;
         }

core/src/main/java/com/wmdigit/core/opencv/OpencvRepository.java

 package com.wmdigit.core.opencv;
 
 import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
 import android.graphics.Canvas;
 import android.graphics.Color;
 import android.graphics.Paint;
+import android.graphics.Rect;
+import android.graphics.YuvImage;
 
 import com.elvishew.xlog.XLog;
 import com.wmdigit.common.font.TypefaceHelper;
@@ -16,10 +19,13 @@ import com.wmdigit.data.database.entity.ProductsPO;
 import org.opencv.android.OpenCVLoader;
 import org.opencv.android.Utils;
 import org.opencv.core.Mat;
+import org.opencv.core.MatOfByte;
 import org.opencv.core.Point;
 import org.opencv.core.Scalar;
+import org.opencv.imgcodecs.Imgcodecs;
 import org.opencv.imgproc.Imgproc;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.List;
 
@@ -139,4 +145,24 @@ public class OpencvRepository {
         }
         return bitmapCopy;
     }
+
+    public Bitmap encodeYuvToJpeg(byte[] yuvData, int width, int height, int format){
+        try {
+            // 将YUV数据转换为Bitmap
+            YuvImage yuvImage = new YuvImage(yuvData, format, width, height, null);
+            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            yuvImage.compressToJpeg(new Rect(0, 0, width, height), 90, out);
+            byte[] imageBytes = out.toByteArray();
+            return BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.length);
+//            // 使用OpenCV进一步压缩
+//            Mat mat = Imgcodecs.imdecode(new MatOfByte(imageBytes), Imgcodecs.IMREAD_UNCHANGED);
+//            MatOfByte mob = new MatOfByte();
+//            Imgcodecs.imencode(".jpg", mat, mob);
+//
+//            return mob.toArray();
+        } catch (Exception e) {
+            XLog.e(e);
+            return null;
+        }
+    }
 }

history.txt

@@ -21,6 +21,10 @@ v1.0.2.1 2025/04/22 1.特征表增加字段记录图片地址，数据库版本
                     4.增加本地图片学习
                     5.增加LOGO
                     6.修复安装APK后出现两个图标的问题
+v1.0.2.2 2025/04/25 1.UvcCamera从mjpeg改为yuv
+                    2.增加推理时的日志
+                    3.修改索引库默认阈值为0.77
+
 
                     todo 增加学习记录管理模块
                     todo 替换LOGO

module-setting/src/main/java/com/wmdigit/setting/viewmodel/DataLearningViewModel.java

@@ -17,6 +17,7 @@ import com.wmdigit.common.model.CropValueDTO;
 import com.wmdigit.common.model.IdentifyRecordDTO;
 import com.wmdigit.common.model.ProductsVO;
 import com.wmdigit.common.utils.ParcelHelper;
+import com.wmdigit.common.utils.StopWatch;
 import com.wmdigit.core.catering.TargetDetectionRepository;
 import com.wmdigit.core.catering.model.TargetDetectResult;
 import com.wmdigit.core.hnsw.HnswRepository;
@@ -255,11 +256,17 @@ public class DataLearningViewModel extends BaseViewModel {
                 // 从当前内存中拷贝出图片
                 bitmap = ParcelHelper.copy(frame.getValue());
             }
+            StopWatch stopWatch = new StopWatch();
+            stopWatch.start("图像推理");
             // 图片推理
             detectResult = TargetDetectionRepository.getInstance().processImage(bitmap, true);
             if (detectResult == null || detectResult.getRectArray() == null) {
+                stopWatch.stop();
+                XLog.i(stopWatch.prettyPrint());
                 return;
             }
+            stopWatch.stop();
+            stopWatch.start("数据检索");
             // 根据推理结果去索引库查询
             List<ProductsPO> result = new ArrayList<>();
             for (int i = 0; i < detectResult.getFeatures().length; i++) {
@@ -268,8 +275,12 @@ public class DataLearningViewModel extends BaseViewModel {
                 result.add(product);
             }
             detectResult.setProducts(result);
+            stopWatch.stop();
+            stopWatch.start("图像绘制");
             // 在图片上绘制框和商品名
             drawOnBitmap(detectResult);
+            stopWatch.stop();
+            XLog.i(stopWatch.prettyPrint());
         }
     }
 

module-setting/src/main/java/com/wmdigit/setting/viewmodel/DataManagerViewModel.java

@@ -103,6 +103,7 @@ public class DataManagerViewModel extends BaseViewModel {
                             // 学习失败，隐藏加载中信息并提示用户失败
                             loadingProgressText.postValue("");
                             toastMessage.postValue(getApplication().getString(R.string.learn_local_image_fail));
+                            XLog.e(error);
                         })
         );
     }

service/src/main/java/com/wmdigit/service/aidl/CateringInterfaceImpl.java

@@ -19,6 +19,7 @@ import com.wmdigit.common.model.ProductsDTO;
 import com.wmdigit.common.utils.BitmapUtils;
 import com.wmdigit.common.utils.DateUtils;
 import com.wmdigit.common.utils.ParcelHelper;
+import com.wmdigit.common.utils.StopWatch;
 import com.wmdigit.core.catering.TargetDetectionRepository;
 import com.wmdigit.core.catering.model.TargetDetectResult;
 import com.wmdigit.core.hnsw.HnswRepository;
@@ -377,14 +378,19 @@ public class CateringInterfaceImpl extends ICateringInterface.Stub{
                 isUpload = true;
                 bitmap = ParcelHelper.copy(bitmapCopy);
             }
+            StopWatch stopWatch = new StopWatch();
+            stopWatch.start("图像推理");
             // 对图像进行处理，尝试检测目标
             targetDetectResult = TargetDetectionRepository.getInstance().processImage(bitmap, true);
             // 如果没有检测到任何矩形对象，直接返回
             if (targetDetectResult.getRectArray() == null) {
+                stopWatch.stop();
+                XLog.i(stopWatch.prettyPrint());
                 return detectResult;
             }
+            stopWatch.stop();
+            stopWatch.start("数据检索");
             List<ProductsPO> productsPOList = new ArrayList<>();
-
             // 根据检测到的特征检索产品，并将结果添加到列表中
             for (int i = 0; i < targetDetectResult.getFeatures().length; i++) {
                 ProductsPO product = HnswRepository.getInstance().retrieveByFeature(targetDetectResult.getFeatures()[i]);
@@ -434,11 +440,15 @@ public class CateringInterfaceImpl extends ICateringInterface.Stub{
                     XLog.e("图片记录上传失败");
                 });
             }
+            stopWatch.stop();
             // 如果需要生成Bitmap，进行相关处理
             if (generateBitmap) {
+                stopWatch.start("绘制结果");
                 // 在Bitmap上绘制检测结果
                 detectResult.setBitmap(OpencvRepository.getInstance().drawDetectResultOnBitmap(targetDetectResult));
+                stopWatch.stop();
             }
+            XLog.i(stopWatch.prettyPrint());
             return detectResult;
         }
     }