Image Processing with GPUImage/OpenCV

Edit me

Image Processing

iOS GPUImage Filters

• GPUImage Versions

  • As of late 2023, three GPUImage versions available.

  Version/Github	Description
  GPUImage	iOS framework for GPU-based image and video processing. Objective-C
  GPUImage2	Swift framework for GPU-accelerated video and image processing.
  GPUImage3	Swift framework for GPU-accelerated video and image processing using Metal.

  • As I mentioned in overview, GPUImage3 has different architecture(Metal Framework) from previous versions and some filters are not implemented. I chose GPUImage2, because I couldn’t make custom filters with GPUImage3.

• Initialize GPUImage and setup filters

override func viewDidLoad() {
    super.viewDidLoad()
    isFlashOn = false
    
    do {
        barcodeFilter = BarcodeBuffer()
        barcodeFilter.dataQRDetectCallback = qrDetectCallback
        barcodeFilter.dataDMDetectCallback = dmDetectCallback
        
        cropSmall = Crop()
    
        brightnessAdjust = BrightnessAdjustment()
        thresholdSketcher = ThresholdSketchFilter()
        brightnessAdjust.brightness = 0.1
        thresholdSketcher.threshold = 0.07

        camera = try Camera(sessionPreset: .hd1920x1080)
        
        camera --> renderView
        camera --> brightnessAdjust --> thresholdSketcher --> cropSmall --> barcodeFilter --> renderViewSmall
        camera.startCapture()
        
        if (self.is15ProOrMax) {
            desiredZoomFactor = 2.0
        } else {
            desiredZoomFactor = 1.3
        }

        try camera.inputCamera.lockForConfiguration()

        let maxFactor:CGFloat = min(10, camera.inputCamera.activeFormat.videoMaxZoomFactor)
        let videoZoomFactor = min(10, min(desiredZoomFactor, maxFactor))
        camera.inputCamera.videoZoomFactor = videoZoomFactor
        camera.inputCamera.focusMode = .continuousAutoFocus
        camera.inputCamera.unlockForConfiguration()
        
    } catch {
        fatalError("Could not initialize rendering pipeline: \(error)")
    }
    //
    adjustBarcodeView()
    
    renderView.fillMode = .preserveAspectRatioAndFill
}

• Original Image

• GPUImage Filters Applied

Android OpenCV Image Processing

• Prepare Matrix image from camera frame, then call OpenCV Processor

private fun GetBarcodeFrame_OpenCV(
    lfrmCamPreview: Frame,
    lbcdCallback: BarcodeGraphicTrackerCallback?
): Frame? {
    val lbmpRgb = getBitmapFromRGBFrame(lfrmCamPreview)
        ?: return null 
    val ldPreviewW = Resources.getSystem().displayMetrics.widthPixels.toDouble()
    val ldPreviewH = Resources.getSystem().displayMetrics.heightPixels.toDouble()
    val ldBmpW = lbmpRgb.width.toDouble()
    val ldBmpH = lbmpRgb.height.toDouble()

    var liOffsetH = (ldBmpH - BarcodeCaptureActivity.ciResizeHeight.toDouble()).toInt()
    if (liOffsetH < 0) liOffsetH = 0
    val liImgHeight = (BarcodeCaptureActivity.ciResizeHeight.toDouble() / 2.0).toInt()
    val liPosY = (liImgHeight.toDouble() / 2.0).toInt() + liOffsetH
    val liPosX =
        (BarcodeCaptureActivity.ciResizeWidth.toDouble() / 2.0 - liImgHeight.toDouble() / 2.0).toInt()
    val matrix = Matrix()
    matrix.postRotate(90f)
    val lbmpPortrait = Bitmap.createBitmap(
        lbmpRgb,
        liPosX,
        liPosY,
        liImgHeight,
        liImgHeight,
        matrix,
        true
    )
    val lbmpResized = Bitmap.createBitmap(
        lbmpPortrait,
        0,
        0,
        liImgHeight,
        liImgHeight
    )
    val lmtRgbaResized = Mat(lbmpResized.height, lbmpResized.width, CvType.CV_8SC4)
    Utils.bitmapToMat(lbmpResized, lmtRgbaResized)
    var lbmpQRImage: Bitmap? = null
    val lmtQRImage = Mat() 
    var lbmpPreview: Bitmap? = null
    var lmtPreview = Mat() 
    var lfrmBarcode: Frame? = null
    val liOptType = 1
    lmtPreview = openCVProcessor(
        liOptType,
        ciEdgeValue,
        0,
        0,
        0,
        0,
        lmtRgbaResized,
        lmtPreview,
        lmtQRImage
    )
    // Preview Mat to Bitmap
    lbmpPreview = Bitmap.createBitmap(
        lmtPreview.width(),
        lmtPreview.height(),
        Bitmap.Config.ARGB_8888
    )
    Utils.matToBitmap(lmtPreview, lbmpPreview)
    lbcdCallback!!.UpdateOpenCVImage("Preview", lbmpPreview)
    lfrmBarcode = Frame.Builder().setBitmap(lbmpPreview!!).build()
    return lfrmBarcode

}

• Process image with OpenCV

public static Mat openCVProcessor(
        int liOptType,
        int liOptArg1,
        int liOptArg2,
        int liOptArg3,
        int liOptArg4,
        int liOptArg5,
        Mat lmtRgba,
        Mat lmtPreview,
        Mat lmtQRImage
) {
    Mat lmtGray = new Mat();
    Mat lmtGrayBeforeHistEqu = new Mat();

    if (liOptType == 1) {
        Imgproc.cvtColor(lmtRgba, lmtGray, Imgproc.COLOR_RGB2GRAY);
        Mat qr_AfterBlur = new Mat();
        Imgproc.GaussianBlur(lmtGray, qr_AfterBlur, new Size(1, 1), 0, 0);
        Mat qr_AfterCanny = new Mat();
        Imgproc.Canny(qr_AfterBlur, qr_AfterCanny, liOptArg1, liOptArg1 * 2);
        Mat qr_AfterInv = new Mat();

        Core.bitwise_not(qr_AfterCanny, qr_AfterInv);

        Mat mGrAfterErode = new Mat();
        int erodeIterations = 1;
        Mat element = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(3, 3));
        Imgproc.erode(qr_AfterInv, mGrAfterErode, element, new Point(-1, -1), erodeIterations);
        Imgproc.resize(mGrAfterErode, lmtQRImage, new Size(mGrAfterErode.cols(), mGrAfterErode.rows()));
        Imgproc.cvtColor(mGrAfterErode, lmtPreview, Imgproc.COLOR_GRAY2RGBA);


        int liPvW = lmtPreview.cols();
        int liPvH = lmtPreview.rows();
        //Rect lrtPreview = new Rect(0, 0, liPvW, liPvH);
        Imgproc.rectangle(lmtPreview, new Point(0, 0), new Point(liPvW, liPvH), new Scalar(255, 255, 255, 255), 15, 8, 0);
        int liLen = liPvW / 5;
        Scalar lcrLen = new Scalar(0, 0, 0, 255);
        int liThk = 10;
        // Top - Left
        Imgproc.line(lmtPreview, new Point(0, liLen), new Point(0, 0), lcrLen, liThk);
        Imgproc.line(lmtPreview, new Point(0, 0), new Point(liLen, 0), lcrLen, liThk);
        // Top - Right
        Imgproc.line(lmtPreview, new Point(liPvW - liLen, 0), new Point(liPvW, 0), lcrLen, liThk);
        Imgproc.line(lmtPreview, new Point(liPvW, 0), new Point(liPvW, liLen), lcrLen, liThk);
        // Bottom - Right
        Imgproc.line(lmtPreview, new Point(liPvW, liPvH - liLen), new Point(liPvW, liPvH), lcrLen, liThk);
        Imgproc.line(lmtPreview, new Point(liPvW, liPvH), new Point(liPvW - liLen, liPvH), lcrLen, liThk);
        // Bottom - Left
        Imgproc.line(lmtPreview, new Point(0, liPvH - liLen), new Point(0, liPvH), lcrLen, liThk);
        Imgproc.line(lmtPreview, new Point(0, liPvH), new Point(liLen, liPvH), lcrLen, liThk);
    }

    return lmtPreview;
}

Android GPUImage Filters

• Prepare bitmap image from camera frame, then process with GPUImage filters

var _GPUIMG: GPUImage? = null
var _GPUSKETCH: GPUImageThresholdSketchFilter? = null

private fun GetBarcodeFrame_GPUImage(
    lfrmCamPreview: Frame,
    lbcdCallback: BarcodeGraphicTrackerCallback?
): Frame? {
    val lbmpRgb = getBitmapFromRGBFrame(lfrmCamPreview)
        ?: return null
    val matrix = Matrix()
    matrix.postRotate(90f)
    val lbmpPortrait =
        Bitmap.createBitmap(lbmpRgb, 0, 0, lbmpRgb.width, lbmpRgb.height, matrix, true)

    if (_GPUIMG == null) {
        _GPUIMG = GPUImage(mContext)
        _GPUSKETCH = GPUImageThresholdSketchFilter()
        cfThresholdPrev = cfThreshold
        _GPUSKETCH!!.setThreshold(cfThreshold)
        _GPUIMG!!.setFilter(_GPUSKETCH)
    }
    if (cfThreshold != cfThresholdPrev) {
        cfThresholdPrev = cfThreshold
        _GPUSKETCH!!.setThreshold(cfThreshold)
    }
    _GPUIMG!!.setImage(lbmpPortrait)
    val destBmp = _GPUIMG!!.bitmapWithFilterApplied
    lbcdCallback.UpdateOpenCVImage("Preview", destBmp)

    lfrmBarcode = Frame.Builder().setBitmap(destBmp).build()
    return lfrmBarcode
}