bytedeco/javacv

calcOpticalFlowPyrLK with UMat cornersB are on top left position

Open

#1,632 opened on Apr 23, 2021

View on GitHub
 (0 comments) (0 reactions) (0 assignees)Java (1,583 forks)batch import
help wantedquestion

Repository metrics

Stars
 (6,985 stars)
PR merge metrics
 (No merged PRs in 30d)

Description

Hi, I try to implement video stabilisation with OpenCL acceleration. I build sample project with samples/OpticalFlowTracker.java. When I use Mat or setUseOpenCL(false); the result is: obraz

When I use UMat the result image is: obraz Corners with UMat are: obraz Corners with Mat are: obraz

As I see only first Corner is on right position. Rest of them are moved to top left position. Problem is in calcOpticalFlowPyrLKwhen I disable OpenCl only for this method everything work fine.

Code with UMat:

import org.bytedeco.javacpp.indexer.*;

import org.bytedeco.opencv.opencv_core.*;
import org.bytedeco.opencv.opencv_highgui.*;
import org.bytedeco.opencv.opencv_imgproc.*;
import org.bytedeco.opencv.opencv_video.*;
import static org.bytedeco.opencv.global.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_highgui.*;
import static org.bytedeco.opencv.global.opencv_imgcodecs.*;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
import static org.bytedeco.opencv.global.opencv_video.*;


public class OpticalFlowTrackerU {
    private static final int MAX_CORNERS = 500;
    private static final int win_size = 15;

    public static void main(String[] args) {
        // Load two images and allocate other structures
        Mat imgA = imread(
                "image0.png",
                IMREAD_GRAYSCALE);
        Mat imgB = imread(
                "image1.png",
                IMREAD_GRAYSCALE);

        // Mat imgC = imread("OpticalFlow1.png",
        // IMREAD_UNCHANGED);
        Mat imgC = imread(
                "image0.png",
                IMREAD_UNCHANGED);

        // Get the features for tracking
        UMat cornersAU = new UMat();
        UMat imgAU = imgA.getUMat(ACCESS_READ);
        UMat imgBU = imgB.getUMat(ACCESS_READ);
        //setUseOpenCL(false);
        goodFeaturesToTrack(imgAU, cornersAU, MAX_CORNERS,
                0.05, 5.0, null, 3, false, 0.04);

        cornerSubPix(imgAU, cornersAU,
                new Size(win_size, win_size), new Size(-1, -1),
                new TermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.03));

        // Call Lucas Kanade algorithm
        UMat features_foundU = new UMat();
        UMat feature_errorsU = new UMat();

        UMat cornersBU = new UMat();
        calcOpticalFlowPyrLK(imgAU, imgBU, cornersAU, cornersBU,
                features_foundU, feature_errorsU, new Size(win_size, win_size), 5,
                new TermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.3), 0, 1e-4);

        // Make an image of the results
        Mat cornersA = cornersAU.getMat(ACCESS_READ);
        Mat cornersB = cornersBU.getMat(ACCESS_READ);
        Mat features_found = features_foundU.getMat(ACCESS_READ);
        Mat feature_errors = feature_errorsU.getMat(ACCESS_READ);
        FloatIndexer cornersAidx = cornersA.createIndexer();
        FloatIndexer cornersBidx = cornersB.createIndexer();
        UByteIndexer features_found_idx = features_found.createIndexer();
        FloatIndexer feature_errors_idx = feature_errors.createIndexer();
        for (int i = 0; i < cornersAidx.size(0); i++) {
            if (features_found_idx.get(i) == 0 || feature_errors_idx.get(i) > 550) {
                System.out.println("Error is " + feature_errors_idx.get(i) + "/n");
                continue;
            }
            System.out.println("Got it/n");
            Point p0 = new Point(Math.round(cornersAidx.get(i,0, 0)),
                    Math.round(cornersAidx.get(i,0, 1)));
            Point p1 = new Point(Math.round(cornersBidx.get(i,0, 0)),
                    Math.round(cornersBidx.get(i,0, 1)));
            line(imgC, p0, p1, RGB(255, 0, 0),
                    2, 8, 0);
        }

        imwrite(
                "image0-1.png",
                imgC);
        namedWindow("LKpyr_OpticalFlow", 0);
        imshow("LKpyr_OpticalFlow", imgC);
        waitKey(0);
    }
}

Code with Mat:


import org.bytedeco.javacpp.indexer.*;

import org.bytedeco.opencv.opencv_core.*;
import org.bytedeco.opencv.opencv_highgui.*;
import org.bytedeco.opencv.opencv_imgproc.*;
import org.bytedeco.opencv.opencv_video.*;
import static org.bytedeco.opencv.global.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_highgui.*;
import static org.bytedeco.opencv.global.opencv_imgcodecs.*;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
import static org.bytedeco.opencv.global.opencv_video.*;


public class OpticalFlowTracker {
    private static final int MAX_CORNERS = 500;
    private static final int win_size = 15;

    public static void main(String[] args) {
        // Load two images and allocate other structures
        Mat imgA = imread(
                "image0.png",
                IMREAD_GRAYSCALE);
        Mat imgB = imread(
                "image1.png",
                IMREAD_GRAYSCALE);

        // Mat imgC = imread("OpticalFlow1.png",
        // IMREAD_UNCHANGED);
        Mat imgC = imread(
                "image0.png",
                IMREAD_UNCHANGED);

        // Get the features for tracking
        Mat cornersA = new Mat();
        goodFeaturesToTrack(imgA, cornersA, MAX_CORNERS,
                0.05, 5.0, null, 3, false, 0.04);

        cornerSubPix(imgA, cornersA,
                new Size(win_size, win_size), new Size(-1, -1),
                new TermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.03));

        // Call Lucas Kanade algorithm
        Mat features_found = new Mat();
        Mat feature_errors = new Mat();

        Mat cornersB = new Mat();
        calcOpticalFlowPyrLK(imgA, imgB, cornersA, cornersB,
                features_found, feature_errors, new Size(win_size, win_size), 5,
                new TermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.3), 0, 1e-4);

        // Make an image of the results
        FloatIndexer cornersAidx = cornersA.createIndexer();
        FloatIndexer cornersBidx = cornersB.createIndexer();
        UByteIndexer features_found_idx = features_found.createIndexer();
        FloatIndexer feature_errors_idx = feature_errors.createIndexer();
        for (int i = 0; i < cornersAidx.size(0); i++) {
            if (features_found_idx.get(i) == 0 || feature_errors_idx.get(i) > 550) {
                System.out.println("Error is " + feature_errors_idx.get(i) + "/n");
                continue;
            }
            System.out.println("Got it/n");
            Point p0 = new Point(Math.round(cornersAidx.get(i,0, 0)),
                    Math.round(cornersAidx.get(i,0, 1)));
            Point p1 = new Point(Math.round(cornersBidx.get(i,0, 0)),
                    Math.round(cornersBidx.get(i,0, 1)));
            line(imgC, p0, p1, RGB(255, 0, 0),
                    2, 8, 0);
        }

        imwrite(
                "image0-1.png",
                imgC);
        namedWindow("LKpyr_OpticalFlow", 0);
        imshow("LKpyr_OpticalFlow", imgC);
        waitKey(0);
    }
}

Pom file:

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">


    <groupId>org.example</groupId>
    <artifactId>untitled</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>11</maven.compiler.source>
        <maven.compiler.target>11</maven.compiler.target>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.bytedeco</groupId>
            <artifactId>javacv-platform</artifactId>
            <version>1.5.5</version>
        </dependency>
    </dependencies>
</project>

There is no difference when I use:

        <dependency>
            <groupId>org.bytedeco</groupId>
            <artifactId>opencv-platform</artifactId>
            <version>4.5.1-1.5.5</version>
        </dependency>

By the way: Instead of all corner indexer get: cornersAidx.get(i, 1) i need to use: cornersAidx.get(i, 0, 1) Because I get out of bounds Exception.

Edit: I reformat inserted code.

Contributor guide