calcOpticalFlowPyrLK with UMat cornersB are on top left position
#1,632 opened on Apr 23, 2021
Repository metrics
- Stars
- (6,985 stars)
- PR merge metrics
- (No merged PRs in 30d)
Description
Hi, I try to implement video stabilisation with OpenCL acceleration. I build sample project with samples/OpticalFlowTracker.java.
When I use Mat or setUseOpenCL(false); the result is:

When I use UMat the result image is:
Corners with UMat are:
Corners with Mat are:

As I see only first Corner is on right position. Rest of them are moved to top left position.
Problem is in calcOpticalFlowPyrLKwhen I disable OpenCl only for this method everything work fine.
Code with UMat:
import org.bytedeco.javacpp.indexer.*;
import org.bytedeco.opencv.opencv_core.*;
import org.bytedeco.opencv.opencv_highgui.*;
import org.bytedeco.opencv.opencv_imgproc.*;
import org.bytedeco.opencv.opencv_video.*;
import static org.bytedeco.opencv.global.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_highgui.*;
import static org.bytedeco.opencv.global.opencv_imgcodecs.*;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
import static org.bytedeco.opencv.global.opencv_video.*;
public class OpticalFlowTrackerU {
private static final int MAX_CORNERS = 500;
private static final int win_size = 15;
public static void main(String[] args) {
// Load two images and allocate other structures
Mat imgA = imread(
"image0.png",
IMREAD_GRAYSCALE);
Mat imgB = imread(
"image1.png",
IMREAD_GRAYSCALE);
// Mat imgC = imread("OpticalFlow1.png",
// IMREAD_UNCHANGED);
Mat imgC = imread(
"image0.png",
IMREAD_UNCHANGED);
// Get the features for tracking
UMat cornersAU = new UMat();
UMat imgAU = imgA.getUMat(ACCESS_READ);
UMat imgBU = imgB.getUMat(ACCESS_READ);
//setUseOpenCL(false);
goodFeaturesToTrack(imgAU, cornersAU, MAX_CORNERS,
0.05, 5.0, null, 3, false, 0.04);
cornerSubPix(imgAU, cornersAU,
new Size(win_size, win_size), new Size(-1, -1),
new TermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.03));
// Call Lucas Kanade algorithm
UMat features_foundU = new UMat();
UMat feature_errorsU = new UMat();
UMat cornersBU = new UMat();
calcOpticalFlowPyrLK(imgAU, imgBU, cornersAU, cornersBU,
features_foundU, feature_errorsU, new Size(win_size, win_size), 5,
new TermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.3), 0, 1e-4);
// Make an image of the results
Mat cornersA = cornersAU.getMat(ACCESS_READ);
Mat cornersB = cornersBU.getMat(ACCESS_READ);
Mat features_found = features_foundU.getMat(ACCESS_READ);
Mat feature_errors = feature_errorsU.getMat(ACCESS_READ);
FloatIndexer cornersAidx = cornersA.createIndexer();
FloatIndexer cornersBidx = cornersB.createIndexer();
UByteIndexer features_found_idx = features_found.createIndexer();
FloatIndexer feature_errors_idx = feature_errors.createIndexer();
for (int i = 0; i < cornersAidx.size(0); i++) {
if (features_found_idx.get(i) == 0 || feature_errors_idx.get(i) > 550) {
System.out.println("Error is " + feature_errors_idx.get(i) + "/n");
continue;
}
System.out.println("Got it/n");
Point p0 = new Point(Math.round(cornersAidx.get(i,0, 0)),
Math.round(cornersAidx.get(i,0, 1)));
Point p1 = new Point(Math.round(cornersBidx.get(i,0, 0)),
Math.round(cornersBidx.get(i,0, 1)));
line(imgC, p0, p1, RGB(255, 0, 0),
2, 8, 0);
}
imwrite(
"image0-1.png",
imgC);
namedWindow("LKpyr_OpticalFlow", 0);
imshow("LKpyr_OpticalFlow", imgC);
waitKey(0);
}
}
Code with Mat:
import org.bytedeco.javacpp.indexer.*;
import org.bytedeco.opencv.opencv_core.*;
import org.bytedeco.opencv.opencv_highgui.*;
import org.bytedeco.opencv.opencv_imgproc.*;
import org.bytedeco.opencv.opencv_video.*;
import static org.bytedeco.opencv.global.opencv_core.*;
import static org.bytedeco.opencv.global.opencv_highgui.*;
import static org.bytedeco.opencv.global.opencv_imgcodecs.*;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
import static org.bytedeco.opencv.global.opencv_video.*;
public class OpticalFlowTracker {
private static final int MAX_CORNERS = 500;
private static final int win_size = 15;
public static void main(String[] args) {
// Load two images and allocate other structures
Mat imgA = imread(
"image0.png",
IMREAD_GRAYSCALE);
Mat imgB = imread(
"image1.png",
IMREAD_GRAYSCALE);
// Mat imgC = imread("OpticalFlow1.png",
// IMREAD_UNCHANGED);
Mat imgC = imread(
"image0.png",
IMREAD_UNCHANGED);
// Get the features for tracking
Mat cornersA = new Mat();
goodFeaturesToTrack(imgA, cornersA, MAX_CORNERS,
0.05, 5.0, null, 3, false, 0.04);
cornerSubPix(imgA, cornersA,
new Size(win_size, win_size), new Size(-1, -1),
new TermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.03));
// Call Lucas Kanade algorithm
Mat features_found = new Mat();
Mat feature_errors = new Mat();
Mat cornersB = new Mat();
calcOpticalFlowPyrLK(imgA, imgB, cornersA, cornersB,
features_found, feature_errors, new Size(win_size, win_size), 5,
new TermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.3), 0, 1e-4);
// Make an image of the results
FloatIndexer cornersAidx = cornersA.createIndexer();
FloatIndexer cornersBidx = cornersB.createIndexer();
UByteIndexer features_found_idx = features_found.createIndexer();
FloatIndexer feature_errors_idx = feature_errors.createIndexer();
for (int i = 0; i < cornersAidx.size(0); i++) {
if (features_found_idx.get(i) == 0 || feature_errors_idx.get(i) > 550) {
System.out.println("Error is " + feature_errors_idx.get(i) + "/n");
continue;
}
System.out.println("Got it/n");
Point p0 = new Point(Math.round(cornersAidx.get(i,0, 0)),
Math.round(cornersAidx.get(i,0, 1)));
Point p1 = new Point(Math.round(cornersBidx.get(i,0, 0)),
Math.round(cornersBidx.get(i,0, 1)));
line(imgC, p0, p1, RGB(255, 0, 0),
2, 8, 0);
}
imwrite(
"image0-1.png",
imgC);
namedWindow("LKpyr_OpticalFlow", 0);
imshow("LKpyr_OpticalFlow", imgC);
waitKey(0);
}
}
Pom file:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>org.example</groupId>
<artifactId>untitled</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>org.bytedeco</groupId>
<artifactId>javacv-platform</artifactId>
<version>1.5.5</version>
</dependency>
</dependencies>
</project>
There is no difference when I use:
<dependency>
<groupId>org.bytedeco</groupId>
<artifactId>opencv-platform</artifactId>
<version>4.5.1-1.5.5</version>
</dependency>
By the way:
Instead of all corner indexer get:
cornersAidx.get(i, 1)
i need to use:
cornersAidx.get(i, 0, 1)
Because I get out of bounds Exception.
Edit: I reformat inserted code.