Repository metrics
- Stars
- (6,985 stars)
- PR merge metrics
- (No merged PRs in 30d)
Description
I've been testing converting Frame instances to BufferedImage instances and noticed it's very slow, because, no matter what, every pixel channel is copied manually from Frame's buffer to BufferedImage (even when pixel layout and values don't have to be altered at all).
I created a small example with a streamlined approach, that can be used instead, if there's no gamma conversion required and Frame's pixel format matches (won't work for every pixel format, obviously, but works for BGR/RGB formats). You can comment/uncomment parts where frame <-> image conversion is done and test it on your machine.
My results are:
- with
Java2DFrameConverter:processing 388 frames took 21.671 (17.904 fps) - with simplified conversion:
processing 388 frames took 5.532 (70.137 fps)
public class FrameGrabberTest {
public static final int PIXEL_FORMAT = AV_PIX_FMT_BGR24;
public static final String IN_PATH = "in.mp4";
public static final String OUT_PATH = "out.mp4";
private static Java2DFrameConverter converter = new Java2DFrameConverter();
public static void main(String[] args) throws FrameGrabber.Exception, FrameRecorder.Exception {
FFmpegFrameGrabber grabber = new FFmpegFrameGrabber(IN_PATH);
grabber.setOption("hwaccel", "videotoolbox"); // does nothing in the current ffmpeg version
grabber.setVideoOption("threads", "auto");
grabber.setPixelFormat(PIXEL_FORMAT);
grabber.start();
FFmpegFrameRecorder recorder = new FFmpegFrameRecorder(OUT_PATH, grabber.getImageWidth(), grabber.getImageHeight());
recorder.setFormat(grabber.getFormat());
recorder.setFrameRate(grabber.getFrameRate());
recorder.setVideoBitrate(grabber.getVideoBitrate());
recorder.setVideoCodecName("h264_videotoolbox");
recorder.setVideoOption("threads", "auto");
recorder.start();
int frameCount = 0;
long startTime = System.currentTimeMillis();
Frame inFrame;
while ((inFrame = grabber.grabFrame(false, true, true, false)) != null) {
BufferedImage image = createBufferedImage(inFrame); // faster conversion
//BufferedImage image = converter.convert(inFrame); // JavaCV conversion
/* process image */
Frame outFrame = createFrame(image); // faster conversion
//Frame outFrame = converter.convert(image); // JavaCV conversion
recorder.record(outFrame, PIXEL_FORMAT);
++frameCount;
}
float secs = (System.currentTimeMillis() - startTime) / 1000.0f;
System.out.println("processing " + frameCount + " frames took " + (Math.round(secs * 1000) / 1000.0f) + " (" + (Math.round(frameCount / secs * 1000) / 1000.0f) + " fps)");
recorder.stop();
grabber.stop();
}
private static byte[] framePixels = null;
public static BufferedImage createBufferedImage(Frame frame) {
ByteBuffer buffer = (ByteBuffer) frame.image[0].position(0);
if(framePixels == null)
framePixels = new byte[buffer.limit()];
buffer.get(framePixels);
ColorSpace cs = ColorSpace.getInstance(ColorSpace.CS_sRGB);
ColorModel cm = new ComponentColorModel(cs, false,false, Transparency.OPAQUE, DataBuffer.TYPE_BYTE);
WritableRaster wr = Raster.createWritableRaster(new ComponentSampleModel(DataBuffer.TYPE_BYTE, frame.imageWidth, frame.imageHeight, frame.imageChannels, frame.imageStride, new int[] {2, 1, 0}), null);
byte[] bufferPixels = ((DataBufferByte) wr.getDataBuffer()).getData();
System.arraycopy(framePixels, 0, bufferPixels, 0, framePixels.length);
return new BufferedImage(cm, wr, false, null);
}
private static byte[] imagePixels = null;
public static Frame createFrame(BufferedImage image) {
DataBufferByte imageBuffer = (DataBufferByte) image.getRaster().getDataBuffer();
if(imagePixels == null)
imagePixels = new byte[imageBuffer.getData().length];
System.arraycopy(imageBuffer.getData(), 0, imagePixels, 0, imagePixels.length);
Frame frame = new Frame(image.getWidth(), image.getHeight(), 8, 3);
ByteBuffer frameBuffer = (ByteBuffer) frame.image[0].position(0);
frameBuffer.put(imagePixels);
return frame;
}
}