OpenCV

Optical Flow

Example that uses cv::calcOpticalFlowPyrLK() to calculate the optical flow between good trackable points.

Flow.cpp

#include <assert.h>
#include <swnt/Flow.h>
#include <swnt/Settings.h>
#include <swnt/Graphics.h>
 
Flow::Flow(Settings& settings, Graphics& graphics) 
  :settings(settings)
  ,graphics(graphics)
  ,prev_image(NULL)
  ,field_vao(0)
  ,field_vbo(0)
  ,field_bytes_allocated(0)
{
}
 
bool Flow::setup() {
 
  size_t nbytes = settings.image_processing_w * settings.image_processing_h ;
  prev_image = new unsigned char[nbytes];
 
  if(!prev_image) {
    printf("Error: cannot allocate the bytes for the previous image.\n");
    return false;
  }
  memset(prev_image, 0x00, nbytes);
 
  if(!setupGraphics()) {
    printf("Error: cannot setup the GL state in Flow.\n");
    return false;
  }
 
  return true;
}
 
bool Flow::setupGraphics() {
  glGenVertexArrays(1, &field_vao);
  glBindVertexArray(field_vao);
 
  glGenBuffers(1, &field_vbo);
  glBindBuffer(GL_ARRAY_BUFFER, field_vbo);
 
  glEnableVertexAttribArray(0); // pos
  glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, sizeof(vec2), (GLvoid*)0); // pos
  return true;
}
 
void Flow::draw(){
 
  if(field_vertices.size()) {
    glBindVertexArray(field_vao);
    glUseProgram(graphics.v_prog);
 
    vec3 color(1.0, 0.0, 0.0);
    mat4 mm;
    mm.translate(0, 240.0, 0);
    glUniform3fv(glGetUniformLocation(graphics.v_prog, "u_color"), 1, color.ptr());
    glUniformMatrix4fv(glGetUniformLocation(graphics.v_prog, "u_mm"), 1, GL_FALSE, mm.ptr());
    glUniformMatrix4fv(glGetUniformLocation(graphics.v_prog, "u_pm"), 1, GL_FALSE, settings.ortho_matrix.ptr());
    glDrawArrays(GL_LINES, 0, field_vertices.size());
    glDrawArrays(GL_POINTS, 0, field_vertices.size());
  }
}
 
/*
 
  This function will find good points to track in the previous frame
  and then tries to find those points in the current image (curr). The first
  time this function is ran we get wrong results as there is no "previous" image
  from which we can get good points to track.
 
  After finding good features in the previous frame, we use cv::calcOpticalFlowPyrLK
  to find those points in the current image. 
 
 */
void Flow::calc(unsigned char* curr) {
  assert(prev_image);
 
  prev_good_points.clear();
  curr_good_points.clear();
  status.clear();
 
  int w = settings.image_processing_w;
  int h = settings.image_processing_h;
  size_t nbytes = w * h;
 
  cv::Mat mat_curr(h, w, CV_8UC1, curr, cv::Mat::AUTO_STEP);
  cv::Mat mat_prev(h, w, CV_8UC1, prev_image, cv::Mat::AUTO_STEP);
 
  cv::goodFeaturesToTrack(mat_prev,            // input, the image from which we want to know good features to track
                          prev_good_points,    // output, the points will be stored in this output vector
                          40,                  // max points, maximum number of good features to track
                          0.05,                // quality level, "minimal accepted quality of corners", the lower the more points we will get
                          10,                  // minDistance, minimum distance between points
                          cv::Mat(),           // mask
                          4,                   // block size
                          false,               // useHarrisDetector, makes tracking a bit better when set to true
                          0.04                 // free parameter for harris detector
                          );
 
 
  if(!prev_good_points.size()) {
    memcpy(prev_image, curr, nbytes);
    return;
  }
 
  cv::TermCriteria termcrit(cv::TermCriteria::COUNT|cv::TermCriteria::EPS,prev_good_points.size(),0.03);
  std::vector<float> error;
 
  curr_good_points.assign(prev_good_points.size(), cv::Point2f());
 
  cv::calcOpticalFlowPyrLK(mat_prev,             // prev image 
                           mat_curr,             // curr image
                           prev_good_points,     // find these points in the new image
                           curr_good_points,     // result of found points
                           status,               // output status vector, found points are set to 1
                           error,                // each point gets an error value (see flag)
                           cv::Size(21, 21),     // size of the window at each pyramid level 
                           0,                    // maxLevel - 0 = no pyramids, > 0 use this level of pyramids
                           termcrit,             // termination criteria
                           0,                    // flags OPTFLOW_USE_INITIAL_FLOW or OPTFLOW_LK_GET_MIN_EIGENVALS
                           0.1                   // minEigThreshold 
                           );
 
 
  updateFieldVertices();
 
  memcpy(prev_image, curr, nbytes);
}
 
void Flow::updateFieldVertices() {
 
  field_vertices.clear();
 
  if(!curr_good_points.size()) {
    return;
  }
 
  if(curr_good_points.size() != prev_good_points.size()) {
    printf("not same size.\n");
    return;
  }
 
  for(size_t i = 0; i < curr_good_points.size(); ++i) {
 
    if(!status[i]) {
      continue;
    }
 
    cv::Point2f& c = curr_good_points[i];
 
    cv::Point2f& p = prev_good_points[i];
    vec2 v(p.x, p.y);
    field_vertices.push_back(v);
 
    v.set(c.x, c.y);
    field_vertices.push_back(v);
  }
 
  glBindBuffer(GL_ARRAY_BUFFER, field_vbo);
  size_t bytes_needed = sizeof(vec2) * field_vertices.size();
  if(bytes_needed > field_bytes_allocated) {
    glBufferData(GL_ARRAY_BUFFER, bytes_needed, field_vertices[0].ptr(), GL_STREAM_DRAW);
    field_bytes_allocated = bytes_needed;
  }
  else {
    glBufferSubData(GL_ARRAY_BUFFER, 0, bytes_needed, field_vertices[0].ptr());
  }
}

Flow.h

#ifndef SWNT_FLOW_H
#define SWNT_FLOW_H
 
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/video/tracking.hpp>
 
#define ROXLU_USE_OPENGL
#define ROXLU_USE_MATH
#define ROXLU_USE_PNG
#include "tinylib.h"
 
class Settings;
class Graphics;
 
class Flow {
 
 public:
  Flow(Settings& settings, Graphics& graphics);
  bool setup();
  void calc(unsigned char* curr);
  void draw();
 
 private:
  bool setupGraphics();
  void updateFieldVertices(); /* update the vertices that are used to draw the vector field */
 
 public:
  Settings& settings;
  Graphics& graphics;
  unsigned char* prev_image;
 
  std::vector<cv::Point2f> prev_good_points;
  std::vector<cv::Point2f> curr_good_points;
  std::vector<unsigned char> status;
 
  /* GL */
  GLuint field_vao;
  GLuint field_vbo;
  std::vector<vec2> field_vertices;
  size_t field_bytes_allocated;
};
 
#endif

Example