data type errors for input images of cv2.calcOpticalFlowPyrLK

Question

I'm running opencv 2.4.1 using python bindings and am having difficulty calculating the optical flow.

Specifically this section of code:

#calculate the opticalflow
if prev_saturation_thresh_img==None:
    prev_saturation_thresh_img=saturation_img
if i >=0:
    prev_img=prev_saturation_thresh_img
    next_img=saturation_thresh_img
    p1,  st, err = cv2.calcOpticalFlowPyrLK(prev_img,next_img,tracks_np,**lk_params)

Returns the error:

<unknown> is not a numpy array

So then I try to convert the images to numpy arrays:

prev_img=prev_saturation_thresh_img
next_img=saturation_thresh_img  

Now I have a new error:

<unknown> data type = 17 is not supported

In a last-ditch effort I convert the images to cvmat (from iplimage) before converting it to a numpy array, just to see what happens

error: ..\..\..\OpenCV-2.4.1\modules\video\src\lkpyramid.cpp:607: error: (-215) nextPtsMat.checkVector(2, CV_32F, true) == npoints

So now I'm stuck. Below is the code in it's entirety for reference

import cv
import cv2
import numpy as np

class Target:
    def __init__(self):
        self.capture = cv.CaptureFromFile("raw_gait_cropped.avi")

    def run(self):
        #initiate font
        font = cv.InitFont(cv.CV_FONT_HERSHEY_SIMPLEX, 1, 1, 0, 3, 8)

        #instantiate images
        img_size=cv.GetSize(cv.QueryFrame(self.capture))
        hsv_img=cv.CreateImage(img_size,8,3)
        saturation_img=cv.CreateImage(img_size,8,1)
        saturation_thresh_img=cv.CreateImage(img_size,8,1)
        prev_saturation_thresh_img=None

        #create params for GoodFeaturesToTrack and calcOpticalFlowPyrLK
        gftt_params = dict( cornerCount=11,
                            qualityLevel=0.2,
                            minDistance=5,
                            mask=None,
                            useHarris=True
                            )

        lk_params = dict(   winSize  = (15, 15), 
                            maxLevel = 2, 
                            criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03),
                            flags = cv2.OPTFLOW_USE_INITIAL_FLOW,
                            minEigThreshold=1
                            )    
        tracks=[]
        writer=cv.CreateVideoWriter("angle_tracking.avi",cv.CV_FOURCC('M','J','P','G'),30,cv.GetSize(hsv_img),1)

        i=0
        while True:
            #grab a frame from the video capture
            img=cv.QueryFrame(self.capture)

            #break the loop when the video is over
            if img == None:
                break

            #convert the image to HSV
            cv.CvtColor(img,hsv_img,cv.CV_BGR2HSV)

            #Get Saturation channel
            cv.MixChannels([hsv_img],[saturation_img],[(1,0)])

            #Apply threshold to saturation channel
            cv.InRangeS(saturation_img,145,255,saturation_thresh_img)

            #locate initial features to track
            if i==0:
                eig_image=temp_image = cv.CreateMat(img.height, img.width, cv.CV_32FC1)
                for (x,y) in cv.GoodFeaturesToTrack(saturation_thresh_img, eig_image, temp_image, **gftt_params):
                    tracks.append([(x,y)])
                    cv.Circle(saturation_thresh_img,(int(x),int(y)),5,(255,255,255),-1,cv.CV_AA,0)
                tracks_np=np.float32(tracks).reshape(-1,2)
                print tracks

            #calculate the opticalflow
            if prev_saturation_thresh_img==None:
                prev_saturation_thresh_img=saturation_img
            if i >=0:
            prev_img=prev_saturation_thresh_img
                next_img=saturation_thresh_img
                p1,  st, err = cv2.calcOpticalFlowPyrLK(prev_img,next_img,tracks_np,**lk_params)
                prev_saturation_thresh_img=saturation_img   
            i=i+1
            print i
            #display frames to users
            cv.ShowImage("Raw Video",img)
            cv.ShowImage("Saturation Channel",saturation_img)
            cv.ShowImage("Saturation Thresholded",saturation_thresh_img)

            # Listen for ESC or ENTER key
            c = cv.WaitKey(7) % 0x100
            if c == 27 or c == 10:
                break
        #close all windows once video is done
        cv.DestroyAllWindows()



if __name__=="__main__":
    t = Target()
    t.run()

Answer 1

OpenCV can be very picky about the data formats it accepts. The following code extract works for me:

prev = cv.LoadImage('images/'+file_list[0])
prev = np.asarray(prev[:,:])
prev_gs = cv2.cvtColor(prev, cv2.COLOR_BGR2GRAY)

current = cv.LoadImage('images/'+file)
current = np.asarray(current[:,:])
current_gs = cv2.cvtColor(current, cv2.COLOR_BGR2GRAY)

features, status, track_error = cv2.calcOpticalFlowPyrLK(prev_gs, current_gs, good_features, None,    
**lk_params)

Note the [:,:] when converting from images to numpy arrays, I have found that they are required.

I hope that this may solve your problem.