Detecting object regions in image opencv

Question

We're currently trying to detect the object regions in medical instruments images using the methods available in OpenCV, C++ version. An example image is shown below:

Here are the steps we're following:

• Converting the image to gray scale
• Applying median filter
• Find edges using sobel filter
• Convert the result to binary image using a threshold of 25
• Skeletonize the image to make sure we have neat edges
• Finding X largest connected components

This approach works perfectly for the image 1 and here is the result:

• The yellow borders are the connected components detected.
• The rectangles are just to highlight the presence of a connected component.
• To get understandable results, we just removed the connected components that are completely inside any another one, so the end result is something like this:

So far, everything was fine but another sample of image complicated our work shown below.

Having a small light green towel under the objects results this image:

After filtering the regions as we did earlier, we got this:

Obviously, it is not what we need..we're excepting something like this:

I'm thinking about clustering the closest connected components found(somehow!!) so we can minimize the impact of the presence of the towel, but don't know yet if it's something doable or someone has tried something like this before? Also, does anyone have any better idea to overcome this kind of problems?

Thanks in advance.

Answer 1

Here's what I tried.

In the images, the background is mostly greenish and the area of the background is considerably larger than that of the foreground. So, if you take a color histogram of the image, the greenish bins will have higher values. Threshold this histogram so that bins having smaller values are set to zero. This way we'll most probably retain the greenish (higher value) bins and discard other colors. Then backproject this histogram. The backprojection will highlight these greenish regions in the image.

Backprojection:

• Then threshold this backprojection. This gives us the background.

Background (after some morphological filtering):

• Invert the background to get foreground.

Foreground (after some morphological filtering):

• Then find the contours of the foreground.

I think this gives a reasonable segmentation, and using this as mask you may be able to use a segmentation like GrabCut to refine the boundaries (I haven't tried this yet).

EDIT: I tried the GrabCut approach and it indeed refines the boundaries. I've added the code for GrabCut segmentation.

Contours:

GrabCut segmentation using the foreground as mask:

I'm using the OpenCV C API for the histogram processing part.

// load the color image
IplImage* im = cvLoadImage("bFly6.jpg");

// get the color histogram
IplImage* im32f = cvCreateImage(cvGetSize(im), IPL_DEPTH_32F, 3);
cvConvertScale(im, im32f);

int channels[] = {0, 1, 2};
int histSize[] = {32, 32, 32};
float rgbRange[] = {0, 256};
float* ranges[] = {rgbRange, rgbRange, rgbRange};

CvHistogram* hist = cvCreateHist(3, histSize, CV_HIST_ARRAY, ranges);
IplImage* b = cvCreateImage(cvGetSize(im32f), IPL_DEPTH_32F, 1);
IplImage* g = cvCreateImage(cvGetSize(im32f), IPL_DEPTH_32F, 1);
IplImage* r = cvCreateImage(cvGetSize(im32f), IPL_DEPTH_32F, 1);
IplImage* backproject32f = cvCreateImage(cvGetSize(im), IPL_DEPTH_32F, 1);
IplImage* backproject8u = cvCreateImage(cvGetSize(im), IPL_DEPTH_8U, 1);
IplImage* bw = cvCreateImage(cvGetSize(im), IPL_DEPTH_8U, 1);
IplConvKernel* kernel = cvCreateStructuringElementEx(3, 3, 1, 1, MORPH_ELLIPSE);

cvSplit(im32f, b, g, r, NULL);
IplImage* planes[] = {b, g, r};
cvCalcHist(planes, hist);

// find min and max values of histogram bins
float minval, maxval;
cvGetMinMaxHistValue(hist, &minval, &maxval);

// threshold the histogram. this sets the bin values that are below the threshold to zero
cvThreshHist(hist, maxval/32);

// backproject the thresholded histogram. backprojection should contain higher values for the
// background and lower values for the foreground
cvCalcBackProject(planes, backproject32f, hist);

// convert to 8u type
double min, max;
cvMinMaxLoc(backproject32f, &min, &max);
cvConvertScale(backproject32f, backproject8u, 255.0 / max);

// threshold backprojected image. this gives us the background
cvThreshold(backproject8u, bw, 10, 255, CV_THRESH_BINARY);

// some morphology on background
cvDilate(bw, bw, kernel, 1);
cvMorphologyEx(bw, bw, NULL, kernel, MORPH_CLOSE, 2);

// get the foreground
cvSubRS(bw, cvScalar(255, 255, 255), bw);
cvMorphologyEx(bw, bw, NULL, kernel, MORPH_OPEN, 2);
cvErode(bw, bw, kernel, 1);

// find contours of the foreground
//CvMemStorage* storage = cvCreateMemStorage(0);
//CvSeq* contours = 0;
//cvFindContours(bw, storage, &contours);
//cvDrawContours(im, contours, CV_RGB(255, 0, 0), CV_RGB(0, 0, 255), 1, 2);

// grabcut
Mat color(im);
Mat fg(bw);
Mat mask(bw->height, bw->width, CV_8U);

mask.setTo(GC_PR_BGD);
mask.setTo(GC_PR_FGD, fg);

Mat bgdModel, fgdModel;
grabCut(color, mask, Rect(), bgdModel, fgdModel, GC_INIT_WITH_MASK);

Mat gcfg = mask == GC_PR_FGD;

vector<vector<cv::Point>> contours;
vector<Vec4i> hierarchy;
findContours(gcfg, contours, hierarchy, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE, cv::Point(0, 0));
for(int idx = 0; idx < contours.size(); idx++)
{
    drawContours(color, contours, idx, Scalar(0, 0, 255), 2);
}

// cleanup ...

UPDATE: We can do the above using the C++ interface as shown below.

const int channels[] = {0, 1, 2};
const int histSize[] = {32, 32, 32};
const float rgbRange[] = {0, 256};
const float* ranges[] = {rgbRange, rgbRange, rgbRange};

Mat hist;
Mat im32fc3, backpr32f, backpr8u, backprBw, kernel;

Mat im = imread("bFly6.jpg");

im.convertTo(im32fc3, CV_32FC3);
calcHist(&im32fc3, 1, channels, Mat(), hist, 3, histSize, ranges, true, false);
calcBackProject(&im32fc3, 1, channels, hist, backpr32f, ranges);

double minval, maxval;
minMaxIdx(backpr32f, &minval, &maxval);
threshold(backpr32f, backpr32f, maxval/32, 255, THRESH_TOZERO);
backpr32f.convertTo(backpr8u, CV_8U, 255.0/maxval);
threshold(backpr8u, backprBw, 10, 255, THRESH_BINARY);

kernel = getStructuringElement(MORPH_ELLIPSE, Size(3, 3));

dilate(backprBw, backprBw, kernel);
morphologyEx(backprBw, backprBw, MORPH_CLOSE, kernel, Point(-1, -1), 2);

backprBw = 255 - backprBw;

morphologyEx(backprBw, backprBw, MORPH_OPEN, kernel, Point(-1, -1), 2);
erode(backprBw, backprBw, kernel);

Mat mask(backpr8u.rows, backpr8u.cols, CV_8U);

mask.setTo(GC_PR_BGD);
mask.setTo(GC_PR_FGD, backprBw);

Mat bgdModel, fgdModel;
grabCut(im, mask, Rect(), bgdModel, fgdModel, GC_INIT_WITH_MASK);

Mat fg = mask == GC_PR_FGD;