Using SIFT/SURF for Object Recognition in OpenCV Java

Hi All,

Today my post is on, how you can use SIFT/SURF algorithms for Object Recognition with OpenCV Java. I have shared this post on SURF feature detector previously. This is fully based on that post and therefore I'm just trying to show you how you can implement the same logic in OpenCV Java. It's important that you have to download previous OpenCV versions, so that you have SURF feature detector in your library. Because in the newer versions they have removed these Non-Free modules from the java wrapper. You can check this out.

I believe that you have some basic knowledge in working with OpenCV Java. If you want some beginner help you can refer the following links.

• TutorialsPoint
• Opencv 2.4.9 Documentation ( Java API )
• Using OpenCV Java with Eclipse

Now let's move on to our development. I'm using Eclipse for developing this. I'm going to use OpencCV 2.4.11. You can download it here. 

First create the user library for OpenCV as described in the previous link and add it to the build path. Then we can start developing the code for object recognition. Following is my eclipse project. I have added the OpenCV 2.4.11 library as a user library and added it to the build path. 

I'm using following images for object recognition. 

1. Object we are going to recognize. 

2. Scene that we are going to recognize the object from. 

Now following is the Java implementation of our SURF feature detector. 

	package com.dummys.learning;
	import org.opencv.calib3d.Calib3d;
	import org.opencv.core.*;
	import org.opencv.features2d.*;
	import org.opencv.highgui.Highgui;
	import java.io.File;
	import java.util.LinkedList;
	import java.util.List;
	/**
	* Created by Kinath on 8/6/2016.
	*/
	public class SURFDetector {
	public static void main(String[] args) {
	File lib = null;
	String os = System.getProperty("os.name");
	String bitness = System.getProperty("sun.arch.data.model");
	if (os.toUpperCase().contains("WINDOWS")) {
	if (bitness.endsWith("64")) {
	lib = new File("libs//x64//" + System.mapLibraryName("opencv_java2411"));
	} else {
	lib = new File("libs//x86//" + System.mapLibraryName("opencv_java2411"));
	}
	}
	System.out.println(lib.getAbsolutePath());
	System.load(lib.getAbsolutePath());
	String bookObject = "images//bookobject.jpg";
	String bookScene = "images//bookscene.jpg";
	System.out.println("Started....");
	System.out.println("Loading images...");
	Mat objectImage = Highgui.imread(bookObject, Highgui.CV_LOAD_IMAGE_COLOR);
	Mat sceneImage = Highgui.imread(bookScene, Highgui.CV_LOAD_IMAGE_COLOR);
	MatOfKeyPoint objectKeyPoints = new MatOfKeyPoint();
	FeatureDetector featureDetector = FeatureDetector.create(FeatureDetector.SURF);
	System.out.println("Detecting key points...");
	featureDetector.detect(objectImage, objectKeyPoints);
	KeyPoint[] keypoints = objectKeyPoints.toArray();
	System.out.println(keypoints);
	MatOfKeyPoint objectDescriptors = new MatOfKeyPoint();
	DescriptorExtractor descriptorExtractor = DescriptorExtractor.create(DescriptorExtractor.SURF);
	System.out.println("Computing descriptors...");
	descriptorExtractor.compute(objectImage, objectKeyPoints, objectDescriptors);
	// Create the matrix for output image.
	Mat outputImage = new Mat(objectImage.rows(), objectImage.cols(), Highgui.CV_LOAD_IMAGE_COLOR);
	Scalar newKeypointColor = new Scalar(255, 0, 0);
	System.out.println("Drawing key points on object image...");
	Features2d.drawKeypoints(objectImage, objectKeyPoints, outputImage, newKeypointColor, 0);
	// Match object image with the scene image
	MatOfKeyPoint sceneKeyPoints = new MatOfKeyPoint();
	MatOfKeyPoint sceneDescriptors = new MatOfKeyPoint();
	System.out.println("Detecting key points in background image...");
	featureDetector.detect(sceneImage, sceneKeyPoints);
	System.out.println("Computing descriptors in background image...");
	descriptorExtractor.compute(sceneImage, sceneKeyPoints, sceneDescriptors);
	Mat matchoutput = new Mat(sceneImage.rows() * 2, sceneImage.cols() * 2, Highgui.CV_LOAD_IMAGE_COLOR);
	Scalar matchestColor = new Scalar(0, 255, 0);
	List<MatOfDMatch> matches = new LinkedList<MatOfDMatch>();
	DescriptorMatcher descriptorMatcher = DescriptorMatcher.create(DescriptorMatcher.FLANNBASED);
	System.out.println("Matching object and scene images...");
	descriptorMatcher.knnMatch(objectDescriptors, sceneDescriptors, matches, 2);
	System.out.println("Calculating good match list...");
	LinkedList<DMatch> goodMatchesList = new LinkedList<DMatch>();
	float nndrRatio = 0.7f;
	for (int i = 0; i < matches.size(); i++) {
	MatOfDMatch matofDMatch = matches.get(i);
	DMatch[] dmatcharray = matofDMatch.toArray();
	DMatch m1 = dmatcharray[0];
	DMatch m2 = dmatcharray[1];
	if (m1.distance <= m2.distance * nndrRatio) {
	goodMatchesList.addLast(m1);
	}
	}
	if (goodMatchesList.size() >= 7) {
	System.out.println("Object Found!!!");
	List<KeyPoint> objKeypointlist = objectKeyPoints.toList();
	List<KeyPoint> scnKeypointlist = sceneKeyPoints.toList();
	LinkedList<Point> objectPoints = new LinkedList<>();
	LinkedList<Point> scenePoints = new LinkedList<>();
	for (int i = 0; i < goodMatchesList.size(); i++) {
	objectPoints.addLast(objKeypointlist.get(goodMatchesList.get(i).queryIdx).pt);
	scenePoints.addLast(scnKeypointlist.get(goodMatchesList.get(i).trainIdx).pt);
	}
	MatOfPoint2f objMatOfPoint2f = new MatOfPoint2f();
	objMatOfPoint2f.fromList(objectPoints);
	MatOfPoint2f scnMatOfPoint2f = new MatOfPoint2f();
	scnMatOfPoint2f.fromList(scenePoints);
	Mat homography = Calib3d.findHomography(objMatOfPoint2f, scnMatOfPoint2f, Calib3d.RANSAC, 3);
	Mat obj_corners = new Mat(4, 1, CvType.CV_32FC2);
	Mat scene_corners = new Mat(4, 1, CvType.CV_32FC2);
	obj_corners.put(0, 0, new double[]{0, 0});
	obj_corners.put(1, 0, new double[]{objectImage.cols(), 0});
	obj_corners.put(2, 0, new double[]{objectImage.cols(), objectImage.rows()});
	obj_corners.put(3, 0, new double[]{0, objectImage.rows()});
	System.out.println("Transforming object corners to scene corners...");
	Core.perspectiveTransform(obj_corners, scene_corners, homography);
	Mat img = Highgui.imread(bookScene, Highgui.CV_LOAD_IMAGE_COLOR);
	Core.line(img, new Point(scene_corners.get(0, 0)), new Point(scene_corners.get(1, 0)), new Scalar(0, 255, 0), 4);
	Core.line(img, new Point(scene_corners.get(1, 0)), new Point(scene_corners.get(2, 0)), new Scalar(0, 255, 0), 4);
	Core.line(img, new Point(scene_corners.get(2, 0)), new Point(scene_corners.get(3, 0)), new Scalar(0, 255, 0), 4);
	Core.line(img, new Point(scene_corners.get(3, 0)), new Point(scene_corners.get(0, 0)), new Scalar(0, 255, 0), 4);
	System.out.println("Drawing matches image...");
	MatOfDMatch goodMatches = new MatOfDMatch();
	goodMatches.fromList(goodMatchesList);
	Features2d.drawMatches(objectImage, objectKeyPoints, sceneImage, sceneKeyPoints, goodMatches, matchoutput, matchestColor, newKeypointColor, new MatOfByte(), 2);
	Highgui.imwrite("output//outputImage.jpg", outputImage);
	Highgui.imwrite("output//matchoutput.jpg", matchoutput);
	Highgui.imwrite("output//img.jpg", img);
	} else {
	System.out.println("Object Not Found");
	}
	System.out.println("Ended....");
	}
	}

view raw SURFDetector.java hosted with ❤ by GitHub

Following are our outputs. 

1. Identified key-points of the object.

2. Matching the object keypoints with the scene. 

3. Object recognized image. 

You can do the same with SIFT feature detector by just changing the Feature Detector and Descriptor Extractor name to SIFT. 

So that's it. 

Hope that helps. 

Thank You. :-) 

Develop OpenCV Cross Platform Applications with Java

Hi All,

Today I'm going to show you how you can start developing OpenCV applications with Java that can work independently from the underlying OS. This might not be the best implementation, but what I'm going to share works fine.

I assume you have some basic knowledge in working with OpenCV in Java. You can easily download OpenCV for Windos/Linux/Mac from OpenCV downloads. Here I'm going to use OpenCV 2.4.11. You can download your preferred version.

I'm going to use the following simple code to demonstrate that OpenCV is working.

import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;

public class Main
{
 public static void main(String[] args)
 {
  System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
  Mat mat = Mat.eye(3, 3, CvType.CV_8UC1);
                System.out.println("mat = " + mat.dump());
 }
}

Our purpose here is to provide the above code to work in any platform. ( Here I'm going to show only for Windows and Linux. You can use similar method for Mac)

Therefore first we have to build necessary .dll and .so files so that we can use them in our application. For Windows it's very easy because, when you download OpenCV for windows and extract it to a particular location you can find the .dll files in \build\java\x64 and in \build\java\x86.

Next we have to build the .so files.

In your Linux machine that you are going to build .so files, make sure you have installed the following programs before you follow the next set of instructions.

1. g++
2. cmake
3. ant

You can install the above programs by just using 'sudo apt-get install ' .
Make sure you have set the Java path in your terminal before you execute the commands. You can use 'export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64' command to do this.

Once you have set up your environment you can use the following instructions to build the .jar and .so files for opencv.

1. Download OpenCV for Linux.
2. Extract the .zip file to a location.
3. CD into that location.
4. Execute the following commands.

mkdir release
cd release
cmake -DBUILD_SHARED_LIBS=OFF ..
make -j8

When you run the cmake -DBUILD_SHARED_LIBS=OFF .. command make sure you get the following output. Otherwise you won't get the .so and .jar files created.

Inside the release folder you can find the newly created files. Inside 'bin' folder you can find the 'opencv-2411.jar' and inside 'lib' folder you can find the 'libopencv_java2411.so' file.

You can build for both Linux 32 and 64 bit environments.

Now we are ready to develop our OpenCV program for both Windows and Linux environments along with their respective bitness (64 or 32).

I am using Eclipse to develop my application. First add the 'opencv-2411.jar' to your project and add it to the Build Path.

Then copy the .dll and .so files for both 32 and 64bit versions to a folder inside the project. Now with the following code, you can start loading the libraries for the respective OS.

if (os.toUpperCase().contains("WINDOWS"))
  {
   if (bitness.endsWith("64"))
   {
    lib = new File("native/64x" + System.mapLibraryName("opencv_java2411"));
   }
   else
   {
    lib = new File("native/86x" + System.mapLibraryName("opencv_java2411"));
   }
  }
  else if (os.toUpperCase().contains("LINUX"))
  {
   if (bitness.endsWith("64"))
   {
    lib = new File("native/64x" + System.mapLibraryName("opencv_java2411"));
   }
   else
   {
    lib = new File("native/86x" + System.mapLibraryName("opencv_java2411"));
   }
  }
  
  System.out.println(lib.getAbsolutePath());
  System.load(lib.getAbsolutePath());

Insert the above code block inside your class. Then you can run the previous demo code without using the 'System.loadLibrary(Core.NATIVE_LIBRARY_NAME);' line, because we have already loaded the library from the above block.

Following is the complete code. I have loaded the library inside the main method. In your applications you can use it inside a Static code block according to your application.

import java.io.File;

import org.opencv.core.CvType;
import org.opencv.core.Mat;

public class Main
{
 public static void main(String[] args)
 {
  File lib = null;
  String os = System.getProperty("os.name");
  String bitness = System.getProperty("sun.arch.data.model");
  
  if (os.toUpperCase().contains("WINDOWS"))
  {
   if (bitness.endsWith("64"))
   {
    lib = new File("lib/opencv/64x" + System.mapLibraryName("opencv_java2411"));
   }
   else
   {
    lib = new File("lib/opencv/86x" + System.mapLibraryName("opencv_java2411"));
   }
  }
  else if (os.toUpperCase().contains("LINUX"))
  {
   if (bitness.endsWith("64"))
   {
    lib = new File("lib/opencv/64x" + System.mapLibraryName("opencv_java2411"));
   }
   else
   {
    lib = new File("lib/opencv/86x" + System.mapLibraryName("opencv_java2411"));
   }
  }
  
  System.out.println(lib.getAbsolutePath());
  System.load(lib.getAbsolutePath());
  
  Mat mat = Mat.eye(3, 3, CvType.CV_8UC1);
        System.out.println("mat = " + mat.dump());
 }
}

Here my native .dll and .so files are inside a folder named 'lib/opencv/'.

You will get the following output. 

Hope this helps.

Thank you..

:-)