From 60c5254a471f841e0cb8edc07b7c8631c6ab366f Mon Sep 17 00:00:00 2001
From: Epicalert <epicalert@protonmail.com>
Date: Sun, 6 Jun 2021 20:20:43 +0800
Subject: [PATCH] CV: Add eye look direction
---
CMakeLists.txt | 1 +
src/cv.cpp | 27 +++++++++++--
src/eye.cpp | 104 +++++++++++++++++++++++++++++++++++++++++++++++++
src/eye.hpp | 10 +++++
4 files changed, 139 insertions(+), 3 deletions(-)
create mode 100644 src/eye.cpp
create mode 100644 src/eye.hpp
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2b96d3a..dfba7be 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -61,6 +61,7 @@ add_executable( fc2d
src/toml.c
src/tomlcpp.cpp
src/error.cpp
+ src/eye.cpp
)
target_link_libraries( fc2d ${OpenCV_LIBS} ${OPENGL_LIBRARIES} ${WEBP_LIBRARIES}
FreeGLUT::freeglut GLEW::glew zip Boxer fmt )
diff --git a/src/cv.cpp b/src/cv.cpp
index 31d997a..146ca14 100644
--- a/src/cv.cpp
+++ b/src/cv.cpp
@@ -7,6 +7,7 @@
#include <paths.hpp>
#include <args.hpp>
#include <cv.hpp>
+#include <eye.hpp>
#include <modelpart.hpp>
cv::Ptr<cv::face::Facemark> facemark;
@@ -98,9 +99,9 @@ void cvFrame() {
std::vector<std::vector<cv::Point2f>> landmarks;
if (facemark->fit (frame, faces, landmarks)) {
- //for (int i = 0; i < landmarks[biggestFace].size(); i++) {
- // cv::circle (frame, landmarks[biggestFace][i], 2, cv::Scalar (255, 255, 255));
- //}
+ for (int i = 0; i < landmarks[biggestFace].size(); i++) {
+ cv::circle (frame, landmarks[biggestFace][i], 2, cv::Scalar (255, 255, 255));
+ }
cv::circle(frame, cv::Point2f(
(landmarks[biggestFace][2].x + landmarks[biggestFace][14].x) / 2,
(landmarks[biggestFace][2].y + landmarks[biggestFace][14].y) / 2
@@ -109,6 +110,20 @@ void cvFrame() {
cv::circle (frame, landmarks[biggestFace][66], 3, cv::Scalar (0, 255, 0));
cv::circle (frame, landmarks[biggestFace][62], 3, cv::Scalar (0, 255, 0));
+ //get ROI for eyes
+ float eyeWidth = landmarks[biggestFace][45].x - landmarks[biggestFace][42].x;
+ cv::Rect eyeRect(landmarks[biggestFace][42].x, landmarks[biggestFace][42].y - eyeWidth / 2, eyeWidth, eyeWidth);
+
+ cv::rectangle(frame, eyeRect, cv::Scalar(255, 255, 255));
+
+ cv::Mat eyeROI;
+ eyeROI = gray(eyeRect);
+
+ glm::vec2 eyeVector = eyeDirection(eyeROI);
+
+ cv::imshow("eye", eyeROI);
+ cv::waitKey(1);
+
//send control information to graphics
float faceSize = landmarks[biggestFace][14].x - landmarks[biggestFace][2].x;
@@ -124,6 +139,7 @@ void cvFrame() {
landmarks[biggestFace][30].x * 2 / (float)frame.cols - 1,
landmarks[biggestFace][30].y * 2 / (float)frame.rows - 1
);
+
faceData.triggers[TRIGGER_NULL] = false;
faceData.triggers[TRIGGER_MOUTH_OPEN] =
(landmarks[biggestFace][66].y - landmarks[biggestFace][62].y) / faceSize > 0.04f;
@@ -133,6 +149,11 @@ void cvFrame() {
(float)(landmarks[biggestFace][2].x - landmarks[biggestFace][14].x));
faceData.scale = faceSize * 6 / (float)frame.cols;
+
+ cv::line(frame, cv::Point(50,50), cv::Point(50,50) + cv::Point(eyeVector.x * 25, eyeVector.y * 25), cv::Scalar(255,128,128));
+ std::cout << eyeVector.x << "," << eyeVector.y << std::endl;
+ std::cout << "SIZE:" << eyeROI.cols << "," << eyeROI.rows << std::endl;
+
updateModel(faceData);
}
}
diff --git a/src/eye.cpp b/src/eye.cpp
new file mode 100644
index 0000000..ead36e7
--- /dev/null
+++ b/src/eye.cpp
@@ -0,0 +1,104 @@
+#include <eye.hpp>
+
+
+float objectiveFunction(cv::Point2f c, cv::Mat gX, cv::Mat gY) {
+ float sum = 0;
+ for(int xx = 0; xx < gX.cols; xx++) {
+ for(int xy = 0; xy < gX.rows; xy++) {
+ cv::Point2f x(xx, xy);
+ cv::Point2f g(gX.at<float>(xx, xy), gY.at<float>(xx, xy)); // gradient
+ double gMag = std::sqrt(g.x * g.x + g.y * g.y);
+ if(gMag < 200.0) continue; // ignore gradients in homogenous regions
+ //g = g / gMag;
+ cv::Point2f d = x - c; // displacement
+ d /= std::sqrt(d.x * d.x + d.y * d.y); // normalize d
+
+ float dotProduct = d.y * g.x + d.x * g.y;
+
+ dotProduct = std::max(0.0f, dotProduct);
+
+ sum += dotProduct * dotProduct;
+ }
+ }
+ sum /= gX.rows * gX.cols;
+
+ return sum;
+}
+
+cv::Point2f derivativeFunction(cv::Point2f c, cv::Mat gX, cv::Mat gY) {
+ cv::Point2f sum(0,0);
+
+ for(int xx = 0; xx < gX.cols; xx++) {
+ for(int xy = 0; xy < gX.rows; xy++) {
+ cv::Point2f x(xy, xx);
+ cv::Point2f g(gY.at<float>(xx, xy), gX.at<float>(xx, xy)); // gradient
+ float gMag = std::sqrt(g.x * g.x + g.y * g.y);
+ if(gMag < 200.0f) continue; // ignore gradients in homogenous regions
+ //g = g / gMag;
+ cv::Point2f d = x - c; // displacement
+ float n = std::sqrt(d.x * d.x + d.y * d.y);
+ if (n == 0) continue;
+ //d /= n; // normalize d
+ float e = d.x * g.y + d.y * g.x;
+
+ //float dotProduct = d.y * g.x + d.x * g.y;
+
+ //e = std::max(0.0f, e);
+
+ //sum += dotProduct * dotProduct;// / (gray.rows * gray.cols);
+
+ sum += (d * (e * e) - g * e * (n * n)) / (n * n * n * n);
+ }
+ }
+
+ return sum;
+}
+
+glm::vec2 eyeDirection(cv::Mat roi) {
+ cv::Mat gX, gY;
+ cv::Sobel(roi, gX, CV_32F, 1, 0);
+ cv::Sobel(roi, gY, CV_32F, 0, 1);
+
+ float stepSize = roi.rows / 10;
+ float maxVal = 0;
+
+ cv::Point2f irisPosition;
+ for(int i = 0; i < 32; i++) {
+ cv::Point2f c(std::rand() % roi.cols, std::rand() % roi.rows); //start at a random point
+
+ float prevVal = 0;
+ for(int j = 0; j < 8; j++) {
+ cv::Point2f gradient = derivativeFunction(c, gX, gY); // calculate gradient
+ gradient /= std::sqrt(gradient.x * gradient.x +
+ gradient.y * gradient.y); // normalize
+
+ for(int k = 0; k < 6; k++) {
+ cv::Point2f newC = c + gradient * stepSize;
+ if (newC.x < 0 || newC.x > roi.cols || newC.y < 0 || newC.y > roi.rows) continue;
+
+ float newVal = objectiveFunction(newC, gX, gY);
+ if (newVal > prevVal) {
+ //c += sum * 3;// * 2 / (gray.rows * gray.cols);
+ c = newC;
+ prevVal = newVal;
+ break;
+ //cv::drawMarker(frame, c, cv::Scalar(0,0,255));
+ } else {
+ stepSize /= 2;
+ }
+ }
+ }
+
+ if(prevVal > maxVal) {
+ maxVal = prevVal;
+ irisPosition = cv::Point2f(c.y, c.x); // no idea why but the coordinates are flipped
+ }
+ }
+
+ cv::drawMarker(roi, irisPosition, cv::Scalar(128,128,128));
+
+ // convert result to vector for graphics
+ return glm::vec2(
+ (irisPosition.x - roi.rows / 2.0) / (roi.rows / 2.0),
+ (irisPosition.y - roi.rows / 2.0) / (roi.rows / 2.0));
+}
diff --git a/src/eye.hpp b/src/eye.hpp
new file mode 100644
index 0000000..e437ce9
--- /dev/null
+++ b/src/eye.hpp
@@ -0,0 +1,10 @@
+#ifndef EYE_HPP
+#define EYE_HPP
+
+#include <opencv2/opencv.hpp>
+
+#include<glm/vec2.hpp>
+
+glm::vec2 eyeDirection(cv::Mat roi);
+
+#endif