Python dlib face detection and blink counter

Introduction

Experiments in the field of image processing with python. With the introduction of dlib, we created something that can detect blinks in real time as part of detecting the movement of each number on the face.

Environmental content

Windows10 Edita info.: PyCharm 2019.1.3 (Community Edition) Windows 10 10.0 ・ Cv version == 4.1.0 ・ Numpy version == 1.16.4 ・ Dlib version == 19.17.0

Operation (runtime)

Blink detection and counter start in real time at run time ・ Simultaneous display of not only points but also detected face by "A" key ・ If you press the "A" key again, only points will be displayed. -End with "Esc" key (Escape).

Source code

import cv2 as cv
import numpy as np
import dlib

print('cv version is ', cv.__version__)
print('numpy version is ', np.__version__)
print('dlib version is', dlib.__version__)

#video setting and each definition
cap = cv.VideoCapture(0)
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
showSep = True
showMidResult = True
lastLeftOpen = 1
curLeftOpen = 1
lastRightOpen = 1
curRightOpen = 1
leftBlinkCount = 0
rightBlinkCount = 0

def eye_aspect_ratio(eye):
    # compute the distances between the two sets of
    # vertical eye landmarks (x, y)-coordinates
    dv1 = pow(pow(eye[1].x - eye[5].x, 2) + pow(eye[1].y - eye[5].y, 2), 0.5)
    dv2 = pow(pow(eye[2].x - eye[4].x, 2) + pow(eye[2].y - eye[4].y, 2), 0.5)
    # compute the distance between the horizontal
    # eye landmark (x, y)-coordinates
    dl3 = pow(pow(eye[0].x - eye[3].x, 2) + pow(eye[0].y - eye[3].y, 2), 0.5)
    # compute and return the eye aspect ratio
    return (dv1 + dv2) / dl3

while cap.isOpened():
    # 1. Take each frame
    # ret, frame = cap.read()
    cap.grab()
    ret, frame = cap.retrieve()
    if np.shape(frame) == ():
        continue

    # flip left and right if your camera need
    frame = cv.flip(frame, 1)

    if showSep:
        cv.namedWindow("camera image", 0)
        cv.imshow("camera image", frame)
    else:
        cv.destroyWindow("camera image")

    k = cv.waitKey(1) & 0xFF
    if k == 27:
        break
    # if click A or a
    if k == 65 or k == 97:
        showSep = not showSep

    if k == 65+2 or k == 97 + 2:
        showMidResult = not showMidResult

    # 2. detect face
    dets = detector(frame[:, :, ::-1])
    if len(dets) < 1:
        cv.putText(frame, "Please show more than half face {:d}".format(len(dets)), (0, 80),
                   cv.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)

        cv.namedWindow("blink check", 0)
        cv.imshow("blink check", frame)
        continue

    parts = predictor(frame, dets[0]).parts()
    img = frame
    if showSep:
        img = frame * 0

    for i in parts:
        cv.circle(img, (i.x, i.y), 3, (255, 0, 0), -1)

    # 3. find eye here
    # eye 1
    leftEye = [parts[36], parts[37], parts[38], parts[39], parts[40], parts[41]]
    for i in leftEye:
        cv.circle(img, (i.x, i.y), 3, (0, 255, 0), -1)
    # eye 2
    rightEye = parts[42:48]
    for i in rightEye:
        cv.circle(img, (i.x, i.y), 3, (0, 0, 255), -1)

    if showSep:
        cv.namedWindow("detected", 1)
        cv.imshow("detected", img)
    else:
        cv.destroyWindow("detected")

    # 4. check blink
    checkLeft = eye_aspect_ratio(leftEye)
    lineColor = (0, 255, 0)

    if checkLeft < 0.3:
        curLeftOpen = 0
        lineColor = (0, 0, 255)
    else:
        curLeftOpen = 1
        lineColor = (0, 255, 0)

    if lastLeftOpen == 1 and curLeftOpen == 0:
        leftBlinkCount += 1
    lastLeftOpen = curLeftOpen

    for i in leftEye:
        cv.circle(img, (i.x, i.y), 3, lineColor, -1)

    cv.putText(img, "Left Blinks: {}".format(leftBlinkCount), (10, 30), cv.FONT_HERSHEY_SIMPLEX, 0.7, lineColor, 2)
    cv.putText(img, "Eye ratio: {:.2f}".format(checkLeft), (10, 80), cv.FONT_HERSHEY_SIMPLEX, 0.7, lineColor, 2)

    print("left", checkLeft, leftBlinkCount, '\n')

    # eye 2
    checkRight = eye_aspect_ratio(rightEye)

    lineColor = (0, 255, 0)
    if checkRight < 0.3:
        curRightOpen = 0
        lineColor = (0, 0, 255)
    else:
        curRightOpen = 1
        lineColor = (0, 255, 0)

    if curRightOpen == 0 and lastRightOpen == 1:
        rightBlinkCount += 1
    lastRightOpen = curRightOpen

    for i in rightEye:
        cv.circle(img, (i.x, i.y), 3, lineColor, -1)

    print("right", checkRight, rightBlinkCount, '\n')

    cv.putText(img, "Right Blinks: {}".format(rightBlinkCount), (300, 30), cv.FONT_HERSHEY_SIMPLEX, 0.7, lineColor, 2)
    cv.putText(img, "Eye ratio: {:.2f}".format(checkRight), (300, 80), cv.FONT_HERSHEY_SIMPLEX, 0.7, lineColor, 2)

    # 5. show out result
    cv.namedWindow("blink check", 1)
    cv.imshow("blink check", img)

cap.release()
cv.destroyAllWindows()

at the end

dlib is assigned a number for each point. As long as you can specify the number and operate the point, the code will not be so difficult. I didn't write this time, but the most difficult thing was the introduction of dlib.