I was surprised when I sent Facerank from iPad via udp and relayed, received, and displayed it in Python!
I was surprised when I sent Facerank from iPad via udp and relayed, received, and displayed it in Python!
What surprised me was that I could get the results I wanted with a short code. The figure below shows the screen (real-time video) displayed on the mac by generating the landmark data of 68 points of dlib with the iPad application and relaying it on the server via the Internet. The Python code for this display is [Code] 1] As shown, there are only about 30 lines. It's obvious to Pythonists, but it's amazing to beginners. I felt that it was the strongest for prototyping ideas.
Send Facemarks from iPad via udp
The side that creates face landmarks data from the face image of the camera and sends it with udp, I used the following git Xcode project. (To the same author's git, Android version, There is also a Javascript version. )
https://github.com/Tastenkunst/brfv4_ios_examples
Change track_single_face.hpp in it as follows.
addr.sin_port = htons(5000); addr.sin_addr.s_addr = inet_addr("XXX.XX.XXX.XXX");
With port number 5000 The IP of "XXX.XX.XXX.XXX" matches your settings. When you build and launch an Xcode project and take a picture of your face, it will send the position data of 68 face landmarks to the server as 68x2 bytes of data.
track_single_face.hpp
#ifndef __brf__cpp__BRFCppExample_hpp
#define __brf__cpp__BRFCppExample_hpp
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
namespace brf {
class BRFCppExample: public BRFBasicCppExample {
int sock;
struct sockaddr_in addr;
uint16_t shorts;
uint8_t bytes[sizeof(uint8_t)*136*2];
public: BRFCppExample() : BRFBasicCppExample()
{
addr.sin_family = AF_INET;
addr.sin_port = htons(5000);
addr.sin_addr.s_addr = inet_addr("XXX.XX.XXX.XXX");
//sock = socket(AF_INET, SOCK_DGRAM, 0);
}
public: void initCurrentExample(brf::BRFManager& brfManager, brf::Rectangle& resolution) {
brf::trace("BRFv4 - basic - face tracking - track single face" + brf::to_string("\n")+
"Detect and track one face and draw the 68 facial landmarks.");
}
public: void updateCurrentExample(brf::BRFManager& brfManager, brf::DrawingUtils& draw) {
// In a webcam example imageData is the mirrored webcam video feed.
// In an image example imageData is the (not mirrored) image content.
brfManager.update();
// Drawing the results:
draw.clear();
// Face detection results: a rough rectangle used to start the face tracking.
//draw.drawRects(brfManager.getAllDetectedFaces(), false, 1.0, 0x00a1ff, 0.5);
//draw.drawRects(brfManager.getMergedDetectedFaces(), false, 2.0, 0xffd200, 1.0);
// Get all faces. The default setup only tracks one face.
std::vector< std::shared_ptr<brf::BRFFace> >& faces = brfManager.getFaces();
for(size_t i = 0; i < faces.size(); i++) {
brf::BRFFace& face = *faces[i];
if( face.state == brf::BRFState::FACE_TRACKING_START ||
face.state == brf::BRFState::FACE_TRACKING) {
sock = socket(AF_INET, SOCK_DGRAM, 0);
for(int i = 0; i < 136; i++){
shorts = (int)face.vertices[i];
bytes[i * 2] = (uint8_t) (shorts & 0xFF);
bytes[(i * 2) + 1] = (uint8_t) (shorts >> 8);
}
sendto(sock, bytes, sizeof(bytes), 0, (struct sockaddr *)&addr, sizeof(addr));
close(sock);
//draw.drawTriangles( face.vertices, face.triangles, false, 1.0, 0x00a0ff, 0.4);
draw.drawVertices( face.vertices, 2.0, false, 0x00a0ff, 0.4);
}
}
}
};
}
#endif // __brf__cpp__BRFCppExample_hpp
Server side
Next is the server side. It is a code that only sends landmark data to the LAN side. The port number and IP are adjusted to your environment. Writing in Python makes it simple to write on the server side.
sendBak.py
import socket
import time
print (time.ctime(),flush=True)
host = '192.168.10.101'
port = 5001
bufsize = 512
portFace = 5000
sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
#sock.settimeout(0.01)
sock.bind((host,port))
#
sock2 = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
sock2.settimeout(30.0)
sock2.bind((host,portFace))
while True:
global data, addr
string,addr = sock.recvfrom(bufsize)
print("from:",addr,time.ctime())
while True:
try:
data, (host, port) = sock2.recvfrom(bufsize)
result = sock.sendto(data,addr)
#if result != 272:
# print(result)
except socket.error as e:
#print ('Error: %s' % e)
break
OpenCV version [Code 1]
Finally, the receiving side. It doesn't matter if it's a Python environment, Mac, Windows, or Linux. You can also use Pythoinista on iOS.
FaceLineOpenCV.py
import cv2
import socket
import numpy as np
from struct import unpack
from datetime import datetime
host = 'XXXX.com'
port = 5001
message = b'hello'
if __name__ == "__main__" :
server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
#server.bind(('192.168.2.100', 5000))
width = 600
height = 800
img = np.zeros((height, width, 3), np.uint8)
server.sendto(message, (host, port))
while True:
data, addr = server.recvfrom(1024)
cv2.rectangle(img, (0, 0), (600, 700), color=(255, 0, 0), thickness=-1)
val = unpack('<'+'H'*(len(data)//2), data)
j = 0
for i in range(64):
cv2.circle(img, (int(val[j]), int(val[j+1])), 4, (255, 255, 256), -1)
j = j + 2
cv2.imshow('camera capture', img)
k = cv2.waitKey(1) #
if k == 27:
break
cv2.destroyAllWindows()
The above is the OpenCV version, but it also works with PyQt as shown below.
FaceLineQt.py
import sys
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtCore import QTimer
import socket
import numpy as np
from struct import unpack
host = 'XXXX.com'
port = 5001
bufsize = 512
message = b'hello'
class Widget(QWidget):
def __init__(self, parent=None):
super(Widget, self).__init__(parent)
self.timer = QTimer(self)
self.timer.timeout.connect(self.update)
self.timer.start(20)#update
#
self.server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
#self.server.settimeout(0.5)
#self.server.setblocking(False)
#self.img = np.zeros((height, width, 3), np.uint8)
self.server.sendto(message, (host, port))
def paintEvent(self, event):
data, addr = self.server.recvfrom(bufsize)
val = unpack('<'+'H'*(len(data)//2), data)
painter = QPainter(self)
painter.setPen(Qt.red)
painter.setBrush(Qt.red)
j = 0
for i in range(68):
painter.drawRect(int(val[j]), int(val[j+1]), 5, 5)
j = j + 2
def main():
app = QApplication(sys.argv)
w = Widget()
w.show()
w.raise_()
app.exec_()
if __name__ == '__main__':
main()
I'll add a code description later.
Added Pythonista version. For Pythonista, use a fast scene.
FalceLinePythonista.py
from scene import *
import socket
import numpy as np
from struct import unpack
host = 'XXXX.com'
port = 5001
bufsize = 512
class FaceLine (Scene):
def setup(self):
self.landmarks = []
for i in range(68):
shape = ShapeNode(ui.Path.oval(0,0,5,5), 'white')
self.landmarks.append(shape)
self.add_child(shape)
self.server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
#
message = b'hello'
self.server.sendto(message, (host, port))
def update(self):
#
data, addr = self.server.recvfrom(bufsize)
val = unpack('<'+'H'*(len(data)//2), data)
dtx = np.array(val[::2])
dty = np.array(val[1::2])
for i in range(68):
self.landmarks[i].position = (dtx[i]+200,800-dty[i])
run(FaceLine())
Recommended Posts