Hallo zusammen. @best_not_best. Dieser Artikel ist eine Fortsetzung von First Deep Learning ~ Struggle ~. Wenn Sie es nicht gelesen haben, lesen Sie es bitte zuerst. Es tut mir leid, dass es seit fast einem Jahr veröffentlicht wird ...
Dieser Artikel wurde aus persönlichen Wünschen erstellt und ist nicht die offizielle Ansicht der Organisation, zu der er gehört.
Maschine / Betriebssystem
MacBook Pro (Retina, 15-inch, Mid 2014)
OS X Yosemite 10.10.5
Python
Python-Paket
lxml 3.6.0
selenium 3.0.2
numpy 1.11.2
opencv3 3.1.0 (Installation mit conda.)
tensorflow 0.10.0
Ich verwende Anaconda, um OpenCV mit Python 3.x zu verwenden.
Weitere Informationen zum Prozess finden Sie unter Vorbereitung.
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import lxml.html
from selenium import webdriver
import os
target_url = 'http://hogehoge.co.jp/list.html'
driver = webdriver.PhantomJS(service_log_path = os.path.devnull)
driver.get(target_url)
root = lxml.html.fromstring(driver.page_source)
links = root.cssselect('td.text12m')
for link in links:
if link.text is None:
continue
if link.text.isdigit():
print(link.text)
driver.close()
driver.quit()
Die Mitarbeiter-ID wird in die Standardausgabe ausgegeben. Bitte leiten Sie sie in eine Datei um. Von nun an wird diese Datei als member_id.txt behandelt.
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import os
import urllib.request
import urllib.parse
import time
#Die obige Mitarbeiter-ID-Datei
ID_LIST = '/path/to/member_id.txt'
#URL-Format für Mitarbeiterbilder
URL_FMT = 'http://hogehoge.co.jp/%s.jpg'
#Zielpfadformat für Dateispeicherung
OUTPUT_FMT = '/path/to/photo/%s.jpg'
opener = urllib.request.build_opener()
urllib.request.install_opener(opener)
for id in open(ID_LIST, 'r'):
url = URL_FMT % (id.strip())
try:
img = urllib.request.urlopen(url, timeout=5).read()
if len(img) == 0:
continue
except urllib.request.URLError:
print(url, 'URLError')
except IOError:
print(url, 'IOError')
except UnicodeEncodeError:
print(url, 'EncodeError')
except OSError:
print(url, 'OSError')
else:
output = OUTPUT_FMT % id.strip()
file = open(output, 'wb')
file.write(img)
file.close()
time.sleep(0.1)
Es wird unter folgendem Dateinamen gespeichert.
000001.jpg
000002.jpg
000003.jpg
000004.jpg
000005.jpg
...
Weitere Informationen zum Prozess finden Sie unter Vorbereitung.
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import numpy
import os
import sys
import cv2
#Geben Sie die Definitionsdatei im OpenCV-Paket an
CASCADE_PATH = '/path/to/versions/anaconda3-4.1.1/pkgs/opencv3-3.1.0-py35_0/share/OpenCV/haarcascades/haarcascade_frontalface_alt.xml'
# 1.Verzeichnis gespeichert in
INPUT_DIR_PATH = '/path/to/photos/'
#Speicherverzeichnis für zugeschnittene Bilder
OUTPUT_DIR_PATH = '/path/to/cutout/'
#Format des Bilddateinamens
#Da aus einem Bild mehrere Bilder ausgeschnitten werden können, fügen Sie Seriennummern hinzu.
OUTPUT_FILE_FMT = '%s%s_%d%s'
COLOR = (255, 255, 255)
files = os.listdir(INPUT_DIR_PATH)
for file in files:
input_image_path = INPUT_DIR_PATH + file
#Datei lesen
image = cv2.imread(input_image_path)
#Graustufenumwandlung
try:
image_gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
except cv2.error:
continue
#Ermitteln Sie die Merkmalsmenge des Kaskadenklassifikators
cascade = cv2.CascadeClassifier(CASCADE_PATH)
#Ausführung der Objekterkennung (Gesichtserkennung)
facerect = cascade.detectMultiScale(image_gray, scaleFactor=1.1, minNeighbors=1, minSize=(1, 1))
if len(facerect) > 0:
#Erkennungsergebnisse speichern
i = 1
for rect in facerect:
x = rect[0]
y = rect[1]
w = rect[2]
h = rect[3]
path, ext = os.path.splitext(os.path.basename(file))
output_image_path = OUTPUT_FILE_FMT % (OUTPUT_DIR_PATH, path, i, ext)
try:
im = cv2.resize(image[y:y+h, x:x+w], (96, 96))
cv2.imwrite(output_image_path, im)
except cv2.error:
print(file)
continue
i += 1
Es wird unter folgendem Dateinamen gespeichert.
000001_1.jpg
000002_1.jpg
000003_1.jpg
000003_2.jpg
000004_1.jpg
...
Weitere Informationen zum Prozess finden Sie unter Kampf.
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import sys
import os
import json
import urllib.request
import urllib.parse
import requests
import mimetypes
import re
# API URL
BING_URL = 'https://api.datamarket.azure.com/Bing/Search/Image?'
# API ACCESS KEY
MS_ACCTKEY = 'hogehoge'
QUERY = 'Der Name Ihres Lieblingsstars'
#Speicherverzeichnis der erfassten Bilder
OUTPUT_DIR_PATH = '/path/to/talent/'
opener = urllib.request.build_opener()
urllib.request.install_opener(opener)
def download_urllist(urllist, skip):
for url in urllist:
try:
img = urllib.request.urlopen(url, timeout=5).read()
if len(img) == 0:
continue
url = re.sub(r'\?.*', '', url)
mine_type = mimetypes.guess_type(url)[0]
if mine_type is None:
mine_type = 'jpeg'
else:
mine_type = mine_type.split('/')[1]
file_name = '%s.%s' % (skip, mine_type)
with open(OUTPUT_DIR_PATH + file_name, 'wb') as f:
f.write(img)
except urllib.request.URLError:
print('URLError')
except IOError:
print('IOError')
except UnicodeEncodeError:
print('EncodeError')
except OSError:
print('OSError')
skip += 1
if __name__ == "__main__":
query = urllib.request.quote(QUERY)
step = 20
num = 50
url_param_dict = {
'Query': "'"+QUERY+"'",
'Market': "'ja-JP'",
}
url_param_base = urllib.parse.urlencode(url_param_dict)
url_param_base = url_param_base + '&$format=json&$top=%d&$skip='%(num)
for skip in range(0, num*step, num):
url_param = url_param_base + str(skip)
url = BING_URL + url_param
response = requests.get(url,
auth=(MS_ACCTKEY, MS_ACCTKEY),
headers={'User-Agent': 'My API Robot'})
response = response.json()
urllist = [item['MediaUrl'] for item in response['d']['results']]
download_urllist(urllist, skip)
Weitere Informationen zum Prozess finden Sie unter Kampf.
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import numpy
import os
import sys
import cv2
#Geben Sie die Definitionsdatei im OpenCV-Paket an
CASCADE_PATH = '/path/to/versions/anaconda3-4.1.1/pkgs/opencv3-3.1.0-py35_0/share/OpenCV/haarcascades/haarcascade_frontalface_alt.xml'
# 3.Verzeichnis gespeichert in
INPUT_DIR_PATH = '/path/to/talent/'
#Speicherverzeichnis für zugeschnittene Bilder
OUTPUT_DIR_PATH = '/path/to/talent_cutout/'
#Format des Bilddateinamens
#Da aus einem Bild mehrere Bilder ausgeschnitten werden können, fügen Sie Seriennummern hinzu.
OUTPUT_FILE_FMT = '%s%s_%d%s'
COLOR = (255, 255, 255)
files = os.listdir(INPUT_DIR_PATH)
for file in files:
input_image_path = INPUT_DIR_PATH + file
#Datei lesen
image = cv2.imread(input_image_path)
#Graustufenumwandlung
try:
image_gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
except cv2.error:
continue
#Ermitteln Sie die Merkmalsmenge des Kaskadenklassifikators
cascade = cv2.CascadeClassifier(CASCADE_PATH)
#Ausführung der Objekterkennung (Gesichtserkennung)
facerect = cascade.detectMultiScale(image_gray, scaleFactor=1.1, minNeighbors=1, minSize=(1, 1))
if len(facerect) > 0:
#Erkennungsergebnisse speichern
i = 1
for rect in facerect:
x = rect[0]
y = rect[1]
w = rect[2]
h = rect[3]
path, ext = os.path.splitext(os.path.basename(file))
output_image_path = OUTPUT_FILE_FMT % (OUTPUT_DIR_PATH, count, i, ext)
try:
im = cv2.resize(image[y:y+h, x:x+w], (96, 96))
cv2.imwrite(output_image_path, im)
except cv2.error:
print(file)
continue
i += 1
Es wird unter folgendem Dateinamen gespeichert.
7_3.jpeg
6_1.jpeg
4_1.jpeg
3_1.jpeg
2_1.jpeg
...
Ändern Sie die QUERY und OUTPUT_DIR_PATH des Programms in 3., um es auszuführen.
Dieses Mal habe ich versucht, es mit "QUERY" namens "general public" auszuführen.
QUERY = 'Gewöhnliche Leute'
OUTPUT_DIR_PATH = '/path/to/other_talent/'
Ich werde es weglassen, weil es der gleiche Prozess wie 4 ist.
Erstellen Sie einen Datensatz. Beschriften Sie die Bilddatei Ihres Lieblingsstars mit "1" und sortieren Sie sie nach dem Zufallsprinzip.
$ ls -la /path/to/talent_cutout/*.* | awk '{print $9" 1"}' | gsort -R > talent.txt
Teilen Sie 80% in Trainingsdaten und 20% in Testdaten. (Das Folgende ist in 752 und 189 unterteilt, da die Gesamtzahl der Dateien 941 betrug.)
$ head -752 talent.txt > talent_train.txt
$ tail -189 talent.txt > talent_test.txt
In ähnlicher Weise werden auch andere Bilder als Ihre Lieblingsstars als "2" gekennzeichnet und in Trainingsdaten (commons_train.txt) und Testdaten (commons_test.txt) unterteilt. Alle Trainingsdaten und Testdaten werden kombiniert und zufällig sortiert.
$ cat commons_train.txt talent_train.txt | gsort -R > train.txt
$ cat commons_test.txt talent_test.txt | gsort -R > test.txt
Der Inhalt der Datei ist wie folgt.
$ head -5 train.txt
/path/to/other_talent_cutout/152_16.jpeg 2
/path/to/talent_cutout/371_1.jpg 1
/path/to/talent_cutout/349_1.jpg 1
/path/to/talent_cutout/523_2.jpg 1
/path/to/other_talent_cutout/348_2.jpeg 2
Lass Tensorflow lernen. TensorFlow Um aus einer großen Anzahl von Bildern zu lernen ... ~ (fast) Lösung ~ --Qiita wurde als Referenz verwendet.
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import sys
import cv2
import numpy as np
import tensorflow as tf
import tensorflow.python.platform
NUM_CLASSES = 3
IMAGE_SIZE = 28
IMAGE_PIXELS = IMAGE_SIZE * IMAGE_SIZE * 3
flags = tf.app.flags
FLAGS = flags.FLAGS
#Pfad der Datei zum Speichern des Lernergebnisses
flags.DEFINE_string('save_model', '/path/to/model.ckpt', 'File name of model data')
#Trainingsdatenpfad
flags.DEFINE_string('train', '/path/to/train.txt', 'File name of train data.')
#Datenpfad testen
flags.DEFINE_string('test', '/path/to/test.txt', 'File name of test data.')
flags.DEFINE_string('train_dir', './log_data', 'Directory to put the training data.')
flags.DEFINE_integer('max_steps', 100, 'Number of steps to run trainer.')
flags.DEFINE_integer(
'batch_size',
10,
'Batch size'
'Must divide evenly into the dataset sizes.'
)
flags.DEFINE_float('learning_rate', 1e-4, 'Initial learning rate.')
def inference(images_placeholder, keep_prob):
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(
x,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME'
)
x_images = tf.reshape(images_placeholder, [-1, IMAGE_SIZE, IMAGE_SIZE, 3])
with tf.name_scope('conv1') as scope:
W_conv1 = weight_variable([5, 5, 3, 32])
b_conv1 = bias_variable([32])
h_conv1 = tf.nn.relu(conv2d(x_images, W_conv1) + b_conv1)
with tf.name_scope('pool1') as scope:
h_pool1 = max_pool_2x2(h_conv1)
with tf.name_scope('conv2') as scope:
W_conv2 = weight_variable([5, 5, 32, 64])
b_conv2 = bias_variable([64])
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
with tf.name_scope('pool2') as scope:
h_pool2 = max_pool_2x2(h_conv2)
with tf.name_scope('fc1') as scope:
W_fc1 = weight_variable([7 * 7 * 64, 1024])
b_fc1 = bias_variable([1024])
h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
with tf.name_scope('fc2') as scope:
W_fc2 = weight_variable([1024, NUM_CLASSES])
b_fc2 = bias_variable([NUM_CLASSES])
with tf.name_scope('softmax') as scope:
y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)
return y_conv
def loss(logits, labels):
cross_entropy = -tf.reduce_sum(labels*tf.log(tf.clip_by_value(logits, 1e-10, 1.0)))
tf.scalar_summary('cross_entropy', cross_entropy)
return cross_entropy
def training(loss, learning_rate):
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
return train_step
def accuracy(logits, labels):
correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(labels, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
tf.scalar_summary('accuracy', accuracy)
return accuracy
if __name__ == '__main__':
with open(FLAGS.train, 'r') as f: # train.txt
train_image = []
train_label = []
for line in f:
line = line.rstrip()
l = line.split()
img = cv2.imread(l[0])
img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE))
train_image.append(img.flatten().astype(np.float32) / 255.0)
tmp = np.zeros(NUM_CLASSES)
tmp[int(l[1])] = 1
train_label.append(tmp)
train_image = np.asarray(train_image)
train_label = np.asarray(train_label)
train_len = len(train_image)
with open(FLAGS.test, 'r') as f:
test_image = []
test_label = []
for line in f:
line = line.rstrip()
l = line.split()
img = cv2.imread(l[0])
img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE))
test_image.append(img.flatten().astype(np.float32) / 255.0)
tmp = np.zeros(NUM_CLASSES)
tmp[int(l[1])] = 1
test_label.append(tmp)
test_image = np.asarray(test_image)
test_label = np.asarray(test_label)
test_len = len(test_image)
with tf.Graph().as_default():
images_placeholder = tf.placeholder('float', shape=(None, IMAGE_PIXELS))
labels_placeholder = tf.placeholder('float', shape=(None, NUM_CLASSES))
keep_prob = tf.placeholder('float')
logits = inference(images_placeholder, keep_prob)
loss_value = loss(logits, labels_placeholder)
train_op = training(loss_value, FLAGS.learning_rate)
acc = accuracy(logits, labels_placeholder)
saver = tf.train.Saver()
sess = tf.Session()
sess.run(tf.initialize_all_variables())
summary_op = tf.merge_all_summaries()
summary_writer = tf.train.SummaryWriter(FLAGS.train_dir, sess.graph_def)
if train_len % FLAGS.batch_size is 0:
train_batch = train_len / FLAGS.batch_size
else:
train_batch = (train_len / FLAGS.batch_size) + 1
print('train_batch = ' + str(train_batch))
for step in range(FLAGS.max_steps):
for i in range(int(train_batch)):
batch = FLAGS.batch_size * i
batch_plus = FLAGS.batch_size * (i + 1)
if batch_plus > train_len:
batch_plus = train_len
sess.run(train_op, feed_dict={
images_placeholder: train_image[batch: batch_plus],
labels_placeholder: train_label[batch: batch_plus],
keep_prob: 0.5
})
if step % 10 == 0:
train_accuracy = 0.0
for i in range(int(train_batch)):
batch = FLAGS.batch_size * i
batch_plus = FLAGS.batch_size * (i + 1)
if batch_plus > train_len: batch_plus = train_len
train_accuracy += sess.run(acc, feed_dict={
images_placeholder: train_image[batch: batch_plus],
labels_placeholder: train_label[batch: batch_plus],
keep_prob: 1.0})
if i is not 0: train_accuracy /= 2.0
print('step %d, training accuracy %g' % (step, train_accuracy))
if test_len % FLAGS.batch_size is 0:
test_batch = test_len / FLAGS.batch_size
else:
test_batch = (test_len / FLAGS.batch_size) + 1
print('test_batch = ' + str(test_batch))
test_accuracy = 0.0
for i in range(int(test_batch)):
batch = FLAGS.batch_size * i
batch_plus = FLAGS.batch_size * (i + 1)
if batch_plus > train_len:
batch_plus = train_len
test_accuracy += sess.run(
acc,
feed_dict={
images_placeholder: test_image[batch:batch_plus],
labels_placeholder: test_label[batch:batch_plus],
keep_prob: 1.0
}
)
if i is not 0:
test_accuracy /= 2.0
print('test accuracy %g' % (test_accuracy))
save_path = saver.save(sess, FLAGS.save_model)
Das Trainingsergebnis wird in / path / to / model.ckpt gespeichert.
Wieder bezog ich mich auf TensorFlow, um aus einer großen Anzahl von Bildern zu lernen ... ~ (fast) Lösung ~ --Qiita.
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import os
import sys
import numpy as np
import tensorflow as tf
import cv2
import tensorflow.python.platform
from types import *
NUM_CLASSES = 3
IMAGE_SIZE = 28
IMAGE_PIXELS = IMAGE_SIZE * IMAGE_SIZE * 3
# 2.Speicherverzeichnis des beschnittenen Bildes mit
DIR_PATH = '/path/to/cutout/'
flags = tf.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_string('readmodels', '/path/to/model.ckpt', 'File name of model data')
def inference(images_placeholder, keep_prob):
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(
x,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME'
)
x_image = tf.reshape(images_placeholder, [-1, IMAGE_SIZE, IMAGE_SIZE, 3])
with tf.name_scope('conv1') as scope:
W_conv1 = weight_variable([5, 5, 3, 32])
b_conv1 = bias_variable([32])
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
with tf.name_scope('pool1') as scope:
h_pool1 = max_pool_2x2(h_conv1)
with tf.name_scope('conv2') as scope:
W_conv2 = weight_variable([5, 5, 32, 64])
b_conv2 = bias_variable([64])
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
with tf.name_scope('pool2') as scope:
h_pool2 = max_pool_2x2(h_conv2)
with tf.name_scope('fc1') as scope:
W_fc1 = weight_variable([7 * 7 * 64, 1024])
b_fc1 = bias_variable([1024])
h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
with tf.name_scope('fc2') as scope:
W_fc2 = weight_variable([1024, NUM_CLASSES])
b_fc2 = bias_variable([NUM_CLASSES])
with tf.name_scope('softmax') as scope:
y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)
return y_conv
if __name__ == '__main__':
test_image = []
test_image_name = []
files = os.listdir(DIR_PATH)
for file in files:
if file == '.DS_Store':
continue
img = cv2.imread(DIR_PATH + file)
img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE))
test_image.append(img.flatten().astype(np.float32) / 255.0)
test_image_name.append(file)
test_image = np.asarray(test_image)
images_placeholder = tf.placeholder('float', shape=(None, IMAGE_PIXELS))
labels_placeholder = tf.placeholder('float', shape=(None, NUM_CLASSES))
keep_prob = tf.placeholder('float')
logits = inference(images_placeholder, keep_prob)
sess = tf.InteractiveSession()
saver = tf.train.Saver()
sess.run(tf.initialize_all_variables())
saver.restore(sess,FLAGS.readmodels)
for i in range(len(test_image)):
pr = logits.eval(feed_dict={
images_placeholder: [test_image[i]],
keep_prob: 1.0
})[0]
pred = np.argmax(pr)
if pred == 1:
#Als Lieblingsunterhalter beurteilt
print('%s,%f' % (test_image_name[i], pr[pred] * 100.0))
Das Ergebnis wird in die Standardausgabe ausgegeben. Bitte leiten Sie es entsprechend in eine Datei um. Die Ergebnisse werden in absteigender Reihenfolge nach Punktzahl und Ausgabe sortiert, um sie zu vervollständigen!
$ cat result.csv | sort -r -t, -k 2 | head -5
1xxxx1_1.jpg,0.5406388165003011
1xxxx1_2.jpg,0.5350551152698707
1xxxx6_1.jpg,0.5310078821076752
1xxxx2_1.jpg,0.5183026050695199
1xxxx0_1.jpg,0.5130400958800978
Ich bin mir nicht sicher, aber wir haben es als Beispiel für unsere Bemühungen in unserem Unterausschuss vorgestellt.
Recommended Posts