Impression that scalability-conscious programming and languages such as parallel / parallel processing, functional programming, and reactive programming have recently attracted attention. Therefore, I will summarize the basic writing method of multi-process processing using three types of scripting languages that are often used in Web development.
Vagrantfile
config.vm.provider :virtualbox do |vb|
vb.customize ["modifyvm", :id, "--memory", "1024", "--cpus", "4", "--ioapic", "on"]
end
top -d1
#Press 1 to see all CPUs
Ruby
require "digest/md5"
require "securerandom"
require 'benchmark'
pcount = 4
def single(pcount)
(pcount - 1).times do
arr = []
100000.times do
arr << Digest::MD5.digest(SecureRandom.uuid)
end
end
end
def multi(pcount)
pids = []
(pcount - 1).times do
#Process generation
pids << fork do
arr = []
100000.times do
arr << Digest::MD5.digest(SecureRandom.uuid)
end
end
end
Process.waitall
end
single_time = Benchmark.realtime do
single(pcount)
end
print single_time.to_s() + "\n"
multi_time = Benchmark.realtime do
multi(pcount)
end
print multi_time.to_s() + "\n"
Cpu0 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu1 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu2 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu3 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu0 : 99.0%us, 1.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu1 : 99.0%us, 1.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu2 : 99.0%us, 1.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu3 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
--Single process: 4.066100899999924 --Multi-process: 1.2609145170000602
Python
import multiprocessing as mp
import hashlib
import uuid
import time
import os
def create_hash(res=[]):
print('created: ' + str(os.getpid()))
for _ in range(100000):
res.append(hashlib.md5(str(uuid.uuid4()).encode('utf8')).hexdigest())
return res
def single(pcount):
start_time = time.time()
print('parent: ' + str(os.getpid()))
res = []
for _ in range(pcount):
res = create_hash(res)
end_time = time.time()
print('all finished : ' + str(end_time - start_time))
def multi(pcount):
start_time = time.time()
print('parent: ' + str(os.getpid()))
processes = []
for _ in range(pcount):
processes.append(mp.Process(target=create_hash, args=()))
for process in processes:
process.start()
for process in processes:
process.join()
end_time = time.time()
print('all finished : ' + str(end_time - start_time))
if __name__ == '__main__':
pcount = 4
single(pcount)
multi(pcount)
Cpu0 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu1 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu2 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu3 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu0 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu1 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu2 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu3 : 99.0%us, 1.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
--Single process: 11.234153509140015 --Multi-process: 3.374380588531494
PHP
//Parent process+Child process=4 processes
$pcount = 3;
function single($pcount) {
$time_start = microtime(true);
$res = [];
echo 'parent : '.getmypid()."\n";
foreach(range(0, $pcount - 1) as $i) {
echo 'created : '.getmypid()."\n";
foreach(range(0, 100000) as $ii) {
$res[] = md5(uniqid(mt_rand(), true));
}
}
$time_end = microtime(true);
$time = $time_end - $time_start;
echo "all finished : $time\n";
}
function multi($pcount) {
$processes = [];
$time_start = microtime(true);
echo 'parent : '.getmypid()."\n";
foreach(range(0, $pcount - 1) as $p) {
//Here, it is divided into a parent process and a child process.
//If the parent process successfully spawns a child process, the PID of that process_ID
//If you fail-Get 1
//Child process gets 0
$pid = pcntl_fork();
//Failed to spawn child process
if ($pid === -1) {
echo 'Failed process fork';
exit;
}
//Processing of child processes
if ($pid === 0) {
$res = [];
foreach(range(0, 100000) as $i) {
$res[] = md5(uniqid(mt_rand(), true));
}
echo 'created : '.getmypid()."\n";
exit;
}
//Processing of parent process
$processes[] = $pid;
}
foreach($processes as $process) {
pcntl_waitpid($process, $status);
}
$time_end = microtime(true);
$time = $time_end - $time_start;
echo "all finished : $time\n";
}
single($pcount);
multi($pcount);
Cpu0 :100.0%us, 0.0%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu1 : 0.0%us, 0.0%sy, 0.0%ni,100.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu2 : 0.0%us, 1.0%sy, 0.0%ni, 99.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu3 : 0.0%us, 1.0%sy, 0.0%ni, 99.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu0 : 1.0%us, 0.0%sy, 0.0%ni, 99.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu1 : 99.0%us, 0.0%sy, 0.0%ni, 1.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu2 : 97.0%us, 1.0%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Cpu3 : 96.1%us, 1.0%sy, 0.0%ni, 2.0%id, 0.0%wa, 0.0%hi, 1.0%si, 0.0%st
--Single process: 3.6952289085388 --Multi-process: 1.7085299491882
--There is also a multi-process method for scripting languages. --If the OS is not multi-core, the speed will not be increased even with multi-process processing (obviously). --In an environment with each number of CPU cores, the fastest processing is when the number of processes is the same as the number of CPU cores, and even if the number of processes is increased further, the speed will not be increased (rather slower). --In multi-process, memory is allocated to each process. -Considerations about Elixir and Reactive System and [Message passing that is effective for concurrency](http: // 97 things that programmers should know.com. For an architecture-aware implementation like / essay / message passing useful for concurrency), Ruby is process and Python is gevent seems to be good to use, so investigate separately. PHP has a module called pthreads, but since PHP is originally premised on single thread, it is not enough to force it to work. It seems better to choose the language of.
Recommended Posts