Checking differences due to Linux scheduling policy

Scheduling policies can be set at the time of thread creation and after thread creation in the Linux user process. It was easy to see how the processing order changes depending on the scheduling policy.

Check with the following scheduling policy that can be specified in the user process

SCHED_FIFO FIFO real-time scheduling
SCHED_RR Round robin real-time scheduling
SCHED_OTHER Normal scheduling (not real time)

Measurement environment

Raspberry Pi 3 (ARM 1400MHz * 4 core) (Ubuntu MATE 18.04)
Measure by launching 16 threads with the same load

Measurement result

Since it was difficult to check the processing timing with the text log, visualize the log with the timeline chart The CPU time was about the same for both threads.

SCHED_FIFO (High priority: 3 threads / Medium priority: 10 threads / Low priority: 3 threads)

Process in descending order of priority.
In the case of the same priority, the thread once assigned continues processing unless there is an interrupt or block processing of high priority processing.

SCHED_RR (High priority: 3 threads / Medium priority: 10 threads / Low priority: 3 threads)

Process in descending order of priority.
If the priority is the same, the CPU will be handed over after the period by the round robin method.
I'm worried that the thread with the highest priority (priority = 10) that started operating has not been handed over.

SCHED_RR (Low priority: 16 threads)

If the priority is the same, the processing will be handed over after the period by the round robin method.
Although it is a round robin method, I am worried that the processing end timing varies considerably depending on the thread.

SCHED_OTHER

Normal scheduling.
Alternate evenly processed by timeshare scheduling.

SCHED_FIFO and SCHED_OTHER (FIFO: 3 threads / OTHER: 11 threads)

A mixture of real-time scheduling and regular scheduling.
Since it was measured in a 4-core environment, the CPU is allocated and processed with priority for 3 threads of real-time scheduling. 11 threads share one free core with normal scheduling.

Source code

The source code used for confirmation is as follows.

#include <stdio.h>
#include <stdlib.h>
#include <pthread.h>
#include <unistd.h>
#include <sys/times.h>
#include <vector>
using namespace std;

//timespec difference calculation function
inline void sub_timespec(struct timespec* res, const struct timespec* a, const struct timespec* b)
{
	if (a->tv_nsec >= b->tv_nsec) {
		res->tv_nsec = a->tv_nsec - b->tv_nsec;
		res->tv_sec = a->tv_sec - b->tv_sec;
	} else {
		res->tv_nsec = 1000000000 + a->tv_nsec - b->tv_nsec;
		res->tv_sec = a->tv_sec - b->tv_sec - 1;
	}
}

class PriorityThread
{
public:
	PriorityThread(const char* name, int policy, int priority) {
		this->name = name;
		this->policy = policy;
		this->priority = priority;
	}
	bool create() {
		int ret;
		pthread_attr_t attr;
		//Initialization of thread attribute object
		ret = pthread_attr_init(&attr);
		if (ret != 0) {
			fprintf(stderr, "PriorityThread::create() : pthread_attr_init() ret=%d\n", ret);
			return false;
		}
		//Thread attribute scheduling inheritance: not inherited
		ret = pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED);
		if (ret != 0) {
			fprintf(stderr, "PriorityThread::create() : pthread_attr_setinheritsched() ret=%d\n", ret);
			return false;
		}
		//Thread attribute scheduling policy settings
		ret = pthread_attr_setschedpolicy(&attr, this->policy);
		if (ret != 0) {
			fprintf(stderr, "PriorityThread::create() : pthread_attr_setschedpolicy(policy=%d) ret=%d\n", this->policy, ret);
			return false;
		}
		//Setting thread attribute scheduling parameters
		struct sched_param param;
		param.sched_priority = this->priority;
		ret = pthread_attr_setschedparam(&attr, &param);
		if (ret != 0) {
			fprintf(stderr, "PriorityThread::create() : pthread_attr_setschedparam(priority=%d) ret=%d\n", this->priority, ret);
			return false;
		}
		//Thread creation
		ret = pthread_create(&this->tid, &attr, PriorityThread::_run, (void*)this);
		if (ret != 0) {
			fprintf(stderr, "PriorityThread::create() : pthread_create() ret=%d\n", ret);
			return false;
		}
		//Discard thread attribute object
		pthread_attr_destroy(&attr);
		
		return true;
	}
	bool join() {
		int ret;
		ret = pthread_join(this->tid, NULL);
		if (ret != 0) {
			fprintf(stderr, "PriorityThread::join() : pthread_join() ret=%d\n", ret);
			return false;
		}
		return true;
	}
	//High load processing for measurement(It was troublesome, so repeat the for sentence)
	virtual void* run() {
		int64_t ans = 0;
		for (int64_t i = 0; i <= 1000000000; i++) ans += i;
	}
	//Use mutex to control thread execution start
	static bool lockRun() {
		pthread_mutex_lock(&PriorityThread::g_thread_block);
	}
	static bool unlockRun() {
		pthread_mutex_unlock(&PriorityThread::g_thread_block);
	}
	const char* getName() {
		return name;
	}
	void printlog() {
		struct timespec elapsed;
		printf("  %12s", name);
		sub_timespec(&elapsed, &etime_moni, &stime_moni);
		printf(": time(%ld.%09ld, %ld.%09ld, %ld.%09ld)", stime_moni.tv_sec, stime_moni.tv_nsec,
			etime_moni.tv_sec, etime_moni.tv_nsec,elapsed.tv_sec, elapsed.tv_nsec);
		sub_timespec(&elapsed, &etime_tcpu, &stime_tcpu);
		printf(": cpu(%ld.%09ld, %ld.%09ld, %ld.%09ld)", stime_tcpu.tv_sec, stime_tcpu.tv_nsec,
			etime_tcpu.tv_sec, etime_tcpu.tv_nsec,elapsed.tv_sec, elapsed.tv_nsec);
		printf("\n");
	}
private:
	static void* _run(void* param) {
		PriorityThread* obj = ((PriorityThread*)param);
		//Waiting for unlock
		pthread_mutex_lock(&PriorityThread::g_thread_block);
		pthread_mutex_unlock(&PriorityThread::g_thread_block);
		//Processing start time measurement
		clock_gettime(CLOCK_MONOTONIC, &obj->stime_moni);
		clock_gettime(CLOCK_THREAD_CPUTIME_ID, &obj->stime_tcpu);
		//Processing execution
		void* ret = ((PriorityThread*)param)->run();
		//Processing end time measurement
		clock_gettime(CLOCK_MONOTONIC, &obj->etime_moni);
		clock_gettime(CLOCK_THREAD_CPUTIME_ID, &obj->etime_tcpu);
		
		return ret;
	}
private:
	static pthread_mutex_t g_thread_block;
	const char* name;
	int policy;
	int priority;
	pthread_t tid;
	struct timespec stime_moni;
	struct timespec stime_tcpu;
	struct timespec etime_moni;
	struct timespec etime_tcpu;
};
pthread_mutex_t PriorityThread::g_thread_block = PTHREAD_MUTEX_INITIALIZER;

int main(void)
{
	bool result;
	vector<PriorityThread*>  threads;
	
	//Thread definition
	threads.push_back(new PriorityThread("Thread_0", SCHED_FIFO , 50));
	threads.push_back(new PriorityThread("Thread_1", SCHED_FIFO , 50));
	threads.push_back(new PriorityThread("Thread_2", SCHED_FIFO , 50));
	threads.push_back(new PriorityThread("Thread_3", SCHED_FIFO , 5));
	threads.push_back(new PriorityThread("Thread_4", SCHED_FIFO , 5));
	threads.push_back(new PriorityThread("Thread_5", SCHED_FIFO , 5));
	threads.push_back(new PriorityThread("Thread_6", SCHED_FIFO , 10));
	threads.push_back(new PriorityThread("Thread_7", SCHED_FIFO , 10));
	threads.push_back(new PriorityThread("Thread_8", SCHED_FIFO , 10));
	threads.push_back(new PriorityThread("Thread_9", SCHED_FIFO , 10));
	threads.push_back(new PriorityThread("Thread_A", SCHED_FIFO , 10));
	threads.push_back(new PriorityThread("Thread_B", SCHED_FIFO , 10));
	threads.push_back(new PriorityThread("Thread_C", SCHED_FIFO , 10));
	threads.push_back(new PriorityThread("Thread_D", SCHED_FIFO , 10));
	threads.push_back(new PriorityThread("Thread_E", SCHED_FIFO , 10));
	threads.push_back(new PriorityThread("Thread_F", SCHED_FIFO , 10));
	
	//Lock thread execution start
	PriorityThread::lockRun();

	//Creating a thread
	printf("Thread Create.\n");
	for (auto item : threads) {
		result = item->create();
		if (!result) {
			fprintf(stderr, "Create Error: name=%s\n", item->getName());
			exit(1);
		}
	}
	printf("Thread Create done.\n");
	//Unlock thread execution start
	sleep(1);
	printf("Thread Run.\n");
	PriorityThread::unlockRun();
	
	//Waiting for thread to end
	for (auto item : threads) {
		result = item->join();
		if (!result) {
			fprintf(stderr, "Thread Join Error: name=%s\n", item->getName());
			exit(1);
		}
	}
	printf("Thread Join done.\n");

	//Thread execution time log output
	for (auto item : threads) {
		item->printlog();
	}

	//Discard thread information
	for (auto item : threads) {
		delete item;
	}
	threads.clear();
}

When checking each scheduling policy, The parameter was rewritten when the PriorityThread () class was generated and measured.

If you want to grant the SCHED_FIFO or SCHED_RR attributes, you need root privileges at run time. Please give authority with sudo etc. and execute.

Reference site

https://access.redhat.com/documentation/ja-jp/red_hat_enterprise_linux/6/html/performance_tuning_guide/s-cpu-scheduler https://linuxjm.osdn.jp/html/glibc-linuxthreads/man3/pthread_attr_init.3.html

Check for differences due to Linux scheduling policies (SCHED_FIFO / SCHED_RR / SCHED_OTHER)