Skip to content

fpm binding master and children processes to specific core(s). #10075

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

fpm binding master and children processes to specific core(s). #10075

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0e9058f
fpm binding master and children processes to specific core(s).
devnexen Dec 9, 2022
21acaa8
build
devnexen Feb 6, 2023
4c04aec
changes from feedback
devnexen May 21, 2023
c910c0e
adding new tests
devnexen May 21, 2023
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion sapi/fpm/config.m4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ PHP_ARG_ENABLE([fpm],,
dnl Configure checks.
AC_DEFUN([AC_FPM_STDLIBS],
[
AC_CHECK_FUNCS(clearenv setproctitle setproctitle_fast)
AC_CHECK_FUNCS(clearenv setproctitle setproctitle_fast cpuset_setaffinity sched_setaffinity)
AC_CHECK_HEADERS([sys/cpuset.h sched.h])

AC_SEARCH_LIBS(socket, socket)
AC_SEARCH_LIBS(inet_addr, nsl)
Expand Down
15 changes: 15 additions & 0 deletions sapi/fpm/fpm/fpm_conf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,9 @@ static const struct ini_value_parser_s ini_fpm_global_options[] = {
{ "process_control_timeout", &fpm_conf_set_time, GO(process_control_timeout) },
{ "process.max", &fpm_conf_set_integer, GO(process_max) },
{ "process.priority", &fpm_conf_set_integer, GO(process_priority) },
#if HAVE_FPM_CPUAFFINITY
{ "process.cpu_list", &fpm_conf_set_string, GO(process_cpu_list) },
#endif
{ "daemonize", &fpm_conf_set_boolean, GO(daemonize) },
{ "rlimit_files", &fpm_conf_set_integer, GO(rlimit_files) },
{ "rlimit_core", &fpm_conf_set_rlimit_core, GO(rlimit_core) },
Expand Down Expand Up @@ -131,6 +134,9 @@ static const struct ini_value_parser_s ini_fpm_pool_options[] = {
#endif
{ "process.priority", &fpm_conf_set_integer, WPO(process_priority) },
{ "process.dumpable", &fpm_conf_set_boolean, WPO(process_dumpable) },
#if HAVE_FPM_CPUAFFINITY
{ "process.cpu_list", &fpm_conf_set_string, WPO(process_cpu_list) },
#endif
{ "pm", &fpm_conf_set_pm, WPO(pm) },
{ "pm.max_children", &fpm_conf_set_integer, WPO(pm_max_children) },
{ "pm.start_servers", &fpm_conf_set_integer, WPO(pm_start_servers) },
Expand Down Expand Up @@ -622,6 +628,9 @@ static void *fpm_worker_pool_config_alloc(void)
wp->config->pm_process_idle_timeout = 10; /* 10s by default */
wp->config->process_priority = 64; /* 64 means unset */
wp->config->process_dumpable = 0;
#if HAVE_FPM_CPUAFFINITY
wp->config->process_cpu_list = NULL;
#endif
wp->config->clear_env = 1;
wp->config->decorate_workers_output = 1;
#ifdef SO_SETFIB
Expand Down Expand Up @@ -1732,6 +1741,9 @@ static void fpm_conf_dump(void)
} else {
zlog(ZLOG_NOTICE, "\tprocess.priority = %d", fpm_global_config.process_priority);
}
#if HAVE_FPM_CPUAFFINITY
zlog(ZLOG_NOTICE, "\tprocess.cpu_list = %s", STR2STR(fpm_global_config.process_cpu_list));
#endif
zlog(ZLOG_NOTICE, "\tdaemonize = %s", BOOL2STR(fpm_global_config.daemonize));
zlog(ZLOG_NOTICE, "\trlimit_files = %d", fpm_global_config.rlimit_files);
zlog(ZLOG_NOTICE, "\trlimit_core = %d", fpm_global_config.rlimit_core);
Expand Down Expand Up @@ -1771,6 +1783,9 @@ static void fpm_conf_dump(void)
zlog(ZLOG_NOTICE, "\tprocess.priority = %d", wp->config->process_priority);
}
zlog(ZLOG_NOTICE, "\tprocess.dumpable = %s", BOOL2STR(wp->config->process_dumpable));
#if HAVE_FPM_CPUAFFINITY
zlog(ZLOG_NOTICE, "\tprocess.cpu_list = %s", STR2STR(wp->config->process_cpu_list));
#endif
zlog(ZLOG_NOTICE, "\tpm = %s", PM2STR(wp->config->pm));
zlog(ZLOG_NOTICE, "\tpm.max_children = %d", wp->config->pm_max_children);
zlog(ZLOG_NOTICE, "\tpm.start_servers = %d", wp->config->pm_start_servers);
Expand Down
6 changes: 6 additions & 0 deletions sapi/fpm/fpm/fpm_conf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ struct fpm_global_config_s {
int systemd_watchdog;
int systemd_interval;
#endif
#if HAVE_FPM_CPUAFFINITY
char *process_cpu_list;
#endif
};

extern struct fpm_global_config_s fpm_global_config;
Expand Down Expand Up @@ -107,6 +110,9 @@ struct fpm_worker_pool_config_s {
#ifdef SO_SETFIB
int listen_setfib;
#endif
#if HAVE_FPM_CPUAFFINITY
char *process_cpu_list;
#endif
};

struct ini_value_parser_s {
Expand Down
13 changes: 13 additions & 0 deletions sapi/fpm/fpm/fpm_config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,3 +75,16 @@
#else
# define HAVE_FPM_LQ 0
#endif

#if defined(HAVE_SCHED_SETAFFINITY) || defined(HAVE_CPUSET_SETAFFINITY)
/*
* to expand to other platforms capable of this granularity (some BSD, solaris, ...).
* macOS is a specific case with an api working fine on intel architecture
* whereas on arm the api and semantic behind is different, since it is about
* Quality Of Service, i.e. binding to a group of cores for high performance
* vs cores for low energy consumption.
*/
# define HAVE_FPM_CPUAFFINITY 1
#else
# define HAVE_FPM_CPUAFFINITY 0
#endif
120 changes: 120 additions & 0 deletions sapi/fpm/fpm/fpm_unix.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <errno.h>
#include <pwd.h>
#include <grp.h>

Expand Down Expand Up @@ -35,6 +36,13 @@
#include <selinux/selinux.h>
#endif

#if defined(HAVE_SCHED_H)
#include <sched.h>
#elif defined(HAVE_SYS_CPUSET_H)
#include <sys/cpuset.h>
typedef cpuset_t cpu_set_t;
#endif

#include "fpm.h"
#include "fpm_conf.h"
#include "fpm_cleanup.h"
Expand Down Expand Up @@ -392,6 +400,101 @@ static int fpm_unix_conf_wp(struct fpm_worker_pool_s *wp) /* {{{ */
}
/* }}} */

#if HAVE_FPM_CPUAFFINITY
static long fpm_cpumax(void)
{
static long cpuid = LONG_MIN;
if (cpuid == LONG_MIN) {
cpu_set_t cset;
#if defined(HAVE_SCHED_SETAFFINITY)
if (sched_getaffinity(0, sizeof(cset), &cset) == 0) {
#elif defined(HAVE_CPUSET_SETAFFINITY)
if (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, sizeof(cset), &cset) == 0) {
#endif
cpuid = CPU_COUNT(&cset);
} else {
cpuid = -1;
}
}

return cpuid;
}

static void fpm_cpuaffinity_init(cpu_set_t *c)
{
CPU_ZERO(c);
}

static void fpm_cpuaffinity_add(cpu_set_t *c, int min, int max)
{
int i;

for (i = min; i <= max; i ++) {
if (!CPU_ISSET(i, c)) {
CPU_SET(i, c);
}
}
}

static int fpm_cpuaffinity_set(cpu_set_t *c)
{
#if defined(HAVE_SCHED_SETAFFINITY)
return sched_setaffinity(0, sizeof(c), c);
#elif defined(HAVE_CPUSET_SETAFFINITY)
return cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, sizeof(c), c);
#endif
}

static int fpm_setcpuaffinity(char *cpu_list)
{
char *token, *buf, *ptr;
cpu_set_t c;
int r, cpumax, min, max;

r = -1;
cpumax = fpm_cpumax();

fpm_cpuaffinity_init(&c);
ptr = estrdup(cpu_list);
token = php_strtok_r(ptr, ",", &buf);

while (token) {
char *cpu_listsep;

if (!isdigit(*token)) {
return -1;
}

min = strtol(token, &cpu_listsep, 0);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is too forgiving and allows latters in the string. For example abbbbb123,1 results in min and max 0 for the first element and 1 for min max second element. Obviously this should be error. I converted this to a small program:

#include <errno.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>

char *php_strtok_r(char *s, const char *delim, char **last)
{
    char *spanp;
    int c, sc;
    char *tok;

    if (s == NULL && (s = *last) == NULL) {
		return NULL;
    }

    /*
     * Skip (span) leading delimiters (s += strspn(s, delim), sort of).
     */
cont:
    c = *s++;
    for (spanp = (char *)delim; (sc = *spanp++) != 0; ) {
		if (c == sc) {
			goto cont;
		}
    }

    if (c == 0)		/* no non-delimiter characters */ {
		*last = NULL;
		return NULL;
    }
    tok = s - 1;

    /*
     * Scan token (scan for delimiters: s += strcspn(s, delim), sort of).
     * Note that delim must have one NUL; we stop if we see that, too.
     */
    for (;;)
    {
		c = *s++;
		spanp = (char *)delim;
		do {
			if ((sc = *spanp++) == c) {
				if (c == 0) {
					s = NULL;
				} else {
					char *w = s - 1;
					*w = '\0';
				}
				*last = s;
				return tok;
			}
		}
		while (sc != 0);
    }
    /* NOTREACHED */
}

int parse(const char *cpu_list_const, int cpumax)
{
    char *token, *buf;
	int min, max;

	int rc = 0;

    char *cpu_list = strdup(cpu_list_const);

    printf("\nparsing %s\n", cpu_list);

	token = php_strtok_r(cpu_list, ",", &buf);

	while (token) {
		char *cpu_listsep;

		min = strtol(token, &cpu_listsep, 0);
		if (errno || min < 0 || min > cpumax) {
			rc = -1;
		}
		max = min;
		if (*cpu_listsep == '-') {
			if (strlen(cpu_listsep) > 1) {
				char *err;
				max = strtol(cpu_listsep + 1, &err, 0);
				if (errno || *err != '\0' || max < min || max > cpumax) {
                    rc = -1;
                    goto end;
				}
			} else {
				rc = -1;
                goto end;
			}
		}

		printf("min: %d, max: %d\n", min, max);

		token = php_strtok_r(NULL, ",", &buf);

    }

end:
    printf("result: %d, cpu list: %s\n", rc, cpu_list);
    free(cpu_list);

	return rc;

}

int main()
{
    parse("1,2,5", 10);
    parse("1-2", 4);
    parse("1-2,4-5,7-10", 12);
    // error cases
    parse("1-8", 4);
    parse("abbbbb123,1", 10);
    parse("123afafkalf,1ffff", 10);
}

which results to

parsing 1,2,5
min: 1, max: 1
min: 2, max: 2
min: 5, max: 5
result: 0, cpu list: 1

parsing 1-2
min: 1, max: 2
result: 0, cpu list: 1-2

parsing 1-2,4-5,7-10
min: 1, max: 2
min: 4, max: 5
min: 7, max: 10
result: 0, cpu list: 1-2

parsing 1-8
result: -1, cpu list: 1-8

parsing abbbbb123,1
min: 0, max: 0
min: 1, max: 1
result: 0, cpu list: abbbbb123

parsing 123afafkalf,1ffff
min: 123, max: 123
min: 1, max: 1
result: -1, cpu list: 123afafkalf

This needs to be fixed.
if (errno || (*cpu_listsep != '\0' && *cpu_listsep != '-') || min < 0 || min > cpumax) {
efree(ptr);
return -1;
}
max = min;
if (*cpu_listsep == '-') {
if (strlen(cpu_listsep) > 1) {
char *err;
max = strtol(cpu_listsep + 1, &err, 0);
if (errno || *err != '\0' || max < min || max > cpumax) {
efree(ptr);
return -1;
}
} else {
efree(ptr);
return -1;
}
}

fpm_cpuaffinity_add(&c, min, max);
token = php_strtok_r(NULL, ",", &buf);
}

r = fpm_cpuaffinity_set(&c);
efree(ptr);
return r;
}
#endif

int fpm_unix_init_child(struct fpm_worker_pool_s *wp) /* {{{ */
{
int is_root = !geteuid();
Expand All @@ -416,6 +519,14 @@ int fpm_unix_init_child(struct fpm_worker_pool_s *wp) /* {{{ */
zlog(ZLOG_SYSERROR, "[pool %s] failed to set rlimit_core for this pool. Please check your system limits or decrease rlimit_core. setrlimit(RLIMIT_CORE, %d)", wp->config->name, wp->config->rlimit_core);
}
}
#if HAVE_FPM_CPUAFFINITY
if (wp->config->process_cpu_list) {
if (0 > fpm_setcpuaffinity(wp->config->process_cpu_list)) {
zlog(ZLOG_SYSERROR, "[pool %s] failed to fpm_setcpuaffinity(%s)", wp->config->name, wp->config->process_cpu_list);
return -1;
}
}
#endif

if (is_root && wp->config->chroot && *wp->config->chroot) {
if (0 > chroot(wp->config->chroot)) {
Expand Down Expand Up @@ -662,6 +773,15 @@ int fpm_unix_init_main(void)
}
}

#if HAVE_FPM_CPUAFFINITY
if (fpm_global_config.process_cpu_list) {
if (0 > fpm_setcpuaffinity(fpm_global_config.process_cpu_list)) {
zlog(ZLOG_SYSERROR, "failed to fpm_setcpuaffinity(%s)", fpm_global_config.process_cpu_list);
return -1;
}
}
#endif

fpm_globals.parent_pid = getpid();
for (wp = fpm_worker_all_pools; wp; wp = wp->next) {
if (0 > fpm_unix_conf_wp(wp)) {
Expand Down
13 changes: 13 additions & 0 deletions sapi/fpm/php-fpm.conf.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,19 @@
; Default value: 10
;systemd_interval = 10

; Bind the master process to a cpu set.
; The value can be one cpu id, a range or a list thereof.
;
; Default Value: not set
; Valid syntaxes are:
; process.cpu_list = "cpu id" - bind master process to cpu id
; process.cpu_list = "[min cpu id]-[max cpu id]"
- bind master process from min cpu id
to max cpu id
; process.cpu_list = "[[min cpu id]-[max cpu id],[min cpu id-max cpu id],...]"
- bind master process to cpu id ranges
separated by a comma

;;;;;;;;;;;;;;;;;;;;
; Pool Definitions ;
;;;;;;;;;;;;;;;;;;;;
Expand Down
43 changes: 43 additions & 0 deletions sapi/fpm/tests/cpuaffinity-fail.phpt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
--TEST--
FPM: cpu affinity test
--SKIPIF--
<?php include "skipif.inc";

if (!str_contains(PHP_OS, 'Linux') && !str_contains(PHP_OS, 'FreeBSD')) {
die('skipped supported only on Linux and FreeBSD');
}
?>
--FILE--
<?php

require_once "tester.inc";

$cfg = <<<EOT
[global]
process.cpu_list = 1000024
error_log = {{FILE:LOG}}
pid = {{FILE:PID}}
[unconfined]
listen = {{ADDR:IPv4}}
pm = dynamic
pm.max_children = 1
pm.start_servers = 1
pm.min_spare_servers = 1
pm.max_spare_servers = 1
EOT;

$tester = new FPM\Tester($cfg);
$tester->start();
$tester->expectLogError("failed to fpm_setcpuaffinity\(\d+\): Inappropriate ioctl for device \(\d+\)");
$tester->expectLogError("FPM initialization failed");
$tester->close();

?>
Done
--EXPECT--
Done
--CLEAN--
<?php
require_once "tester.inc";
FPM\Tester::clean();
?>
42 changes: 42 additions & 0 deletions sapi/fpm/tests/cpuaffinity-fail2.phpt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
--TEST--
FPM: cpu affinity test
--SKIPIF--
<?php include "skipif.inc";

if (!str_contains(PHP_OS, 'Linux') && !str_contains(PHP_OS, 'FreeBSD')) {
die('skipped supported only on Linux and FreeBSD');
}
?>
--FILE--
<?php

require_once "tester.inc";

$cfg = <<<EOT
[global]
process.cpu_list = MYcpuListIs1024-2048
error_log = {{FILE:LOG}}
pid = {{FILE:PID}}
[unconfined]
listen = {{ADDR:IPv4}}
pm = dynamic
pm.max_children = 1
pm.start_servers = 1
pm.min_spare_servers = 1
pm.max_spare_servers = 1
EOT;

$tester = new FPM\Tester($cfg);
$tester->start();
$tester->expectLogError("FPM initialization failed");
$tester->close();

?>
Done
--EXPECT--
Done
--CLEAN--
<?php
require_once "tester.inc";
FPM\Tester::clean();
?>
Loading