diff options
| author | Luke Shumaker <lukeshu@lukeshu.com> | 2018-07-12 00:41:28 -0400 |
|---|---|---|
| committer | Luke Shumaker <lukeshu@lukeshu.com> | 2018-08-21 00:51:06 -0400 |
| commit | 386c4d38702b2643c09d8db7d7851a39f8798406 (patch) | |
| tree | 3295403d934ffeb6bcbd1837d90b090fc2c676e8 | |
| parent | e3bb3556afb3a612a724095614fade0863a1a667 (diff) | |
Add NUMA support and vCPU pinning
| -rw-r--r-- | conf.d/qemu.d/beefcake | 25 | ||||
| -rwxr-xr-x | systemd/system/qemu | 22 |
2 files changed, 41 insertions, 6 deletions
diff --git a/conf.d/qemu.d/beefcake b/conf.d/qemu.d/beefcake index 6e671bd..49315cb 100644 --- a/conf.d/qemu.d/beefcake +++ b/conf.d/qemu.d/beefcake @@ -2,17 +2,16 @@ system=x86_64 args=( - #-nodefaults -no-user-config -no-reboot # let the systemd supervisor know we're restarting -machine accel=kvm - -m 32768 # 32 GiB RAM + + -m $((30*1024)) # We have 32 GiB RAM on this socket; leave 2GiB for qemu itself # 1 socket / 8 cores-per-node / 2 threads-per-core = total of 16 CPUs -smp sockets=1,cores=8,threads=2,cpus=16 - # TODO: Split that in to 2 NUMA nodes; 4 cores and 16 GiB RAM each - - #-no-hpet + -numa node,cpus=0-7 + -numa node,cpus=8-15 -uuid 4c42bd65-6ad5-456d-8b76-758ecb795e21 -drive media=disk,file=/dev/vg_build64/lv_beefcake_root,format=raw,if=virtio @@ -24,3 +23,19 @@ args=( -writeconfig /tmp/beefcake.cfg ) + +# numactl=( +# --membind=2,3 +# --cpunodebind=2,3 +# ) +# for i in {0..15}; do +# vcpu2cpulist[$i]=$((16+$i)) +# done + +numactl=( + --membind=0,1 + --cpunodebind=0,1 +) +for i in {0..15}; do + vcpu2cpulist[$i]=$i +done diff --git a/systemd/system/qemu b/systemd/system/qemu index 475ec10..324154b 100755 --- a/systemd/system/qemu +++ b/systemd/system/qemu @@ -2,6 +2,8 @@ shopt -s lastpipe set -euE -o pipefail +declare -A vcpu2cpulist +numactl=() source "/etc/conf.d/qemu.d/$1" args+=( -name "$1",debug-threads=on @@ -11,7 +13,25 @@ args+=( -serial file:"/dev/stdout" -daemonize ) + { - qemu-system-"${system}" "${args[@]}" + numactl "${numactl[@]}" -- qemu-system-"${system}" "${args[@]}" + + if [[ ${#vcpu2cpulist[@]} -gt 0 ]]; then + declare -A vcpu2tid + while read -r line; do + if [[ $line != ?' CPU #'*:*thread_id=* ]]; then + continue + fi + vcpu=${line#?' CPU #'}; vcpu=${vcpu%%:*} + tid=${line#thread_id=}; tid=${tid%% *} + vcpu2tid[$vcpu]=$tid + done < <(socat STDIO UNIX-CONNECT:"/run/qemu-$1/monitor.sock" <<<'info cpus') + + for vcpu in "${!vcpu_pins[@]}"; do + taskset --cpu-list --pid -- "${vcpu2cpulist[@]}" "${vcpu2tid[$vcpu]}" + done + fi + systemd-notify --ready } | exec cat |