From 386c4d38702b2643c09d8db7d7851a39f8798406 Mon Sep 17 00:00:00 2001
From: Luke Shumaker <lukeshu@lukeshu.com>
Date: Thu, 12 Jul 2018 00:41:28 -0400
Subject: Add NUMA support and vCPU pinning

---
 conf.d/qemu.d/beefcake | 25 ++++++++++++++++++++-----
 systemd/system/qemu    | 22 +++++++++++++++++++++-
 2 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/conf.d/qemu.d/beefcake b/conf.d/qemu.d/beefcake
index 6e671bd..49315cb 100644
--- a/conf.d/qemu.d/beefcake
+++ b/conf.d/qemu.d/beefcake
@@ -2,17 +2,16 @@
 
 system=x86_64
 args=(
-	#-nodefaults
 	-no-user-config
 	-no-reboot # let the systemd supervisor know we're restarting
 
 	-machine accel=kvm
-	-m 32768 # 32 GiB RAM
+
+	-m $((30*1024)) # We have 32 GiB RAM on this socket; leave 2GiB for qemu itself
 	# 1 socket / 8 cores-per-node / 2 threads-per-core = total of 16 CPUs
 	-smp sockets=1,cores=8,threads=2,cpus=16
-	# TODO: Split that in to 2 NUMA nodes; 4 cores and 16 GiB RAM each
-
-	#-no-hpet
+	-numa node,cpus=0-7
+	-numa node,cpus=8-15
 
 	-uuid 4c42bd65-6ad5-456d-8b76-758ecb795e21
 	-drive media=disk,file=/dev/vg_build64/lv_beefcake_root,format=raw,if=virtio
@@ -24,3 +23,19 @@ args=(
 
 	-writeconfig /tmp/beefcake.cfg
 )
+
+# numactl=(
+# 	--membind=2,3
+# 	--cpunodebind=2,3
+# )
+# for i in {0..15}; do
+# 	vcpu2cpulist[$i]=$((16+$i))
+# done
+
+numactl=(
+	--membind=0,1
+	--cpunodebind=0,1
+)
+for i in {0..15}; do
+	vcpu2cpulist[$i]=$i
+done
diff --git a/systemd/system/qemu b/systemd/system/qemu
index 475ec10..324154b 100755
--- a/systemd/system/qemu
+++ b/systemd/system/qemu
@@ -2,6 +2,8 @@
 shopt -s lastpipe
 set -euE -o pipefail
 
+declare -A vcpu2cpulist
+numactl=()
 source "/etc/conf.d/qemu.d/$1"
 args+=(
 	-name "$1",debug-threads=on
@@ -11,7 +13,25 @@ args+=(
 	-serial  file:"/dev/stdout"
 	-daemonize
 )
+
 {
-	qemu-system-"${system}" "${args[@]}"
+	numactl "${numactl[@]}" -- qemu-system-"${system}" "${args[@]}"
+
+	if [[ ${#vcpu2cpulist[@]} -gt 0 ]]; then
+		declare -A vcpu2tid
+		while read -r line; do
+			if [[ $line != ?' CPU #'*:*thread_id=* ]]; then
+				continue
+			fi
+			vcpu=${line#?' CPU #'}; vcpu=${vcpu%%:*}
+			tid=${line#thread_id=}; tid=${tid%% *}
+			vcpu2tid[$vcpu]=$tid
+		done < <(socat STDIO UNIX-CONNECT:"/run/qemu-$1/monitor.sock" <<<'info cpus')
+
+		for vcpu in "${!vcpu_pins[@]}"; do
+			taskset --cpu-list --pid -- "${vcpu2cpulist[@]}" "${vcpu2tid[$vcpu]}"
+		done
+	fi
+
 	systemd-notify --ready
 } | exec cat
-- 
cgit v1.2.3-2-g168b