From 799e0edc196c4273b728b9f832d90dc79bbf8bbe Mon Sep 17 00:00:00 2001
From: Luke Shumaker <lukeshu@lukeshu.com>
Date: Sat, 14 Jul 2018 23:34:18 -0400
Subject: Fix vCPU pinning, fix NUMA example

---
 conf.d/qemu.d/beefcake | 24 +++++++++++-------------
 systemd/system/qemu    |  2 +-
 2 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/conf.d/qemu.d/beefcake b/conf.d/qemu.d/beefcake
index 49315cb..21b7e74 100644
--- a/conf.d/qemu.d/beefcake
+++ b/conf.d/qemu.d/beefcake
@@ -7,11 +7,17 @@ args=(
 
 	-machine accel=kvm
 
-	-m $((30*1024)) # We have 32 GiB RAM on this socket; leave 2GiB for qemu itself
-	# 1 socket / 8 cores-per-node / 2 threads-per-core = total of 16 CPUs
-	-smp sockets=1,cores=8,threads=2,cpus=16
-	-numa node,cpus=0-7
-	-numa node,cpus=8-15
+	# RAM: 16 GiB per NUMA node, but leave 1 GiB per node for QEMU itself
+	-m $((30*1024))
+	-object memory-backend-ram,id=ram0,prealloc=on,size=$((15*1024**3)),host-nodes=0,policy=bind
+	-object memory-backend-ram,id=ram1,prealloc=on,size=$((15*1024**3)),host-nodes=1,policy=bind
+	# CPU: 1 socket / 8 cores-per-node / 2 threads-per-core = total of 16 CPUs
+	# Except that it's AMD, and QEMU thinks all threads are Intel HT, so turn off threads
+	# The distance between the two nodes is '16'
+	-smp sockets=1,cores=16,threads=1,cpus=16
+	-numa node,nodeid=0,memdev=ram0,cpus=0-7
+	-numa node,nodeid=1,memdev=ram1,cpus=8-15
+	-numa dist,src=0,dst=1,val=16
 
 	-uuid 4c42bd65-6ad5-456d-8b76-758ecb795e21
 	-drive media=disk,file=/dev/vg_build64/lv_beefcake_root,format=raw,if=virtio
@@ -24,14 +30,6 @@ args=(
 	-writeconfig /tmp/beefcake.cfg
 )
 
-# numactl=(
-# 	--membind=2,3
-# 	--cpunodebind=2,3
-# )
-# for i in {0..15}; do
-# 	vcpu2cpulist[$i]=$((16+$i))
-# done
-
 numactl=(
 	--membind=0,1
 	--cpunodebind=0,1
diff --git a/systemd/system/qemu b/systemd/system/qemu
index 324154b..ad58ae3 100755
--- a/systemd/system/qemu
+++ b/systemd/system/qemu
@@ -17,7 +17,7 @@ args+=(
 {
 	numactl "${numactl[@]}" -- qemu-system-"${system}" "${args[@]}"
 
-	if [[ ${#vcpu2cpulist[@]} -gt 0 ]]; then
+	if [[ ${vcpu2cpulist+${#vcpu2cpulist[@]}} -gt 0 ]]; then
 		declare -A vcpu2tid
 		while read -r line; do
 			if [[ $line != ?' CPU #'*:*thread_id=* ]]; then
-- 
cgit v1.1-4-g5e80