Ceph Crimson OSD CPU 코어 할당에 대한 성능 조사. 1부

# ./do_cmake.sh -DWITH_SEASTAR=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo

/root/bin/run_balanced_crimson.sh -t cyan

crimson_be_table["cyan"]="--cyanstore"
crimson_be_table["sea"]="--seastore --seastore-devs ${STORE_DEVS}"
crimson_be_table["blue"]="--bluestore --bluestore-devs ${STORE_DEVS}"

cyan_8osd_5reactor_8fio_bal_osd_rc_1procs_randread.zip      cyan_8osd_5reactor_8fio_bal_socket_rc_1procs_seqread.zip   cyan_8osd_6reactor_8fio_bal_osd_rc_1procs_randread.zip      cyan_8osd_6reactor_8fio_bal_socket_rc_1procs_seqread.zip
cyan_8osd_5reactor_8fio_bal_osd_rc_1procs_randwrite.zip     cyan_8osd_5reactor_8fio_bal_socket_rc_1procs_seqwrite.zip  cyan_8osd_6reactor_8fio_bal_osd_rc_1procs_randwrite.zip     cyan_8osd_6reactor_8fio_bal_socket_rc_1procs_seqwrite.zip
cyan_8osd_5reactor_8fio_bal_osd_rc_1procs_seqread.zip

# /root/bin/pp_balanced_cpu_cmp.sh -d /tmp/_seastore_8osd_5_6_reactor_8fio_rc_cmp \
   -t sea -o seastore_8osd_5vs6_reactor_8fio_cpu_cmp.md

# jc --pretty /proc/diskstats > /tmp/blue_8osd_6reactor_192at_8fio_socket_cond.json
# fio rbd_fio_examples/randwrite64k.fio && jc --pretty /proc/diskstats \
 | python3 diskstat_diff.py -d /tmp/ -a blue_8osd_6reactor_192at_8fio_socket_cond.json

Jobs: 8 (f=8): [w(8)][30.5%][w=24.3GiB/s][w=398k IOPS][eta 13m:41s]
nvme0n1p2: (groupid=0, jobs=8): err= 0: pid=375444: Fri Jan 31 11:43:35 2025
  write: IOPS=397k, BW=24.2GiB/s (26.0GB/s)(8742GiB/360796msec); 0 zone resets
    slat (nsec): min=1543, max=823010, avg=5969.62, stdev=2226.84
    clat (usec): min=57, max=50322, avg=5152.50, stdev=2982.28
     lat (usec): min=70, max=50328, avg=5158.47, stdev=2982.27
    clat percentiles (usec):
     |  1.00th=[  281],  5.00th=[  594], 10.00th=[ 1037], 20.00th=[ 2008],
     | 30.00th=[ 3032], 40.00th=[ 4080], 50.00th=[ 5145], 60.00th=[ 6194],
     | 70.00th=[ 7242], 80.00th=[ 8291], 90.00th=[ 9241], 95.00th=[ 9634],
     | 99.00th=[10028], 99.50th=[10421], 99.90th=[14091], 99.95th=[16188],
     | 99.99th=[19268]
   bw (  MiB/s): min=15227, max=24971, per=100.00%, avg=24845.68, stdev=88.47, samples=5768
   iops        : min=243638, max=399547, avg=397527.12, stdev=1415.43, samples=5768
  lat (usec)   : 100=0.01%, 250=0.61%, 500=3.18%, 750=2.90%, 1000=2.88%
  lat (msec)   : 2=10.28%, 4=19.25%, 10=59.90%, 20=1.00%, 50=0.01%
  lat (msec)   : 100=0.01%
  cpu          : usr=19.80%, sys=15.60%, ctx=104026691, majf=0, minf=2647
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
     issued rwts: total=0,143224767,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=256
  steadystate  : attained=yes, bw=24.3GiB/s (25.5GB/s), iops=398k, iops mean dev=1.215%

Run status group 0 (all jobs):
  WRITE: bw=24.2GiB/s (26.0GB/s), 24.2GiB/s-24.2GiB/s (26.0GB/s-26.0GB/s), io=8742GiB (9386GB), run=360796-360796msec

# CPU allocation strategies
declare -A bal_ops_table
bal_ops_table["default"]=""
bal_ops_table["bal_osd"]=" --crimson-balance-cpu osd"
bal_ops_table["bal_socket"]="--crimson-balance-cpu socket"

# Run balanced vs default CPU core/reactor distribution in Crimson using either Cyan, Seastore or  Bluestore
fun_run_bal_vs_default_tests() {
  local OSD_TYPE=$1
  local NUM_ALIEN_THREADS=7 # default 
  local title=""

  for KEY in default bal_osd bal_socket; do
    for NUM_OSD in 8; do
      for NUM_REACTORS in 5 6; do
        title="(${OSD_TYPE}) $NUM_OSD OSD crimson, $NUM_REACTORS reactor, fixed FIO 8 cores, response latency "

        cmd="MDS=0 MON=1 OSD=${NUM_OSD} MGR=1 taskset -ac '${VSTART_CPU_CORES}' /ceph/src/vstart.sh \
 --new -x --localhost --without-dashboard\
 --redirect-output ${crimson_be_table[${OSD_TYPE}]} --crimson --crimson-smp ${NUM_REACTORS}\
 --no-restart ${bal_ops_table[${KEY}]}"
        # Alien setup for Bluestore, see below.

        test_name="${OSD_TYPE}_${NUM_OSD}osd_${NUM_REACTORS}reactor_8fio_${KEY}_rc"
        echo "${cmd}"  | tee >> ${RUN_DIR}/${test_name}_cpu_distro.log
        echo $test_name
        eval "$cmd" >> ${RUN_DIR}/${test_name}_cpu_distro.log
        echo "Sleeping for 20 secs..."

        sleep 20 
        fun_show_grid $test_name
        fun_run_fio $test_name
        /ceph/src/stop.sh --crimson
        sleep 60
      done
    done
  done
}

        if [ "$OSD_TYPE" == "blue" ]; then
            NUM_ALIEN_THREADS=$(( 4 *NUM_OSD * NUM_REACTORS ))
            title="${title} alien_num_threads=${NUM_ALIEN_THREADS}"
            cmd="${cmd}  --crimson-alien-num-threads $NUM_ALIEN_THREADS"
            test_name="${OSD_TYPE}_${NUM_OSD}osd_${NUM_REACTORS}reactor_${NUM_ALIEN_THREADS}at_8fio_${KEY}_rc"
        fi

# Take some measurements
if pgrep crimson; then
	bin/ceph daemon -c /ceph/build/ceph.conf osd.0 dump_metrics > /tmp/new_cluster_dump.json
else
	bin/ceph daemon -c /ceph/build/ceph.conf osd.0 perf dump > /tmp/new_cluster_dump.json
fi

# Create the pools
bin/ceph osd pool create rbd
bin/ceph osd pool application enable rbd rbd
[ -z "$NUM_RBD_IMAGES" ] && NUM_RBD_IMAGES=1
for (( i=0; i<$NUM_RBD_IMAGES; i++ )); do
  bin/rbd create --size ${RBD_SIZE} rbd/fio_test_${i}
  rbd du fio_test_${i}
done
bin/ceph status
bin/ceph osd dump | grep 'replicated size'
# Show a pool’s utilization statistics:
rados df
# Turn off auto scaler for existing and new pools - stops PGs being split/merged
bin/ceph osd pool set noautoscale
# Turn off balancer to avoid moving PGs
bin/ceph balancer off
# Turn off deep scrub
bin/ceph osd set nodeep-scrub
# Turn off scrub
bin/ceph osd set noscrub

    pool 'rbd' created
enabled application 'rbd' on pool 'rbd'
NAME        PROVISIONED  USED
fio_test_0       10 GiB   0 B
  cluster:
    id:     da51b911-7229-4eae-afb5-a9833b978a68
    health: HEALTH_OK

  services:
    mon: 1 daemons, quorum a (age 97s)
    mgr: x(active, since 94s)
    osd: 8 osds: 8 up (since 52s), 8 in (since 60s)

  data:
    pools:   2 pools, 33 pgs
    objects: 2 objects, 449 KiB
    usage:   214 MiB used, 57 TiB / 57 TiB avail
    pgs:     27.273% pgs unknown
             21.212% pgs not active
             17 active+clean
             9  unknown
             7  creating+peering

pool 1 '.mgr' replicated size 3 min_size 1 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 15 flags hashpspool,nopgchange,crimson stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89
pool 2 'rbd' replicated size 3 min_size 1 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 33 flags hashpspool,nopgchange,selfmanaged_snaps,crimson stripe_width 0 application rbd read_balance_score 1.50
POOL_NAME     USED  OBJECTS  CLONES  COPIES  MISSING_ON_PRIMARY  UNFOUND  DEGRADED  RD_OPS      RD  WR_OPS       WR  USED COMPR  UNDER COMPR
.mgr       449 KiB        2       0       6                   0        0         0      41  35 KiB      55  584 KiB         0 B          0 B
rbd            0 B        0       0       0                   0        0         0       0     0 B       0      0 B         0 B          0 B

total_objects    2
total_used       214 MiB
total_avail      57 TiB
total_space      57 TiB
noautoscale is set, all pools now have autoscale off
nodeep-scrub is set
noscrub is set

# Option -w (WORKLOAD) is used as index for these:
declare -A m_s_iodepth=( [hockey]="1 2 4 8 16 24 32 40 52 64"  ...)
declare -A m_s_numjobs=( [hockey]="1" ... )

# Prime the volume(s) with a write workloads
   RBD_NAME=fio_test_$i RBD_SIZE="64k" fio ${FIO_JOBS}rbd_prime.fio 2>&1 >/dev/null & 
   echo "== priming $RBD_NAME ==";
 ...
   wait;

  for job in $RANGE_NUMJOBS; do
    for io in $RANGE_IODEPTH; do

      # Take diskstats measurements before FIO instances
      jc --pretty /proc/diskstats > ${DISK_STAT}
...
        for (( i=0; i<${NUM_PROCS}; i++ )); do

        export TEST_NAME=${TEST_PREFIX}_${job}job_${io}io_${BLOCK_SIZE_KB}_${map[${WORKLOAD}]}_p${i};
        echo "== $(date) == ($io,$job): ${TEST_NAME} ==";
        echo fio_${TEST_NAME}.json >> ${OSD_TEST_LIST}
        fio_name=${FIO_JOBS}${FIO_JOB_SPEC}${map[${WORKLOAD}]}.fio

          # Execute FIO
          LOG_NAME=${log_name} RBD_NAME=fio_test_${i} IO_DEPTH=${io} NUM_JOBS=${job} \
            taskset -ac ${FIO_CORES} fio ${fio_name} --output=fio_${TEST_NAME}.json \
            --output-format=json 2> fio_${TEST_NAME}.err &
          fio_id["fio_${i}"]=$!
          global_fio_id+=($!)
      done # loop NUM_PROCS
      sleep 30; # ramp up time
...
    fun_measure "${all_pids}" ${top_out_name} ${TOP_OUT_LIST} &
...
      wait;
      # Measure the diskstats after the completion of FIO
      jc --pretty /proc/diskstats | python3 /root/bin/diskstat_diff.py -a ${DISK_STAT}

      # Exit the loops if the latency disperses too much from the median
      if [ "$RESPONSE_CURVE" = true ] && [ "$RC_SKIP_HEURISTIC" = false ]; then
        mop=${mode[${WORKLOAD}]}
        covar=$(jq ".jobs | .[] | .${mop}.clat_ns.stddev/.${mop}.clat_ns.mean < 0.5 and \
          .${mop}.clat_ns.mean/1000000 < ${MAX_LATENCY}" fio_${TEST_NAME}.json)
                  if [ "$covar" != "true" ]; then
                    echo "== Latency std dev too high, exiting loops =="
                    break 2
                  fi
      fi
    done # loop IODEPTH
  done # loop NUM_JOBS 

fun_measure() {
  local PID=$1 #comma separated list of pids
  local TEST_NAME=$2
  local TEST_TOP_OUT_LIST=$3

  top -b -H -1 -p "${PID}" -n ${NUM_SAMPLES} >> ${TEST_NAME}_top.out
  echo "${TEST_NAME}_top.out" >> ${TEST_TOP_OUT_LIST}
}