mirror of
https://github.com/basecamp/once-campfire.git
synced 2026-09-18 06:22:08 +09:00
Replace Redis with the Solid trifecta
This commit is contained in:
Stanko K.R.
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env ruby
|
||||
require "optparse"
|
||||
require "json"
|
||||
require "net/http"
|
||||
require "fileutils"
|
||||
require "etc"
|
||||
require "time"
|
||||
|
||||
options = {
|
||||
users: 3000, ramp: 480, hold: 120, cpus: 2, memory: "2g",
|
||||
send_interval: 1, sample: 25, build: true, port: 3000, image: "campfire-bench"
|
||||
}
|
||||
|
||||
OptionParser.new do |parser|
|
||||
parser.on("--label LABEL", "Run label (required), e.g. redis-2cpu-2g") { |v| options[:label] = v }
|
||||
parser.on("--users N", Integer, "Ramp target (default 3000, max 10000)") { |v| options[:users] = v }
|
||||
parser.on("--ramp SECONDS", Integer, "Ramp duration (default 480)") { |v| options[:ramp] = v }
|
||||
parser.on("--hold SECONDS", Integer, "Hold duration at target (default 120)") { |v| options[:hold] = v }
|
||||
parser.on("--cpus N", Integer, "App container CPU count via cpuset (default 2)") { |v| options[:cpus] = v }
|
||||
parser.on("--memory SIZE", "App container memory limit (default 2g)") { |v| options[:memory] = v }
|
||||
parser.on("--send-interval S", Float, "Seconds between messages (default 1)") { |v| options[:send_interval] = v }
|
||||
parser.on("--sample N", Integer, "1-in-N receivers log latency lines (default 25)") { |v| options[:sample] = v }
|
||||
parser.on("--port PORT", Integer, "Host port for the app (default 3000)") { |v| options[:port] = v }
|
||||
parser.on("--image TAG", "Image tag (default campfire-bench)") { |v| options[:image] = v }
|
||||
parser.on("--no-build", "Skip docker build") { options[:build] = false }
|
||||
end.parse!
|
||||
|
||||
abort "--label is required" unless options[:label]
|
||||
|
||||
class Bench
|
||||
APP_CONTAINER = "campfire-bench-app"
|
||||
K6_CONTAINER = "campfire-bench-k6"
|
||||
|
||||
attr_reader :opts, :results_dir
|
||||
|
||||
def initialize(opts)
|
||||
@opts = opts
|
||||
@root = File.expand_path("..", __dir__)
|
||||
@perf_dir = File.join(@root, "test/performance")
|
||||
@results_dir = File.join(@root, "tmp/bench", opts[:label])
|
||||
end
|
||||
|
||||
def run
|
||||
FileUtils.mkdir_p(results_dir)
|
||||
build if opts[:build]
|
||||
write_meta
|
||||
start_app
|
||||
wait_for_app
|
||||
sampler = start_stats_sampler
|
||||
run_k6
|
||||
ensure
|
||||
stop(sampler)
|
||||
capture_app_state
|
||||
cleanup
|
||||
end
|
||||
|
||||
private
|
||||
def build
|
||||
sh "docker", "build", "-t", opts[:image], @root
|
||||
end
|
||||
|
||||
def write_meta
|
||||
meta = opts.merge(
|
||||
git_revision: `git rev-parse HEAD`.strip,
|
||||
git_branch: `git rev-parse --abbrev-ref HEAD`.strip,
|
||||
host_cpus: Etc.nprocessors,
|
||||
started_at: Time.now.utc.iso8601
|
||||
)
|
||||
File.write(File.join(results_dir, "meta.json"), JSON.pretty_generate(meta))
|
||||
end
|
||||
|
||||
def start_app
|
||||
system "docker", "rm", "-f", APP_CONTAINER, err: File::NULL, out: File::NULL
|
||||
sh "docker", "run", "-d", "--name", APP_CONTAINER,
|
||||
"--cpuset-cpus", app_cpuset,
|
||||
"--memory", opts[:memory], "--memory-swap", opts[:memory],
|
||||
"-p", "#{opts[:port]}:80",
|
||||
"-e", "SECRET_KEY_BASE=dummy",
|
||||
"-e", "RAILS_ENV=performance",
|
||||
opts[:image]
|
||||
end
|
||||
|
||||
def app_cpuset
|
||||
(0...opts[:cpus]).to_a.join(",")
|
||||
end
|
||||
|
||||
def wait_for_app
|
||||
timeout_at = Time.now + 300
|
||||
uri = URI("http://127.0.0.1:#{opts[:port]}/")
|
||||
loop do
|
||||
raise "App not up after 300s" if Time.now > timeout_at
|
||||
begin
|
||||
break if Net::HTTP.get_response(uri).code.to_i < 400
|
||||
rescue Errno::ECONNREFUSED, Errno::ECONNRESET, EOFError
|
||||
end
|
||||
puts "Waiting for app (seeding 10k users on first boot)..."
|
||||
sleep 2
|
||||
end
|
||||
puts "App is up."
|
||||
end
|
||||
|
||||
def start_stats_sampler
|
||||
stats_path = File.join(results_dir, "stats.csv")
|
||||
fork do
|
||||
File.open(stats_path, "w") do |file|
|
||||
file.puts "epoch_ms,cpu_perc,mem_bytes"
|
||||
file.sync = true
|
||||
loop do
|
||||
line = `docker stats --no-stream --format "{{.CPUPerc}} {{.MemUsage}}" #{APP_CONTAINER} 2>/dev/null`.strip
|
||||
if line =~ /([\d.]+)%\s+([\d.]+)(\w+)/
|
||||
file.puts "#{(Time.now.to_f * 1000).to_i},#{$1},#{to_bytes($2.to_f, $3)}"
|
||||
end
|
||||
sleep 1
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def to_bytes(value, unit)
|
||||
factors = { "B" => 1, "KiB" => 1024, "MiB" => 1024**2, "GiB" => 1024**3,
|
||||
"kB" => 1000, "MB" => 1000**2, "GB" => 1000**3 }
|
||||
(value * factors.fetch(unit, 1)).to_i
|
||||
end
|
||||
|
||||
def run_k6
|
||||
total = opts[:ramp] + opts[:hold]
|
||||
puts "Running k6: ramp to #{opts[:users]} users over #{opts[:ramp]}s, hold #{opts[:hold]}s (total #{total}s)..."
|
||||
system "docker", "rm", "-f", K6_CONTAINER, err: File::NULL, out: File::NULL
|
||||
k6_log = File.join(results_dir, "k6.log")
|
||||
summary = File.join(results_dir, "summary.json")
|
||||
finished = system(
|
||||
"docker", "run", "--rm", "--name", K6_CONTAINER,
|
||||
"--network", "host",
|
||||
"--cpuset-cpus", k6_cpuset,
|
||||
"--ulimit", "nofile=262144:262144",
|
||||
"-u", "#{Process.uid}:#{Process.gid}",
|
||||
"-v", "#{@perf_dir}:/src",
|
||||
"-v", "#{results_dir}:/results",
|
||||
"-e", "HOST=127.0.0.1", "-e", "PORT=#{opts[:port]}",
|
||||
"-e", "USERS=#{opts[:users]}",
|
||||
"-e", "RAMP_S=#{opts[:ramp]}", "-e", "HOLD_S=#{opts[:hold]}",
|
||||
"-e", "SEND_INTERVAL_S=#{opts[:send_interval]}",
|
||||
"-e", "SAMPLE=#{opts[:sample]}",
|
||||
"grafana/k6", "run",
|
||||
"--summary-export", "/results/summary.json",
|
||||
"--quiet",
|
||||
"/src/ramp.js",
|
||||
out: k6_log, err: [ k6_log, "a" ]
|
||||
)
|
||||
if finished
|
||||
puts "k6 finished."
|
||||
else
|
||||
puts "k6 exited non-zero — results still captured in #{k6_log}."
|
||||
end
|
||||
end
|
||||
|
||||
def k6_cpuset
|
||||
(opts[:cpus]...Etc.nprocessors).to_a.join(",")
|
||||
end
|
||||
|
||||
def stop(sampler)
|
||||
Process.kill("TERM", sampler)
|
||||
Process.wait(sampler)
|
||||
rescue Errno::ESRCH, Errno::ECHILD
|
||||
nil
|
||||
end
|
||||
|
||||
def capture_app_state
|
||||
oom = `docker inspect --format '{{.State.OOMKilled}}' #{APP_CONTAINER} 2>/dev/null`.strip
|
||||
File.write(File.join(results_dir, "oom.txt"), oom)
|
||||
system "docker logs --tail 200 #{APP_CONTAINER} > #{File.join(results_dir, 'app.log')} 2>&1"
|
||||
puts "App container was OOM-killed during the run." if oom == "true"
|
||||
end
|
||||
|
||||
def cleanup
|
||||
system "docker", "rm", "-f", APP_CONTAINER, err: File::NULL, out: File::NULL
|
||||
end
|
||||
|
||||
def sh(*cmd)
|
||||
puts "+ #{cmd.join(' ')}"
|
||||
system(*cmd) || abort("Command failed: #{cmd.join(' ')}")
|
||||
end
|
||||
end
|
||||
|
||||
Bench.new(options).run
|
||||
puts "Results in tmp/bench/#{options[:label]}/"
|
||||
Executable
+160
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env ruby
|
||||
require "json"
|
||||
require "csv"
|
||||
|
||||
BIN_MS = 10_000
|
||||
SLO_P95_MS = 500
|
||||
|
||||
class Run
|
||||
attr_reader :label, :dir
|
||||
|
||||
def initialize(label)
|
||||
@label = label
|
||||
@dir = File.expand_path("../tmp/bench/#{label}", __dir__)
|
||||
abort "No results in tmp/bench/#{label}" unless File.exist?(File.join(dir, "k6.log"))
|
||||
end
|
||||
|
||||
def report
|
||||
write_series
|
||||
print_summary
|
||||
end
|
||||
|
||||
def series
|
||||
@series ||= build_series
|
||||
end
|
||||
|
||||
def meta
|
||||
@meta ||= JSON.parse(File.read(File.join(dir, "meta.json")))
|
||||
end
|
||||
|
||||
def summary
|
||||
@summary ||= JSON.parse(File.read(File.join(dir, "summary.json")))
|
||||
rescue Errno::ENOENT
|
||||
{}
|
||||
end
|
||||
|
||||
def capacity
|
||||
breach = first_sustained_breach
|
||||
if breach
|
||||
breach[:connected]
|
||||
else
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
def build_series
|
||||
events = parse_k6_log
|
||||
stats = parse_stats
|
||||
first_ts = events.map { |e| e[:ts] }.min
|
||||
return [] unless first_ts
|
||||
|
||||
bins = Hash.new { |h, k| h[k] = { lats: [], conns: 0, sents: 0, errs: 0, cpu: [], mem: [] } }
|
||||
events.each do |event|
|
||||
bin = (event[:ts] - first_ts) / BIN_MS
|
||||
case event[:kind]
|
||||
when "CONN" then bins[bin][:conns] += 1
|
||||
when "LAT" then bins[bin][:lats] << event[:value]
|
||||
when "SENT" then bins[bin][:sents] += 1
|
||||
when "ERR" then bins[bin][:errs] += 1
|
||||
end
|
||||
end
|
||||
stats.each do |stat|
|
||||
bin = (stat[:ts] - first_ts) / BIN_MS
|
||||
next if bin < 0
|
||||
bins[bin][:cpu] << stat[:cpu]
|
||||
bins[bin][:mem] << stat[:mem]
|
||||
end
|
||||
|
||||
connected = 0
|
||||
(0..bins.keys.max).map do |bin|
|
||||
data = bins[bin]
|
||||
connected = [ connected + data[:conns], meta["users"] ].min
|
||||
{
|
||||
elapsed_s: bin * BIN_MS / 1000,
|
||||
connected: connected,
|
||||
sent: data[:sents],
|
||||
samples: data[:lats].size,
|
||||
lat_p50: percentile(data[:lats], 50),
|
||||
lat_p95: percentile(data[:lats], 95),
|
||||
lat_max: data[:lats].max,
|
||||
errors: data[:errs],
|
||||
cpu_perc: average(data[:cpu]),
|
||||
mem_mb: average(data[:mem])&.then { |bytes| (bytes / 1024.0 / 1024.0).round(1) }
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
def parse_k6_log
|
||||
events = []
|
||||
File.foreach(File.join(dir, "k6.log")) do |line|
|
||||
if line =~ /BENCH (CONN|LAT|SENT|ERR) (\d+)(?: (\S+))?/
|
||||
events << { kind: $1, ts: $2.to_i, value: $3.to_i }
|
||||
end
|
||||
end
|
||||
events
|
||||
end
|
||||
|
||||
def parse_stats
|
||||
path = File.join(dir, "stats.csv")
|
||||
return [] unless File.exist?(path)
|
||||
CSV.read(path, headers: true).map do |row|
|
||||
{ ts: row["epoch_ms"].to_i, cpu: row["cpu_perc"].to_f, mem: row["mem_bytes"].to_i }
|
||||
end
|
||||
end
|
||||
|
||||
def percentile(values, pct)
|
||||
return nil if values.empty?
|
||||
sorted = values.sort
|
||||
sorted[((pct / 100.0) * (sorted.size - 1)).round]
|
||||
end
|
||||
|
||||
def average(values)
|
||||
return nil if values.empty?
|
||||
(values.sum / values.size.to_f).round(1)
|
||||
end
|
||||
|
||||
def write_series
|
||||
CSV.open(File.join(dir, "series.csv"), "w") do |csv|
|
||||
csv << series.first.keys
|
||||
series.each { |row| csv << row.values }
|
||||
end
|
||||
end
|
||||
|
||||
def first_sustained_breach
|
||||
series.each_cons(2) do |a, b|
|
||||
if breached?(a) && breached?(b)
|
||||
return a
|
||||
end
|
||||
end
|
||||
nil
|
||||
end
|
||||
|
||||
def breached?(bin)
|
||||
if bin[:samples] > 0 && bin[:lat_p95] && bin[:lat_p95] > SLO_P95_MS
|
||||
true
|
||||
else
|
||||
bin[:errors] > 0
|
||||
end
|
||||
end
|
||||
|
||||
def print_summary
|
||||
latency = summary.dig("metrics", "delivery_latency") || {}
|
||||
puts
|
||||
puts "== #{label} (#{meta['cpus']} CPU, #{meta['memory']} RAM, ramp to #{meta['users']}) =="
|
||||
puts " delivery latency: p50=#{latency['med']&.round(1)}ms p95=#{latency['p(95)']&.round(1)}ms max=#{latency['max']&.round(1)}ms"
|
||||
puts " messages received: #{summary.dig('metrics', 'bench_messages_received', 'count')}"
|
||||
puts " socket errors: #{summary.dig('metrics', 'bench_socket_errors', 'count') || 0}"
|
||||
peak = series.map { |bin| bin[:mem_mb] }.compact.max
|
||||
puts " peak container memory: #{peak} MB"
|
||||
if capacity
|
||||
puts " capacity at SLO (p95 <= #{SLO_P95_MS}ms, no errors): ~#{capacity} concurrent users"
|
||||
else
|
||||
puts " capacity at SLO: not reached (>= #{series.last[:connected]} concurrent users)"
|
||||
end
|
||||
puts " series: tmp/bench/#{label}/series.csv"
|
||||
end
|
||||
end
|
||||
|
||||
abort "Usage: bin/bench-report LABEL [LABEL...]" if ARGV.empty?
|
||||
ARGV.each { |label| Run.new(label).report }
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env ruby
|
||||
|
||||
require_relative "../config/environment"
|
||||
require "solid_queue/cli"
|
||||
|
||||
SolidQueue::Cli.start(ARGV)
|
||||
@@ -8,9 +8,6 @@ app_root="$(
|
||||
)"
|
||||
export PATH="$app_root/bin:$PATH"
|
||||
|
||||
REDIS_PORT=6379
|
||||
REDIS_HOST=localhost
|
||||
|
||||
if [ "$RAILS_ENV" = "production" ]; then
|
||||
echo "RAILS_ENV is production; bailing out"
|
||||
exit 1
|
||||
@@ -45,10 +42,6 @@ step() {
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
redis_running() {
|
||||
nc -z "$REDIS_HOST" "$REDIS_PORT" 2>/dev/null
|
||||
}
|
||||
|
||||
echo
|
||||
gum style --foreground 214 " ) "
|
||||
gum style --foreground 208 " ) \\ campfire"
|
||||
@@ -88,21 +81,6 @@ if [[ $* == *--reset* ]]; then
|
||||
fi
|
||||
step "Preparing the database" rails db:prepare
|
||||
|
||||
# Start Redis if not running
|
||||
if ! redis_running; then
|
||||
if command -v docker &>/dev/null; then
|
||||
if docker ps -aq -f name=campfire-redis | grep -q .; then
|
||||
step "Starting Redis" docker start campfire-redis
|
||||
else
|
||||
step "Setting up Redis" docker run -d --name campfire-redis -p "$REDIS_PORT:$REDIS_PORT" redis:7
|
||||
fi
|
||||
else
|
||||
echo "Couldn't start Redis"
|
||||
echo "Install either docker or redis and then run this command again"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Install GitHub Actions linting tools
|
||||
for tool in actionlint shellcheck zizmor; do
|
||||
if ! command -v "$tool" &> /dev/null; then
|
||||
|
||||
Reference in New Issue
Block a user