Keep bot page totals on SQLite-maintained rooms.messages_count

X-Total-Count on GET /rooms/:id/:bot_key/messages was COUNT(*) of the
room on every page. Serve it from rooms.messages_count updated by SQLite
triggers so Rails, bulk SQL, and foreign writers stay in step — without
ActiveRecord counter_cache callbacks those paths skip.

Fixes #309.

Co-authored-by: Thomas Klemm <github@tklemm.eu>

test/controllers/messages/by_bots_controller_test.rb

@@ -1,6 +1,9 @@
 require "test_helper"
+require "active_record/testing/query_assertions"
 
 class Messages::ByBotsControllerTest < ActionDispatch::IntegrationTest
+  include ActiveRecord::Assertions::QueryAssertions
+
   setup do
     @room = rooms(:watercooler)
   end
@@ -93,11 +96,14 @@ class Messages::ByBotsControllerTest < ActionDispatch::IntegrationTest
       @room.messages.create!(body: "Filler #{i}", creator: users(:jason), client_message_id: "filler-#{i}")
     end
 
-    get room_bot_messages_url(@room, users(:bender).bot_key)
+    assert_no_queries_match(/SELECT COUNT\(\*\) FROM "messages"/i) do
+      get room_bot_messages_url(@room, users(:bender).bot_key)
+    end
     assert_response :success
 
     json = JSON.parse(response.body)
     assert_equal Message::PAGE_SIZE, json.size
+    assert_equal @room.reload.messages_count.to_s, response.headers["X-Total-Count"]
     assert_equal "41", response.headers["X-Total-Count"]
     assert_not_includes json.map { it["id"] }, messages(:fourth).id
 
@@ -119,7 +125,10 @@ class Messages::ByBotsControllerTest < ActionDispatch::IntegrationTest
   end
 
   test "index in a room with no messages" do
-    get room_bot_messages_url(rooms(:bender_and_kevin), users(:bender).bot_key)
+    room = rooms(:bender_and_kevin)
+    assert_equal 0, room.messages_count
+
+    get room_bot_messages_url(room, users(:bender).bot_key)
     assert_response :success
 
     assert_equal [], JSON.parse(response.body)

test/models/room/messages_count_test.rb

@@ -0,0 +1,174 @@
+require "test_helper"
+require "sqlite3"
+
+class Room::MessagesCountTest < ActiveSupport::TestCase
+  setup do
+    @room = rooms(:designers)
+    @other_room = rooms(:pets)
+    synchronize!(@room, @other_room)
+  end
+
+  test "ActiveRecord create and destroy adjust the counter once" do
+    assert_difference -> { @room.reload.messages_count }, +1 do
+      assert_difference -> { @room.messages.count }, +1 do
+        @room.messages.create!(creator: users(:jason), body: "Hello", client_message_id: "count-ar-create")
+      end
+    end
+
+    assert_equal @room.messages.count, @room.reload.messages_count
+
+    assert_difference -> { @room.reload.messages_count }, -1 do
+      assert_difference -> { @room.messages.count }, -1 do
+        @room.messages.order(:id).last.destroy
+      end
+    end
+
+    assert_equal @room.messages.count, @room.reload.messages_count
+  end
+
+  test "bulk insert_all and delete_all keep the counter in step" do
+    rows = Array.new(3) do |i|
+      {
+        room_id: @room.id,
+        creator_id: users(:david).id,
+        client_message_id: "count-bulk-#{i}",
+        created_at: Time.current,
+        updated_at: Time.current
+      }
+    end
+
+    assert_difference -> { @room.reload.messages_count }, +3 do
+      Message.insert_all!(rows)
+    end
+
+    assert_equal @room.messages.count, @room.reload.messages_count
+
+    assert_difference -> { @room.reload.messages_count }, -3 do
+      @room.messages.where(client_message_id: rows.map { it[:client_message_id] }).delete_all
+    end
+
+    assert_equal @room.messages.count, @room.reload.messages_count
+  end
+
+  test "rolled back writes leave the counter unchanged" do
+    before = @room.reload.messages_count
+
+    Message.transaction do
+      @room.messages.create!(creator: users(:jason), body: "Nope", client_message_id: "count-rollback")
+      raise ActiveRecord::Rollback
+    end
+
+    assert_equal before, @room.reload.messages_count
+    assert_nil Message.find_by(client_message_id: "count-rollback")
+  end
+
+  test "moving a message between rooms moves the counter" do
+    message = @room.messages.create!(creator: users(:jason), body: "Move me", client_message_id: "count-move")
+    synchronize!(@room, @other_room)
+
+    assert_difference -> { @room.reload.messages_count }, -1 do
+      assert_difference -> { @other_room.reload.messages_count }, +1 do
+        message.update!(room: @other_room)
+      end
+    end
+
+    assert_equal @room.messages.count, @room.reload.messages_count
+    assert_equal @other_room.messages.count, @other_room.reload.messages_count
+  end
+
+  test "ensure! installs missing triggers without changing existing counts" do
+    before = @room.reload.messages_count
+    Room::MessagesCount.uninstall!
+
+    assert_not Room::MessagesCount.trigger_installed?(ActiveRecord::Base.connection, Room::MessagesCount::INSERT_TRIGGER)
+
+    Room::MessagesCount.ensure!
+
+    assert Room::MessagesCount.trigger_installed?(ActiveRecord::Base.connection, Room::MessagesCount::INSERT_TRIGGER)
+    assert_equal before, @room.reload.messages_count
+
+    assert_difference -> { @room.reload.messages_count }, +1 do
+      @room.messages.create!(creator: users(:jason), body: "After ensure", client_message_id: "count-ensure")
+    end
+  ensure
+    Room::MessagesCount.ensure!
+  end
+
+  test "backfill plus triggers do not double-count ActiveRecord writes" do
+    Room::MessagesCount.uninstall!
+    ActiveRecord::Base.connection.execute <<~SQL
+      UPDATE rooms SET messages_count = (
+        SELECT COUNT(*) FROM messages WHERE messages.room_id = rooms.id
+      )
+    SQL
+    Room::MessagesCount.install!
+
+    assert_equal @room.messages.count, @room.reload.messages_count
+
+    assert_difference -> { @room.reload.messages_count }, +1 do
+      assert_difference -> { @room.messages.count }, +1 do
+        @room.messages.create!(creator: users(:jason), body: "Once", client_message_id: "count-no-double")
+      end
+    end
+
+    assert_equal @room.messages.count, @room.reload.messages_count
+  ensure
+    Room::MessagesCount.ensure!
+  end
+
+  private
+    def synchronize!(*rooms)
+      Room::MessagesCount.ensure!
+      Room::MessagesCount.backfill!
+      rooms.each(&:reload)
+    end
+end
+
+# Foreign connections cannot join the transactional fixture lock; run outside it.
+class Room::MessagesCountForeignConnectionTest < ActiveSupport::TestCase
+  self.use_transactional_tests = false
+
+  setup do
+    Room::MessagesCount.ensure!
+    @room = rooms(:designers)
+    Room::MessagesCount.backfill!
+    @room.reload
+  end
+
+  teardown do
+    Message.where(client_message_id: "count-foreign-insert").delete_all
+    Room::MessagesCount.backfill!
+  end
+
+  test "foreign SQLite connections keep the counter in step" do
+    path = File.expand_path(ActiveRecord::Base.connection_db_config.database)
+    now = Time.current.utc.strftime("%Y-%m-%d %H:%M:%S.%6N")
+    before = @room.reload.messages_count
+
+    # Release AR's checkout so the native connection can write.
+    ActiveRecord::Base.connection_pool.release_connection
+
+    SQLite3::Database.new(path) do |db|
+      db.busy_timeout = 5_000
+      db.execute(
+        "INSERT INTO messages (room_id, creator_id, client_message_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
+        [ @room.id, users(:david).id, "count-foreign-insert", now, now ]
+      )
+    end
+
+    assert_equal before + 1, @room.reload.messages_count
+    assert_equal @room.messages.count, @room.messages_count
+
+    foreign_id = Message.find_by!(client_message_id: "count-foreign-insert").id
+
+    ActiveRecord::Base.connection_pool.release_connection
+
+    SQLite3::Database.new(path) do |db|
+      db.busy_timeout = 5_000
+      db.execute("DELETE FROM messages WHERE id = ?", [ foreign_id ])
+    end
+
+    assert_equal before, @room.reload.messages_count
+    assert_equal @room.messages.count, @room.messages_count
+  end
+end

test/test_helper.rb

@@ -6,10 +6,25 @@ require "mocha/minitest"
 require "webmock/minitest"
 require "turbo/broadcastable/test_helper"
 
+# maintain_test_schema! may reload schema.rb after after_initialize; triggers are
+# not dumped, so ensure they exist before fixtures insert messages.
+Room::MessagesCount.ensure!
+
 WebMock.enable!
 
+module RoomMessagesCountFixtures
+  # Fixture YAML loads alphabetically, so messages are inserted before rooms.
+  # INSERT triggers then update zero room rows; reconcile once after load.
+  def load_fixtures(config)
+    fixtures = super
+    Room::MessagesCount.backfill!
+    fixtures
+  end
+end
+
 class ActiveSupport::TestCase
   include ActiveJob::TestHelper
+  prepend RoomMessagesCountFixtures
 
   parallelize(workers: :number_of_processors)
 
@@ -19,6 +34,9 @@ class ActiveSupport::TestCase
   include SessionTestHelper, MentionTestHelper, TurboTestHelper, DnsTestHelper
 
   setup do
+    # DROP TRIGGER commits outside transactional fixtures; put triggers back each test.
+    Room::MessagesCount.ensure!
+
     ActionCable.server.pubsub.clear
 
     Rails.configuration.tap do |config|
@@ -34,3 +52,4 @@ class ActiveSupport::TestCase
     WebMock.reset!
   end
 end
+