diff --git a/Library/Homebrew/cleanup.rb b/Library/Homebrew/cleanup.rb
index 50cbba762f..605a7e17c9 100644
--- a/Library/Homebrew/cleanup.rb
+++ b/Library/Homebrew/cleanup.rb
@@ -480,11 +480,24 @@ module Homebrew
 
       return if portable_rubies_to_remove.empty?
 
-      bundler_path = vendor_dir/"bundle/ruby"
-      if dry_run?
-        puts Utils.popen_read("git", "-C", HOMEBREW_REPOSITORY, "clean", "-nx", bundler_path).chomp
-      else
-        puts Utils.popen_read("git", "-C", HOMEBREW_REPOSITORY, "clean", "-ffqx", bundler_path).chomp
+      bundler_paths = (vendor_dir/"bundle/ruby").children.select do |child|
+        basename = child.basename.to_s
+
+        next false if basename == ".homebrew_gem_groups"
+        next true unless child.directory?
+
+        [
+          "#{Version.new(portable_ruby_latest_version).major_minor}.0",
+          RbConfig::CONFIG["ruby_version"],
+        ].uniq.exclude?(basename)
+      end
+
+      bundler_paths.each do |bundler_path|
+        if dry_run?
+          puts Utils.popen_read("git", "-C", HOMEBREW_REPOSITORY, "clean", "-nx", bundler_path).chomp
+        else
+          puts Utils.popen_read("git", "-C", HOMEBREW_REPOSITORY, "clean", "-ffqx", bundler_path).chomp
+        end
       end
 
       portable_rubies_to_remove.each do |portable_ruby|