[ruby/rubygems] Preload git sources that share a cache one after another

Gems sourced from the same git repository share one cache directory.
Since git sources are preloaded in parallel, two workers could run git
fetch in that directory at the same time, and the fetches collided on
the shallow file and its lock, failing bundle install.

Group the sources by cache path and preload each group in a single job,
so sources sharing a cache run one after another while separate
repositories still run in parallel.

Fixes https://github.com/ruby/rubygems/pull/9946

https://github.com/ruby/rubygems/commit/c20901e6a1

lib/bundler/definition.rb

@@ -1293,7 +1293,7 @@ module Bundler
     def preload_git_source_worker
       workers = Bundler.settings.installation_parallelization
 
-      @preload_git_source_worker ||= Bundler::Worker.new(workers, "Git source preloading", ->(source, _) { source.specs })
+      @preload_git_source_worker ||= Bundler::Worker.new(workers, "Git source preloading", ->(sources, _) { sources.each(&:specs) })
     end
 
     def preload_git_sources
@@ -1304,7 +1304,13 @@ module Bundler
       end
 
       begin
-        needed_git_sources.each {|source| preload_git_source_worker.enq(source) }
+        # Sources from the same repository share one cache directory, and
+        # running git in that directory from two workers at once makes the
+        # fetches collide. Preload sources that share a cache in one job, one
+        # after the other, and run the separate caches in parallel.
+        needed_git_sources.group_by(&:cache_path).each_value do |sources|
+          preload_git_source_worker.enq(sources)
+        end
       ensure
         preload_git_source_worker.stop
       end

spec/bundler/bundler/definition_spec.rb

@@ -21,6 +21,45 @@ RSpec.describe Bundler::Definition do
     end
   end
 
+  describe "#preload_git_sources" do
+    before do
+      allow(Bundler::SharedHelpers).to receive(:find_gemfile) { bundled_app_gemfile }
+      allow(Bundler.settings).to receive(:installation_parallelization).and_return(4)
+    end
+
+    subject { Bundler::Definition.new(bundled_app_lock, [], Bundler::SourceList.new, {}) }
+
+    it "never preloads two sources sharing a git cache at the same time" do
+      skip "git sources are preloaded one by one before Ruby 3.3" if Gem.ruby_version < Gem::Version.new("3.3")
+
+      lock = Mutex.new
+      running = Hash.new(0)
+      max_running = Hash.new(0)
+      max_total = 0
+
+      sources = %w[monorepo monorepo monorepo other].map do |repo|
+        source = double("git source #{repo}", cache_path: Pathname.new("/cache/git/#{repo}"))
+        allow(source).to receive(:specs) do
+          lock.synchronize do
+            running[repo] += 1
+            max_running[repo] = [max_running[repo], running[repo]].max
+            max_total = [max_total, running.values.sum].max
+          end
+          sleep 0.1
+          lock.synchronize { running[repo] -= 1 }
+        end
+        source
+      end
+      allow(subject).to receive(:needed_git_sources).and_return(sources)
+
+      subject.send(:preload_git_sources)
+
+      sources.each {|source| expect(source).to have_received(:specs).once }
+      expect(max_running["monorepo"]).to eq(1)
+      expect(max_total).to eq(2)
+    end
+  end
+
   describe "#lock" do
     before do
       allow(Bundler::SharedHelpers).to receive(:find_gemfile) { bundled_app_gemfile }