From 05fc71fb34ba2e50451f6154ac0be08e7bd786eb Mon Sep 17 00:00:00 2001 From: Shaisolaris <66793913+Shaisolaris@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:51:05 -0700 Subject: [PATCH] [ruby/rubygems] Preload git sources that share a cache one after another Gems sourced from the same git repository share one cache directory. Since git sources are preloaded in parallel, two workers could run git fetch in that directory at the same time, and the fetches collided on the shallow file and its lock, failing bundle install. Group the sources by cache path and preload each group in a single job, so sources sharing a cache run one after another while separate repositories still run in parallel. Fixes https://github.com/ruby/rubygems/pull/9946 https://github.com/ruby/rubygems/commit/c20901e6a1 --- lib/bundler/definition.rb | 10 +++++-- spec/bundler/bundler/definition_spec.rb | 39 +++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb index 4b66e3eef31..c6c104b5ff5 100644 --- a/lib/bundler/definition.rb +++ b/lib/bundler/definition.rb @@ -1293,7 +1293,7 @@ module Bundler def preload_git_source_worker workers = Bundler.settings.installation_parallelization - @preload_git_source_worker ||= Bundler::Worker.new(workers, "Git source preloading", ->(source, _) { source.specs }) + @preload_git_source_worker ||= Bundler::Worker.new(workers, "Git source preloading", ->(sources, _) { sources.each(&:specs) }) end def preload_git_sources @@ -1304,7 +1304,13 @@ module Bundler end begin - needed_git_sources.each {|source| preload_git_source_worker.enq(source) } + # Sources from the same repository share one cache directory, and + # running git in that directory from two workers at once makes the + # fetches collide. Preload sources that share a cache in one job, one + # after the other, and run the separate caches in parallel. + needed_git_sources.group_by(&:cache_path).each_value do |sources| + preload_git_source_worker.enq(sources) + end ensure preload_git_source_worker.stop end diff --git a/spec/bundler/bundler/definition_spec.rb b/spec/bundler/bundler/definition_spec.rb index 8c4a5a0331e..d263afa0902 100644 --- a/spec/bundler/bundler/definition_spec.rb +++ b/spec/bundler/bundler/definition_spec.rb @@ -21,6 +21,45 @@ RSpec.describe Bundler::Definition do end end + describe "#preload_git_sources" do + before do + allow(Bundler::SharedHelpers).to receive(:find_gemfile) { bundled_app_gemfile } + allow(Bundler.settings).to receive(:installation_parallelization).and_return(4) + end + + subject { Bundler::Definition.new(bundled_app_lock, [], Bundler::SourceList.new, {}) } + + it "never preloads two sources sharing a git cache at the same time" do + skip "git sources are preloaded one by one before Ruby 3.3" if Gem.ruby_version < Gem::Version.new("3.3") + + lock = Mutex.new + running = Hash.new(0) + max_running = Hash.new(0) + max_total = 0 + + sources = %w[monorepo monorepo monorepo other].map do |repo| + source = double("git source #{repo}", cache_path: Pathname.new("/cache/git/#{repo}")) + allow(source).to receive(:specs) do + lock.synchronize do + running[repo] += 1 + max_running[repo] = [max_running[repo], running[repo]].max + max_total = [max_total, running.values.sum].max + end + sleep 0.1 + lock.synchronize { running[repo] -= 1 } + end + source + end + allow(subject).to receive(:needed_git_sources).and_return(sources) + + subject.send(:preload_git_sources) + + sources.each {|source| expect(source).to have_received(:specs).once } + expect(max_running["monorepo"]).to eq(1) + expect(max_total).to eq(2) + end + end + describe "#lock" do before do allow(Bundler::SharedHelpers).to receive(:find_gemfile) { bundled_app_gemfile }