diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb index 4b66e3eef31..c6c104b5ff5 100644 --- a/lib/bundler/definition.rb +++ b/lib/bundler/definition.rb @@ -1293,7 +1293,7 @@ module Bundler def preload_git_source_worker workers = Bundler.settings.installation_parallelization - @preload_git_source_worker ||= Bundler::Worker.new(workers, "Git source preloading", ->(source, _) { source.specs }) + @preload_git_source_worker ||= Bundler::Worker.new(workers, "Git source preloading", ->(sources, _) { sources.each(&:specs) }) end def preload_git_sources @@ -1304,7 +1304,13 @@ module Bundler end begin - needed_git_sources.each {|source| preload_git_source_worker.enq(source) } + # Sources from the same repository share one cache directory, and + # running git in that directory from two workers at once makes the + # fetches collide. Preload sources that share a cache in one job, one + # after the other, and run the separate caches in parallel. + needed_git_sources.group_by(&:cache_path).each_value do |sources| + preload_git_source_worker.enq(sources) + end ensure preload_git_source_worker.stop end diff --git a/spec/bundler/bundler/definition_spec.rb b/spec/bundler/bundler/definition_spec.rb index 8c4a5a0331e..d263afa0902 100644 --- a/spec/bundler/bundler/definition_spec.rb +++ b/spec/bundler/bundler/definition_spec.rb @@ -21,6 +21,45 @@ RSpec.describe Bundler::Definition do end end + describe "#preload_git_sources" do + before do + allow(Bundler::SharedHelpers).to receive(:find_gemfile) { bundled_app_gemfile } + allow(Bundler.settings).to receive(:installation_parallelization).and_return(4) + end + + subject { Bundler::Definition.new(bundled_app_lock, [], Bundler::SourceList.new, {}) } + + it "never preloads two sources sharing a git cache at the same time" do + skip "git sources are preloaded one by one before Ruby 3.3" if Gem.ruby_version < Gem::Version.new("3.3") + + lock = Mutex.new + running = Hash.new(0) + max_running = Hash.new(0) + max_total = 0 + + sources = %w[monorepo monorepo monorepo other].map do |repo| + source = double("git source #{repo}", cache_path: Pathname.new("/cache/git/#{repo}")) + allow(source).to receive(:specs) do + lock.synchronize do + running[repo] += 1 + max_running[repo] = [max_running[repo], running[repo]].max + max_total = [max_total, running.values.sum].max + end + sleep 0.1 + lock.synchronize { running[repo] -= 1 } + end + source + end + allow(subject).to receive(:needed_git_sources).and_return(sources) + + subject.send(:preload_git_sources) + + sources.each {|source| expect(source).to have_received(:specs).once } + expect(max_running["monorepo"]).to eq(1) + expect(max_total).to eq(2) + end + end + describe "#lock" do before do allow(Bundler::SharedHelpers).to receive(:find_gemfile) { bundled_app_gemfile }