[Misc #22206] Derive generated dependencies from Makefile targets

Merged: https://github.com/ruby/ruby/pull/18007

ext/etc/depend

@@ -1,5 +1,3 @@
-# mkdepend: generated constdefs.h
-
 constdefs.h : $(srcdir)/mkconstants.rb
 	@echo "generating constant definitions"
 	@$(RUBY) $(srcdir)/mkconstants.rb -o constdefs.h

ext/rbconfig/sizeof/depend

@@ -2,9 +2,6 @@
 
 # mkdepend: scan limits.c => template/limits.c.tmpl
 # mkdepend: scan sizes.c => template/sizes.c.tmpl
-# mkdepend: generated limits.c
-# mkdepend: generated sizes.c
-
 limits.c: $(top_srcdir)/tool/generic_erb.rb $(top_srcdir)/template/limits.c.tmpl
 	$(Q) $(RUBY) $(top_srcdir)/tool/generic_erb.rb --output=$@ \
 		$(top_srcdir)/template/limits.c.tmpl \

ext/ripper/depend

@@ -8,9 +8,6 @@ LRAMA = $(BASERUBY) $(top_srcdir)/tool/lrama/exe/lrama
 # mkdepend: define ripper.y => RIPPER
 # mkdepend: scan ripper.y => parse.y
 # mkdepend: scan ripper_init.c => ext/ripper/ripper_init.c.tmpl
-# mkdepend: generated eventids1.h
-# mkdepend: generated eventids2.h
-# mkdepend: generated ripper_init.h
 # mkdepend: depends eventids2table.c => {$(VPATH)}eventids2table.c
 # mkdepend: depends ripper.c => {$(VPATH)}ripper.c
 # mkdepend: depends ripper_init.c => {$(VPATH)}ripper_init.c

ext/socket/depend

@@ -1,6 +1,3 @@
-# mkdepend: generated constdefs.c
-# mkdepend: generated constdefs.h
-
 srcs: constdefs.h constdefs.c
 
 getnameinfo.o: getnameinfo.c $(arch_hdrdir)/ruby/config.h addrinfo.h sockport.h rubysocket.h

tool/mkdepend.rb

@@ -15,9 +15,6 @@
 #     `*` and `**` patterns are expanded from matching SOURCE files and
 #     substituted into NAME.  Both sides must have the same pattern count.
 #
-#   # mkdepend: generated NAME
-#     Treat NAME as generated without reading it.
-#
 #   # mkdepend: depends NAME => DEPENDENCY...
 #     Associate NAME with dependencies.  They replace NAME when it is
 #     included and are added when NAME is a source being scanned.  Make
@@ -40,7 +37,7 @@ TOP_SRCDIR = File.expand_path("..", __dir__)
 
 class Mkdepend
   Declarations = Struct.new(
-    :scan, :generated, :dependencies, :defines, :undefines,
+    :scan, :dependencies, :defines, :undefines,
   )
 end
 
@@ -48,13 +45,13 @@ class Mkdepend::Scanner
   attr_reader :unresolved
 
   def initialize(root:, include_dirs:, quote_dirs: [], macros: {},
-                 generated: [], aliases: {}, dependencies: {}, defined: [],
+                 targets: [], aliases: {}, dependencies: {}, defined: [],
                  undefined: [])
     @root = File.expand_path(root)
     @include_dirs = include_dirs.map {|dir| File.expand_path(dir, @root)}
     @quote_dirs = quote_dirs.map {|dir| File.expand_path(dir, @root)}
     @macros = macros
-    @generated = generated.to_set
+    @targets = targets.to_set
     @aliases = aliases
     @dependencies = dependencies
     @defined = defined.to_set
@@ -148,7 +145,7 @@ class Mkdepend::Scanner
     when /\A([A-Za-z_]\w*)/
       name = $1
       unless @aliases.key?(name) || @dependencies.key?(name) ||
-          @generated.include?(name)
+          @targets.include?(name)
         name = @macros[name]
       end
       quoted = true
@@ -171,7 +168,7 @@ class Mkdepend::Scanner
       dependencies.add(name)
     elsif path = resolve(name, current, quoted)
       visit(path, dependencies, visited)
-    elsif @generated.include?(name)
+    elsif @targets.include?(name)
       dependencies.add(name)
     else
       @unresolved.add(name)
@@ -202,6 +199,8 @@ class Mkdepend
   def initialize(root: TOP_SRCDIR)
     @root = File.expand_path(root)
     @dependency_declarations = {}
+    @dependency_targets = {}
+    @dependency_contents = {}
   end
 
   def expand_scan_declaration(name, source)
@@ -227,12 +226,13 @@ class Mkdepend
     end
   end
 
-  def parse_dependency_declarations(path)
-    declarations = Declarations.new({}, {}, {}, {}, {})
-    return declarations unless File.file?(path)
+  def parse_dependency_declarations(path, content = nil)
+    declarations = Declarations.new({}, {}, {}, {})
+    content ||= File.read(path) if File.file?(path)
+    return declarations unless content
 
     lineno = 0
-    File.foreach(path) do |line|
+    content.each_line do |line|
       lineno += 1
       case line[/\A#\s*mkdepend:\s*\K.*/]
       when nil
@@ -243,8 +243,6 @@ class Mkdepend
           raise "#{path}:#{lineno}: invalid mkdepend declaration: #{line.strip}"
         end
         declarations.scan.update(scan)
-      when /\Agenerated\s+(\S+)\s*\z/
-        declarations.generated[$1] = nil
       when /\Adepends\s+(\S+)\s*=>\s*(.+?)\s*\z/
         name, dependencies = $1, $2.split
         if declarations.dependencies.key?(name)
@@ -262,18 +260,32 @@ class Mkdepend
     declarations
   end
 
-  def dependency_declarations(input = nil, source: nil)
+  def dependency_file_content(path, content = nil)
+    path = File.expand_path(path, @root)
+    if content
+      @dependency_contents[path] = content
+    elsif @dependency_contents.key?(path)
+      @dependency_contents[path]
+    elsif File.file?(path)
+      @dependency_contents[path] = File.read(path)
+    end
+  end
+
+  def dependency_declarations(input = nil, source: nil, content: nil)
     input ||= dependency_input(source)
     input = relative_source(input || "depend")
     @dependency_declarations[input] ||= begin
-      global = parse_dependency_declarations(File.join(@root, "depend"))
       if input == "depend"
-        global
+        path = File.join(@root, input)
+        parse_dependency_declarations(path, dependency_file_content(path, content))
       else
-        local = parse_dependency_declarations(File.join(@root, input))
+        global = dependency_declarations("depend")
+        path = File.absolute_path?(input) ? input : File.join(@root, input)
+        local = parse_dependency_declarations(
+          path, dependency_file_content(path, content),
+        )
         Declarations.new(
           global.scan.merge(local.scan),
-          global.generated.merge(local.generated),
           global.dependencies.merge(local.dependencies),
           global.defines.merge(local.defines),
           global.undefines.merge(local.undefines),
@@ -307,6 +319,34 @@ class Mkdepend
     [key, map[key]] if key
   end
 
+  def dependency_targets(input, content = nil)
+    input = relative_source(input)
+    @dependency_targets[input] ||= begin
+      targets = []
+      path = File.absolute_path?(input) ? input : File.join(@root, input)
+      content = dependency_file_content(path, content)
+      if content
+        content.each_line do |line|
+          if line.start_with?(MARK_START)..line.start_with?(MARK_END)
+            next
+          elsif target = line[/\A[^\s#][^:=]*?(?=\s*:(?!=))/]
+            targets.concat(target.split)
+          end
+        end
+      end
+      targets.uniq
+    end
+  end
+
+  def dependency_target(name, input)
+    input = relative_source(input)
+    targets = dependency_targets(input)
+    candidates = [name]
+    dir = File.dirname(input)
+    candidates << name.delete_prefix(dir + "/") unless dir == "."
+    candidates.uniq.find {|candidate| targets.include?(candidate)}
+  end
+
   def relative_source(path)
     expanded = File.expand_path(path)
     expanded = File.expand_path(path, @root) unless File.exist?(expanded)
@@ -356,9 +396,8 @@ class Mkdepend
       file = relative_source(file)
       dep = if file.start_with?('$(', '{$(')
         file
-      elsif generated = declaration(declarations.generated, file,
-                                     declaration_input)
-        generated[0]
+      elsif target = dependency_target(file, declaration_input)
+        target
       elsif extension_dir
         extension_dependency(file, extension_dir)
       else
@@ -403,11 +442,11 @@ class Mkdepend
     "{$(VPATH)}"
   end
 
-  def generated_source?(source, input = nil)
+  def dependency_source?(source, input = nil)
     declarations = dependency_declarations(input, source: source)
     declaration(declarations.scan, source, input) ||
-      declaration(declarations.generated, source, input) ||
-      declaration(declarations.dependencies, source, input)
+      declaration(declarations.dependencies, source, input) ||
+      dependency_target(source, input || dependency_input(source))
   end
 
   def dependency_scanner(src, declarations, input)
@@ -428,7 +467,7 @@ class Mkdepend
       ],
       quote_dirs: [File.join(@root, "include/ruby")],
       macros: macros,
-      generated: declarations.generated.keys,
+      targets: dependency_targets(input),
       aliases: declarations.scan,
       dependencies: declarations.dependencies,
       defined: defined,
@@ -508,7 +547,11 @@ class Mkdepend
   def dependency_files
     Dir.chdir(@root) do
       %w[depend enc/depend].concat(Dir.glob("ext/**/depend")).select do |file|
-        File.file?(file) && File.read(file).include?(MARK_START)
+        next false unless File.file?(file)
+        content = File.read(file)
+        next false unless content.include?(MARK_START)
+        dependency_file_content(file, content)
+        true
       end
     end
   end
@@ -548,7 +591,7 @@ class Mkdepend
 
     missing = "missing/#{src}"
     return missing if File.file?(File.join(@root, missing))
-    return src if generated_source?(src, input)
+    return src if dependency_source?(src, input)
 
     raise "source file not found: #{src} in #{input}"
   end
@@ -620,7 +663,9 @@ class Mkdepend
       if input.end_with?(".c", ".y")
         out.puts makedepend(input)
       else
-        deps = File.read(input)
+        deps = dependency_file_content(input) || File.read(input)
+        dependency_declarations(input, content: deps)
+        dependency_targets(input, deps)
         match = MARK_SECTION.match(deps)
         next unless match
         raise "missing #{MARK_END} in #{input}" unless match.begin(1)

tool/test/test_mkdepend.rb

@@ -49,7 +49,7 @@ class TestMkdepend < Test::Unit::TestCase
         root: dir,
         include_dirs: [dir],
         macros: {'GENERATED_HEADER' => 'generated.h'},
-        generated: %w[parse.h],
+        targets: %w[parse.h],
         aliases: {'aliased.h' => 'aliased.h.erb'},
         dependencies: {'generated.h' => %w[virtual.h]},
         defined: %w[ENABLED],
@@ -141,6 +141,14 @@ class TestMkdepend < Test::Unit::TestCase
     )
     console = mkdepend.dependency_declarations('ext/io/console/depend')
     assert_equal(%w[$(VK_HEADER)], console.dependencies['win32_vk.inc'])
+    assert_include(
+      mkdepend.dependency_targets('ext/socket/depend'),
+      'constdefs.h',
+    )
+    assert_include(
+      mkdepend.dependency_targets('ext/ripper/depend'),
+      'eventids1.h',
+    )
     assert_path_not_exist(File.expand_path('../mkdepend', __dir__))
   end
 
@@ -201,10 +209,9 @@ class TestMkdepend < Test::Unit::TestCase
       input = File.join(dir, 'ext/example/depend')
       File.write(input, <<~DEPEND)
         # mkdepend: scan generated.c => template.c
-        # mkdepend: generated generated.h
-        # mkdepend: generated vpath.inc
         # mkdepend: depends vpath.inc => {$(VPATH)}vpath.inc
         # mkdepend: depends selected.inc => $(SELECTED_HEADER)
+        generated.h: template.c
         #{MARK_START}
         generated.o: generated.c
         #{MARK_END}
@@ -229,6 +236,7 @@ class TestMkdepend < Test::Unit::TestCase
       path = File.join(dir, 'depend')
       [
         '# mkdepend: depends ignored.h =>',
+        '# mkdepend: generated generated.h',
         '# mkdepend: generated generated.h => $(GENERATED_HEADER)',
         '# mkdepend: unknown header.h',
         '# mkdepend: scan generated/*.c => template.c.erb',
@@ -263,6 +271,26 @@ class TestMkdepend < Test::Unit::TestCase
     end
   end
 
+  def test_dependency_metadata_uses_passed_content
+    Dir.mktmpdir('mkdepend-declarations') do |dir|
+      path = File.join(dir, 'missing-depend')
+      content = <<~DEPEND
+        # mkdepend: depends generated.h => $(GENERATED_HEADER)
+        generated.h: template.h
+        #{MARK_START}
+        generated.o: generated.c
+        #{MARK_END}
+      DEPEND
+      declarations = Mkdepend.new(root: dir).
+        parse_dependency_declarations(path, content)
+      assert_equal(%w[$(GENERATED_HEADER)], declarations.dependencies['generated.h'])
+      assert_equal(
+        %w[generated.h],
+        Mkdepend.new(root: dir).dependency_targets(path, content),
+      )
+    end
+  end
+
   def test_dependency_files
     files = mkdepend.dependency_files
     assert_include(files, 'depend')
@@ -271,6 +299,24 @@ class TestMkdepend < Test::Unit::TestCase
     assert(files.all? {|file| File.read(File.join(TOP_SRCDIR, file)).include?(MARK_START)})
   end
 
+  def test_dependency_files_caches_contents
+    Dir.mktmpdir('mkdepend-files') do |dir|
+      path = File.join(dir, 'depend')
+      ignored = File.join(dir, 'ext/example/depend')
+      FileUtils.mkdir_p(File.dirname(ignored))
+      content = "#{MARK_START}\n#{MARK_END}\n"
+      File.write(path, content)
+      File.write(ignored, "manual: rule\n")
+      mkdepend = Mkdepend.new(root: dir)
+
+      assert_equal(%w[depend], mkdepend.dependency_files)
+      File.unlink(path)
+      File.unlink(ignored)
+      assert_equal(content, mkdepend.dependency_file_content(path))
+      assert_nil(mkdepend.dependency_file_content(ignored))
+    end
+  end
+
   def test_unicode_header
     assert_equal(
       ['$(UNICODE_HDR_DIR)/name2ctype.h'],