[tools] Fix Python scripts for processing --trace-gc-nvp

The Python scripts used for processing `--trace-gc-nvp` logs expect
that all lines in the trace log have the same fields. For different
types of GCs (full, scavenger, minor-ms) what is actually printed is
however different. With this CL, tools/gc-nvp-to-csv.py now:

1. takes the union of the fields of all lines in the log, appropriately
   merging; and
2. preserves the order of the fields (they were previously arbitrarily
   reordered).

Change-Id: I4a7d40d86698391929e4f0a19c96c03fe0e8c804
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/4714697
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Commit-Queue: Nikolaos Papaspyrou <nikolaos@chromium.org>
Cr-Commit-Position: refs/heads/main@{#89201}

tools/gc-nvp-to-csv.py

@@ -5,31 +5,42 @@
 # found in the LICENSE file.
 
 #
-# This is an utility for generating csv files based on GC traces produced by
+# This is a utility for generating csv files based on GC traces produced by
 # V8 when run with flags --trace-gc --trace-gc-nvp.
 #
-# Usage: gc-nvp-to-csv.py <GC-trace-filename>
+# Usage: gc-nvp-to-csv.py [-h] [filename]
 #
 
 
 # for py2/py3 compatibility
 from __future__ import print_function
 
+import argparse
 import sys
 import gc_nvp_common
 
 
-def process_trace(filename):
-  trace = gc_nvp_common.parse_gc_trace(filename)
-  if len(trace):
-    keys = trace[0].keys()
-    print(', '.join(keys))
+def process_trace(f):
+  trace, keys = gc_nvp_common.parse_gc_trace_with_keys(f)
+  if trace:
+    print(*keys, sep=', ')
     for entry in trace:
-      print(', '.join(map(lambda key: str(entry[key]), keys)))
+      print(*(entry.get(key, '') for key in keys), sep=', ')
 
 
-if len(sys.argv) != 2:
-  print("Usage: %s <GC-trace-filename>" % sys.argv[0])
-  sys.exit(1)
+def main(args):
+  if args.filename:
+    with open(args.filename, 'rt') as f:
+      process_trace(f)
+  else:
+    process_trace(sys.stdin)
 
-process_trace(sys.argv[1])
+
+if __name__ == '__main__':
+  # Command line options.
+  parser = argparse.ArgumentParser(
+      description='Helper script for converting --trace-gc-nvp logs to CSV')
+  parser.add_argument('filename', type=str, nargs='?', help='GC trace log file')
+  args = parser.parse_args()
+  # Call the main function.
+  main(args)

tools/gc_nvp_common.py

@@ -10,23 +10,59 @@
 from __future__ import with_statement
 import re
 
+
 def split_nvp(s):
   t = {}
-  for (name, value) in re.findall(r"([._\w]+)=([-\w]+(?:\.[0-9]+)?)", s):
+  for name, value in re.findall(r"([._\w]+)=([-\w]+(?:\.[0-9]+)?)", s):
     try:
       t[name] = float(value)
     except ValueError:
       t[name] = value
-
   return t
 
 
+def split_nvp_with_keys(s):
+  t, k = {}, []
+  for name, value in re.findall(r"([._\w]+)=([-\w]+(?:\.[0-9]+)?)", s):
+    try:
+      t[name] = float(value)
+    except ValueError:
+      t[name] = value
+    k.append(name)
+  return t, k
+
+
 def parse_gc_trace(input):
+  # If input is string, treat it as a filename and open the file for reading.
+  if isinstance(input, str):
+    with open(input, 'rt') as f:
+      return parse_gc_trace(f)
+  # Otherwise, treat it as a file-like object.
   trace = []
-  with open(input) as f:
-    for line in f:
-      info = split_nvp(line)
-      if info and 'pause' in info and info['pause'] > 0:
-        info['i'] = len(trace)
-        trace.append(info)
+  for line in input:
+    info = split_nvp(line)
+    if info and 'pause' in info and info['pause'] > 0:
+      info['i'] = len(trace)
+      trace.append(info)
   return trace
+
+
+def parse_gc_trace_with_keys(input):
+  # If input is string, treat it as a filename and open the file for reading.
+  if isinstance(input, str):
+    with open(input, 'rt') as f:
+      return parse_gc_trace_with_keys(f)
+  # Otherwise, treat it as a file-like object.
+  trace = []
+  keys = {'i'}
+  key_list = ['i']
+  for line in input:
+    info, info_keys = split_nvp_with_keys(line)
+    if info and 'pause' in info and info['pause'] > 0:
+      info['i'] = len(trace)
+      trace.append(info)
+      for key in info_keys:
+        if key not in keys:
+          keys.add(key)
+          key_list.append(key)
+  return trace, key_list