[lmi-commits] [lmi] master 3280f70 2/2: Save a sorted list of regressions

[Greg Chicares]

branch: master
commit 3280f708918a5b646e15f761c632dc8147834084
Author: Gregory W. Chicares <gchicares@sbcglobal.net>
Commit: Gregory W. Chicares <gchicares@sbcglobal.net>

    Save a sorted list of regressions
    
    Added "$(test_dir)/regressions.tsv" to the 'system_test' recipe. This
    TSV file is readable as either a text file or a spreadsheet. Its
    contents are sorted in decreasing order by absolute error (primary)
    and relative error (secondary), making it easy to see which files have
    the severest regressions.
    
    Of course, the datestamped "analysis" file can be manipulated in other
    ways by running altered versions of the command in the recipe, without
    without re-running the same regression test. For example, this command
    sorts in the opposite order, switching the primary and secondary sort
    keys, and suppresses lines reporting identical testcases:
    
    < /analysis-[datestamp] sed \
      -e 's/   Summary: max abs diff: /\t/' \
      -e 's/ max rel err:  /\t/' \
      -e'/0\t0/d' \
      | sort --key=3g --key=2g \
      2>&1 |less
---
 workhorse.make | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/workhorse.make b/workhorse.make
index 794df3c..9b1b5bf 100644
--- a/workhorse.make
+++ b/workhorse.make
@@ -1441,12 +1441,15 @@ test_result_suffixes := test test0 test1 
monthly_trace.* mec.tsv mec.xml gpt.tsv
 # names to accumulate history in $(test_dir), which permits comparison
 # across several versions. An undatestamped copy of the md5sums is
 # made for storage in a version-control system (which naturally keeps
-# historical versions in its own way).
+# historical versions in its own way). An undatestamped copy of the
+# analysis, stripped of descriptive text and sorted in descending
+# order, is saved as 'regressions.tsv'.
 
-system_test_analysis := $(test_dir)/analysis-$(yyyymmddhhmm)
-system_test_diffs    := $(test_dir)/diffs-$(yyyymmddhhmm)
-system_test_md5sums  := $(test_dir)/md5sums-$(yyyymmddhhmm)
-system_test_md5sums2 := $(test_dir)/md5sums
+system_test_analysis    := $(test_dir)/analysis-$(yyyymmddhhmm)
+system_test_regressions := $(test_dir)/regressions.tsv
+system_test_diffs       := $(test_dir)/diffs-$(yyyymmddhhmm)
+system_test_md5sums     := $(test_dir)/md5sums-$(yyyymmddhhmm)
+system_test_md5sums2    := $(test_dir)/md5sums
 
 %.cns:  test_emission := emit_quietly,emit_test_data
 %.ill:  test_emission := emit_quietly,emit_test_data
@@ -1506,6 +1509,11 @@ system_test: $(datadir)/configurable_settings.xml 
$(touchstone_md5sums) install
          -e '/rel err.*e-0*1[5-9]/d' \
          -e '/abs.*0\.00.*rel/d' \
          -e '/abs diff: 0 /d'
+       @-< $(system_test_analysis) $(SED) \
+         -e 's/   Summary: max abs diff: /\t/' \
+         -e 's/ max rel err:  /\t/' \
+         | $(SORT) --key=2gr --key=3gr \
+         > $(system_test_regressions)
        @$(DIFF) --brief $(system_test_md5sums) $(touchstone_md5sums) \
          && $(ECHO) "All `<$(touchstone_md5sums) $(WC) -l` files match." \
          || $(MAKE) --file=$(this_makefile) system_test_discrepancies