branch master updated: Compute lint warnings for packages in chunks

[Christopher Baines]

This is an automated email from the git hooks/post-receive script.

cbaines pushed a commit to branch master
in repository data-service.

The following commit(s) were added to refs/heads/master by this push:
     new aa8c9db  Compute lint warnings for packages in chunks
aa8c9db is described below

commit aa8c9dbffa8bc0f1019e45289eeb19a17dc5af6f
Author: Christopher Baines <mail@cbaines.net>
AuthorDate: Mon Sep 5 08:40:27 2022 +0100

    Compute lint warnings for packages in chunks
    
    In an attempt to reduce the peak memory usage, and avoid running in to the:
    
      Too many heap sections: Increase MAXHINCR or MAX_HEAP_SECTS
    
    issue.
---
 guix-data-service/jobs/load-new-guix-revision.scm | 50 ++++++++++++++---------
 1 file changed, 30 insertions(+), 20 deletions(-)

diff --git a/guix-data-service/jobs/load-new-guix-revision.scm 
b/guix-data-service/jobs/load-new-guix-revision.scm
index cb1d198..6f39bbc 100644
--- a/guix-data-service/jobs/load-new-guix-revision.scm
+++ b/guix-data-service/jobs/load-new-guix-revision.scm
@@ -457,7 +457,7 @@ WHERE job_id = $1")
       "vi_VN.UTF-8"
       "zh_CN.UTF-8"))
 
-  (define (lint-warnings-for-checker checker-name)
+  (define (lint-warnings-for-checker packages checker-name)
     `(lambda (store)
        (let* ((checker (find (lambda (checker)
                                (eq? (lint-checker-name checker)
@@ -590,25 +590,35 @@ WHERE job_id = $1")
               '()
               (with-time-logging (simple-format #f "getting ~A lint warnings"
                                                 name)
-                (format (current-error-port)
-                        "inferior heap size: ~a MiB~%"
-                        (round
-                         (/ (inferior-eval '(assoc-ref (gc-stats) 'heap-size) 
inf)
-                            (expt 2. 20))))
-
-                (let ((warnings
-                       (inferior-eval-with-store
-                        inf
-                        store
-                        (lint-warnings-for-checker name))))
-
-                  ;; Clean the cached store connections, as there are caches
-                  ;; associated with these that take up lots of memory
-                  (inferior-eval
-                   '(when (defined? '%store-table) (hash-clear! %store-table))
-                   inf)
-
-                  warnings))))))
+                (let loop ((packages-chunks
+                            (chunk packages 15000))
+                           (warnings '()))
+
+                  (if (null? packages-chunks)
+                      warnings
+                      (let ((new-warnings
+                             (inferior-eval-with-store
+                              inf
+                              store
+                              (lint-warnings-for-checker (car packages-chunks) 
name))))
+
+                        ;; Clean the cached store connections, as there are 
caches
+                        ;; associated with these that take up lots of memory
+                        (inferior-eval
+                         '(when (defined? '%store-table) (hash-clear! 
%store-table))
+                         inf)
+
+                        (inferior-eval '(gc) inf)
+
+                        (format (current-error-port)
+                                "inferior heap size: ~a MiB~%"
+                                (round
+                                 (/ (inferior-eval '(assoc-ref (gc-stats) 
'heap-size) inf)
+                                    (expt 2. 20))))
+
+                        (loop (cdr packages-chunks)
+                              (append! warnings
+                                       new-warnings))))))))))
       checkers))))
 
 (define (all-inferior-package-derivations store inf packages)