Re: [O] [PATCH] Re: Problems with org publish cache checking

[Matt Lundin]

Nicolas Goaziou <address@hidden> writes:

> Hello,
>
> Matt Lundin <address@hidden> writes:
>
>> OK, I've worked up a patch that solves several of these issues. The
>> basic idea is to check when publishing an org file whether it includes
>> other org files and then to store that data in the cache. That way,
>> org-publish-cache-file-needs-publishing does not need to open each
>> buffer but rather can compare the stored timestamp data against the
>> actual modified times of the included files.
>
> This is much better, indeed. Thank you.
>
> One suggestion: wouldn't it make sense to also apply check to SETUPFILE
> keywords?

Yes, that's a great idea. I've added it to the patch.

One caveat: this patch does not implement recursive checking of included
files (i.e., included files that include other files), but this could be
added in the future.

Thanks,
Matt

>From cc66884f2836bee1203d06618828c0339ea2e4e2 Mon Sep 17 00:00:00 2001
From: Matt Lundin <address@hidden>
Date: Thu, 26 Nov 2015 19:22:00 -0600
Subject: [PATCH] Speed up publishing by caching included file data

* lisp/ox-publish.el: (org-publish-cache-get-included-files): New function
  (org-publish-org-to): Use new function
  (org-publish-cache-file-needs-publishing): Use cache instead of
  visiting every file in a project.

Org-publish can now quickly determine a) whether an org source includes
other files (either via #+INCLUDE or #+SETUPFILE) and b) whether those
files have changed. This speeds up the publishing process and makes
tracking of changes in included files more reliable.
---
 lisp/ox-publish.el | 70 ++++++++++++++++++++++++++++++------------------------
 1 file changed, 39 insertions(+), 31 deletions(-)

diff --git a/lisp/ox-publish.el b/lisp/ox-publish.el
index 90f307c..02eb06d 100644
--- a/lisp/ox-publish.el
+++ b/lisp/ox-publish.el
@@ -574,6 +574,7 @@ Return output file name."
             (let ((output-file
                    (org-export-output-file-name extension nil pub-dir))
                   (body-p (plist-get plist :body-only)))
+              (when org-publish-cache (org-publish-cache-get-included-files))
               (org-export-to-file backend output-file
                 nil nil nil body-p
                 ;; Add `org-publish--collect-references' and
@@ -1221,42 +1222,49 @@ If FREE-CACHE, empty the cache."
 (defun org-publish-cache-file-needs-publishing
     (filename &optional pub-dir pub-func _base-dir)
   "Check the timestamp of the last publishing of FILENAME.
-Return non-nil if the file needs publishing.  Also check if
-any included files have been more recently published, so that
-the file including them will be republished as well."
+Return non-nil if the file needs publishing.  Also use the cache
+to check if any included files have changed, so that the file
+including them will be republished."
   (unless org-publish-cache
     (error
      "`org-publish-cache-file-needs-publishing' called, but no cache present"))
-  (let* ((case-fold-search t)
-        (key (org-publish-timestamp-filename filename pub-dir pub-func))
+  (let* ((key (org-publish-timestamp-filename filename pub-dir pub-func))
         (pstamp (org-publish-cache-get key))
-        (org-inhibit-startup t)
-        (visiting (find-buffer-visiting filename))
-        included-files-ctime buf)
-    (when (equal (file-name-extension filename) "org")
-      (setq buf (find-file (expand-file-name filename)))
-      (with-current-buffer buf
-       (goto-char (point-min))
-       (while (re-search-forward "^[ \t]*#\\+INCLUDE:" nil t)
-         (let* ((element (org-element-at-point))
-                (included-file
-                 (and (eq (org-element-type element) 'keyword)
-                      (let ((value (org-element-property :value element)))
-                        (and value
-                             (string-match "^\\(\".+?\"\\|\\S-+\\)" value)
-                             ;; Ignore search suffix.
-                             (car (split-string
-                                   (org-remove-double-quotes
-                                    (match-string 1 value)))))))))
-           (when included-file
-             (push (org-publish-cache-ctime-of-src
-                    (expand-file-name included-file))
-                   included-files-ctime)))))
-      (unless visiting (kill-buffer buf)))
+        (ctime (when pstamp (org-publish-cache-ctime-of-src filename))))
     (or (null pstamp)
-       (let ((ctime (org-publish-cache-ctime-of-src filename)))
-         (or (< pstamp ctime)
-             (cl-some (lambda (ct) (< ctime ct)) included-files-ctime))))))
+       (< pstamp ctime)
+       (cl-some (lambda (incl)
+                  ;; See if cached time is before modification time.
+                  (< (cdr incl)
+                     (org-publish-cache-ctime-of-src (car incl))))
+         (org-publish-cache-get-file-property filename :includes)))))
+
+(defun org-publish-cache-get-included-files ()
+  "Get data about included files in current buffer.
+Store file names and modification times in cache. Also store data
+about setupfiles."
+  (let ((case-fold-search t)
+       included)
+    (save-excursion
+      (goto-char (point-min))
+      (while (re-search-forward "^[ \t]*#\\+\\(INCLUDE\\|SETUPFILE\\):" nil t)
+       (let* ((element (org-element-at-point))
+              (included-file
+               (and (eq (org-element-type element) 'keyword)
+                    (let ((value (org-element-property :value element)))
+                      (and value
+                           (string-match "^\\(\".+?\"\\|\\S-+\\)" value)
+                           ;; Ignore search suffix.
+                           (car (split-string
+                                 (org-remove-double-quotes
+                                  (match-string 1 value)))))))))
+         (when included-file
+           (let ((iname (expand-file-name included-file)))
+             (push (cons iname (org-publish-cache-ctime-of-src
+                                (expand-file-name iname)))
+                   included))))))
+    (org-publish-cache-set-file-property (buffer-file-name)
+                                        :includes included)))
 
 (defun org-publish-cache-set-file-property
   (filename property value &optional project-name)
-- 
2.6.2