[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[shepherd] 05/05: Add log rotation service.
|
From: |
Ludovic Courtès |
|
Subject: |
[shepherd] 05/05: Add log rotation service. |
|
Date: |
Sat, 18 May 2024 16:59:04 -0400 (EDT) |
civodul pushed a commit to branch devel
in repository shepherd.
commit 0484726801c2b5c1b7deecb9a96054c2c510ac71
Author: Ludovic Courtès <ludo@gnu.org>
AuthorDate: Sat May 18 11:41:53 2024 +0200
Add log rotation service.
* modules/shepherd/service/log-rotation.scm,
tests/services/log-rotation-internal.scm,
tests/services/log-rotation.sh: New files.
* Makefile.am (dist_servicesub_DATA): Add the module.
(TESTS): Add the tests.
* configure.ac: Add ‘--with-gzip’ and ‘--with-zstd’. Substitute ‘GZIP’
and ‘ZSTD’.
* modules/shepherd/system.scm.in (%gzip-program, %zstd-program): New
variables.
* po/POTFILES.in: Add log-rotation.scm.
* .guix/modules/shepherd-package.scm (shepherd)[arguments]: Pass
‘--with-gzip’ and ‘--with-zstd’.
[inputs]: Add GZIP and ZSTD.
---
.guix/modules/shepherd-package.scm | 13 +-
Makefile.am | 5 +-
configure.ac | 21 +++
doc/shepherd.texi | 79 +++++++++++
modules/shepherd/service/log-rotation.scm | 211 ++++++++++++++++++++++++++++++
modules/shepherd/system.scm.in | 12 +-
po/POTFILES.in | 1 +
tests/services/log-rotation-internal.scm | 74 +++++++++++
tests/services/log-rotation.sh | 100 ++++++++++++++
9 files changed, 512 insertions(+), 4 deletions(-)
diff --git a/.guix/modules/shepherd-package.scm
b/.guix/modules/shepherd-package.scm
index 57b77ca..86f5f36 100644
--- a/.guix/modules/shepherd-package.scm
+++ b/.guix/modules/shepherd-package.scm
@@ -44,6 +44,7 @@
#:use-module (guix modules)
#:use-module (gnu packages)
#:use-module (gnu packages autotools)
+ #:use-module (gnu packages compression)
#:use-module (gnu packages gettext)
#:use-module (gnu packages guile)
#:use-module (gnu packages guile-xyz)
@@ -86,7 +87,15 @@
(source source-checkout)
(build-system gnu-build-system)
(arguments
- (list #:configure-flags #~'("--localstatedir=/var")
+ (list #:configure-flags
+ #~(list "--localstatedir=/var"
+ (string-append "--with-gzip="
+ #$(this-package-input "gzip")
+ "/bin/gzip")
+ (string-append "--with-zstd="
+ #$(this-package-input "zstd")
+ "/bin/zstd"))
+
#:modules '((guix build gnu-build-system)
((guix build guile-build-system)
#:select (target-guile-effective-version))
@@ -127,7 +136,7 @@
(append (map specification->package development-packages)
(list pkg-config guile-3.0-latest
guile-fibers-1.3))) ;for cross-compilation
- (inputs (list guile-3.0-latest guile-fibers-1.3))
+ (inputs (list guile-3.0-latest guile-fibers-1.3 gzip zstd))
(synopsis "System service manager")
(description
"The GNU Shepherd is a daemon-managing daemon, meaning that it supervises
diff --git a/Makefile.am b/Makefile.am
index 19cbf7a..17c79c0 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -49,6 +49,7 @@ nodist_shepherdsub_DATA = \
modules/shepherd/config.scm \
modules/shepherd/system.scm
dist_servicesub_DATA = \
+ modules/shepherd/service/log-rotation.scm \
modules/shepherd/service/monitoring.scm \
modules/shepherd/service/repl.scm \
modules/shepherd/service/timer.scm
@@ -282,7 +283,9 @@ TESTS = \
tests/services/monitoring.sh \
tests/services/repl.sh \
tests/services/timer.sh \
- tests/services/timer-events.scm
+ tests/services/timer-events.scm \
+ tests/services/log-rotation.sh \
+ tests/services/log-rotation-internal.scm
TEST_EXTENSIONS = .sh .scm
EXTRA_DIST += $(TESTS)
diff --git a/configure.ac b/configure.ac
index 4c9a6ae..c992e92 100644
--- a/configure.ac
+++ b/configure.ac
@@ -21,6 +21,27 @@ AC_CANONICAL_HOST
AC_PROG_MKDIR_P
AC_PROG_SED
+dnl Compression programs.
+AC_ARG_WITH([gzip],
+ AS_HELP_STRING([--with-gzip=PROGRAM],
+ [gzip program to use for log file compression]),
+ [GZIP="$withval"],
+ [GZIP="$(type -P gzip || echo gzip)"])
+
+AC_MSG_CHECKING([for gzip])
+AC_MSG_RESULT([$GZIP])
+AC_SUBST([GZIP])
+
+AC_ARG_WITH([zstd],
+ AS_HELP_STRING([--with-zstd=PROGRAM],
+ [zstd program to use for log file compression]),
+ [ZSTD="$withval"],
+ [ZSTD="$(type -P zstd || echo zstd)"])
+
+AC_MSG_CHECKING([for zstd])
+AC_MSG_RESULT([$ZSTD])
+AC_SUBST([ZSTD])
+
dnl The 'timeout' program, introduced in GNU Coreutils 7.0 (2008).
AC_PATH_PROG([TIMEOUT], [timeout], [not-found])
AM_CONDITIONAL([HAVE_TIMEOUT], [test "x$TIMEOUT" != "xnot-found"])
diff --git a/doc/shepherd.texi b/doc/shepherd.texi
index d7268c4..fa6c88f 100644
--- a/doc/shepherd.texi
+++ b/doc/shepherd.texi
@@ -1136,6 +1136,11 @@ When @var{log-file} is true, it names the file to which
the service's
standard output and standard error are redirected. @var{log-file} is
created if it does not exist, otherwise it is appended to.
+@quotation Note
+@xref{Log Rotation Service}, for a service to rotate log files specified
+via the @code{#:log-file} parameter.
+@end quotation
+
Guile's @code{setrlimit} procedure is applied on the entries in
@var{resource-limits}. For example, a valid value would be:
@@ -1920,10 +1925,84 @@ The Shepherd comes with a collection of services that
let you control it
or otherwise extend its functionality. This chapter documents them.
@menu
+* Log Rotation Service:: Tidying up log files.
* Monitoring Service:: Monitoring shepherd resource usage.
* REPL Service:: Interacting with a running shepherd.
@end menu
+@node Log Rotation Service
+@section Log Rotation Service
+
+@cindex log rotation
+@cindex rotating log files of services
+All these services produce many log files; they're useful, for sure, but
+wouldn't it be nice to archive them or even delete them periodically?
+The @dfn{log rotation} service does exactly that. Once you've enabled
+it, it periodically @dfn{rotates} the log files of services---those
+specified via the @code{#:log-file} argument of service constructors
+(@pxref{Service De- and Constructors}).
+
+By ``rotating'' we mean this: if a service produces
+@file{/var/log/my-service.log}, then rotating it consists in
+periodically renaming it and compressing it to obtain, say,
+@file{/var/log/my-service.log.1.gz}---after renaming the @emph{previous}
+@file{my-service.1.log.gz} to @file{my-service.log.2.gz}, and so
+on@footnote{This is comparable to what the venerable logrotate tool
+would do.}. Files older than some configured threshold are deleted
+instead of being renamed. The process is race-free: if the service is
+running, not a single line that it logs is lost during rotation.
+
+To enable the log rotation service, you can add the following lines to
+your configuration file (@pxref{Service Examples}):
+
+@lisp
+(use-modules (shepherd service log-rotation))
+
+(register-services
+ ;; Create a service that rotates log files once a week.
+ (list (log-rotation-service)))
+
+;; Start it.
+(start-service (lookup-service 'log-rotation))
+@end lisp
+
+This creates a @code{log-rotation} service, which is in fact a timed
+service (@pxref{Timers}). By default it rotates logs once a week and
+you can see past and upcoming runs in the usual way:
+
+@example
+herd status log-rotation
+@end example
+
+You can also trigger it explicitly at any time, like so:
+
+@example
+herd trigger log-rotation
+@end example
+
+The default settings should be good for most use cases, but you can
+change them by passing the @code{log-rotation-service} procedure a
+number of arguments---see the reference documentation below.
+
+@deffn {Procedure} log-rotation-service [@var{event}] @
+ [#:provision '(log-rotation)] [#:requirement '()] @
+ [#:compression (%default-log-compression)] @
+ [#:expiry (%default-log-expiry)] @
+ [#:rotation-size-threshold (%default-rotation-size-threshold)]
+Return a timed service that rotates service logs on every occurrence of
+@var{event}, a calendar event.
+
+Compress log files according to @var{method}, which can be one of
+@code{'gzip}, @code{'zstd}, @code{'none}, or a one-argument procedure that
+is passed the file name. Log files smaller than @var{rotation-size-threshold}
+are not rotated; copies older than @var{expiry} seconds are deleted.
+
+Last, @var{provision} and @var{requirement} are lists of symbols specifying
+what the service provides and requires, respectively. Specifying
+@var{requirement} is useful to ensure, for example, that log rotation runs
+only if the service that mounts the file system that hosts log files is up.
+@end deffn
+
@node Monitoring Service
@section Monitoring Service
diff --git a/modules/shepherd/service/log-rotation.scm
b/modules/shepherd/service/log-rotation.scm
new file mode 100644
index 0000000..bbb5ff9
--- /dev/null
+++ b/modules/shepherd/service/log-rotation.scm
@@ -0,0 +1,211 @@
+;; log-rotation.scm -- Rotating service log files.
+;; Copyright (C) 2024 Ludovic Courtès <ludo@gnu.org>
+;;
+;; This file is part of the GNU Shepherd.
+;;
+;; The GNU Shepherd is free software; you can redistribute it and/or modify it
+;; under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 3 of the License, or (at
+;; your option) any later version.
+;;
+;; The GNU Shepherd is distributed in the hope that it will be useful, but
+;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;; GNU General Public License for more details.
+;;
+;; You should have received a copy of the GNU General Public License
+;; along with the GNU Shepherd. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (shepherd service log-rotation)
+ #:use-module (shepherd support)
+ #:use-module (shepherd logger)
+ #:autoload (shepherd service timer) (calendar-event
+ make-timer-constructor
+ make-timer-destructor
+ timer-trigger-action)
+ #:autoload (shepherd service) (service
+ service-logger
+ for-each-service)
+ #:autoload (shepherd system) (%gzip-program %zstd-program)
+ #:use-module (srfi srfi-1)
+ #:use-module (srfi srfi-26)
+ #:use-module (srfi srfi-71)
+ #:autoload (ice-9 ftw) (scandir)
+ #:use-module (ice-9 match)
+ #:export (rotate-past-logs
+
+ %default-rotation-size-threshold
+ %default-log-expiry
+ %default-log-compression
+ log-rotation-service))
+
+(define %compression-extensions
+ ;; Compressed file extensions.
+ '("gz" "bz2" "lz" "zst"))
+
+(define (next-rotated-file-name reference file)
+ "Return the file name @var{file} should be renamed to and its rotation
+number (an integer). Return #f and #f if @var{file} is not rotated from the
+@var{reference} file."
+ (define (next-file base n compression)
+ (let ((base (string-join base ".")))
+ (if (and (string=? base (basename reference))
+ (>= n 0)
+ (or (not compression)
+ (member compression %compression-extensions)))
+ (values (string-append (dirname file) "/" base
+ "." (number->string (+ 1 n))
+ (if compression
+ (string-append "." compression)
+ ""))
+ n)
+ (values #f #f))))
+
+ (match (string-split (basename file) #\.)
+ ((base ... (= string->number (? integer? n)) compression)
+ (next-file base n compression))
+ ((base ... (= string->number (? integer? n)))
+ (next-file base n #f))
+ (_
+ (values #f #f))))
+
+(define* (rotate-past-logs log-file
+ #:optional (rotate rename-file))
+ "Rotate previously-rotated archives of @var{log-file}--e.g.,
+@code{\"/var/log/ntpd.log\"}--by calling @var{rotate} with the old and new
+file names."
+ (define directory
+ (dirname log-file))
+
+ (define (file->rotation file)
+ (let* ((file (in-vicinity directory file))
+ (next level (next-rotated-file-name log-file file)))
+ (and next level
+ (list level file next))))
+
+ (let* ((candidates (scandir directory
+ (let ((base (basename log-file)))
+ (lambda (file)
+ (and (not (string=? base file))
+ (string-prefix? base file))))))
+ (logs (filter-map file->rotation candidates)))
+ ;; Rotate logs starting from the highest level.
+ (for-each (match-lambda
+ ((level file next)
+ (rotate file next)))
+ (sort logs
+ (match-lambda*
+ (((level1 . _) (level2 . _))
+ (< level2 level1)))))))
+
+(define %default-rotation-size-threshold
+ ;; Default size in bytes below which log files are not considered for
+ ;; rotation.
+ (make-parameter 8192))
+
+(define %default-log-compression
+ ;; Default log file compression method.
+ (make-parameter 'gzip))
+
+(define (compress-file method file)
+ "Compress @var{file} in place according to @var{method}, which can be either
+a symbol denoting a supported compression method, or a procedure that gets
+called with @var{file}."
+ ;; Note: Delegate compression to a separate process rather than use an
+ ;; in-process compression library to reduce risks of memory corruption or
+ ;; leaks and to reduce the attack surface.
+ (match method
+ ('gzip (system* %gzip-program "-9" file))
+ ('zstd (system* %zstd-program "-9" file))
+ ('none #f)
+ ((? procedure? proc) (proc file))
+ (_ #f)))
+
+(define* (rotate-logs logger #:optional (rotate rename-file)
+ #:key
+ (compression (%default-log-compression))
+ (rotation-size-threshold
+ (%default-rotation-size-threshold)))
+ "Rotate the log file associated with @var{logger}, if any, along with any
+previously-archived log files. Compress the log file of @var{logger}
+according to @var{method}. Call @var{rotate} with the old a new file name for
+each rotation. If the size of the log file is below
+@var{rotation-size-threshold}, do not rotate it."
+ (match (logger-file logger)
+ (#f #f)
+ (file
+ (if (> (stat:size (stat file)) rotation-size-threshold)
+ (begin
+ ;; First rotate past logs, to free "FILE.1".
+ (rotate-past-logs file rotate)
+ ;; Then rename FILE to "FILE.1".
+ (let* ((next (string-append file ".1"))
+ (rotated? (rotate-log-file logger next)))
+ (when rotated?
+ (compress-file compression next)
+ (local-output (l10n "Rotated '~a'.") file))))
+ (local-output
+ (l10n "Not rotating '~a', which is below the ~a B threshold.")
+ file rotation-size-threshold)))))
+
+(define* (rotate-service-logs #:optional (rotate rename-file)
+ #:key (rotation-size-threshold
+ (%default-rotation-size-threshold)))
+ "Rotate the log files of all the currently running services."
+ (for-each-service (lambda (service)
+ (match (service-logger service)
+ (#f #f)
+ (logger (rotate-logs logger rotate
+ #:rotation-size-threshold
+ rotation-size-threshold))))))
+
+(define %default-calendar-event
+ ;; Default calendar event when log rotation is triggered.
+ (calendar-event #:minutes '(0)
+ #:hours '(22)
+ #:days-of-week '(sunday)))
+
+(define %default-log-expiry
+ ;; Default duration in seconds after which log files are deleted.
+ (make-parameter (* 3 30 24 3600)))
+
+(define* (log-rotation-service #:optional
+ (event %default-calendar-event)
+ #:key
+ (provision '(log-rotation))
+ (requirement '())
+ (compression (%default-log-compression))
+ (expiry (%default-log-expiry))
+ (rotation-size-threshold
+ (%default-rotation-size-threshold)))
+ "Return a timed service that rotates service logs on every occurrence of
+@var{event}, a calendar event.
+
+Compress log files according to @var{method}, which can be one of
+@code{'gzip}, @code{'zstd}, @code{'none}, or a one-argument procedure that
+is passed the file name. Log files smaller than @var{rotation-size-threshold}
+are not rotated; copies older than @var{expiry} seconds are deleted.
+
+Last, @var{provision} and @var{requirement} are lists of symbols specifying
+what the service provides and requires, respectively. Specifying
+@var{requirement} is useful to ensure, for example, that log rotation runs
+only if the service that mounts the file system that hosts log files is up."
+ (define (rotate old new)
+ (let ((stat (stat old))
+ (now (current-time)))
+ (if (<= (- now (stat:mtime stat)) expiry)
+ (rename-file old new)
+ (begin
+ (local-output (l10n "Deleting old log file '~a'.") old)
+ (delete-file old)))))
+
+ (define action
+ (lambda ()
+ (rotate-service-logs rotate
+ #:rotation-size-threshold
+ rotation-size-threshold)))
+
+ (service provision
+ #:start (make-timer-constructor event action)
+ #:stop (make-timer-destructor)
+ #:actions (list timer-trigger-action)))
diff --git a/modules/shepherd/system.scm.in b/modules/shepherd/system.scm.in
index ec8f032..4c83175 100644
--- a/modules/shepherd/system.scm.in
+++ b/modules/shepherd/system.scm.in
@@ -1,5 +1,5 @@
;; system.scm -- Low-level operating system interface.
-;; Copyright (C) 2013-2014, 2016, 2018, 2020, 2022-2023 Ludovic Courtès
<ludo@gnu.org>
+;; Copyright (C) 2013-2014, 2016, 2018, 2020, 2022-2024 Ludovic Courtès
<ludo@gnu.org>
;; Copyright (C) 2018 Carlo Zancanaro <carlo@zancanaro.id.au>
;; Copyright (C) 2020 Jan (janneke) Nieuwenhuizen <janneke@gnu.org>
;;
@@ -42,6 +42,8 @@
unblock-signals
set-blocked-signals
with-blocked-signals
+ %gzip-program
+ %zstd-program
without-automatic-finalization))
;; The <sys/reboot.h> constants.
@@ -298,6 +300,14 @@ current thread."
(call-with-blocked-signals signals (lambda () exp ...)))
+;;;
+;;; External programs.
+;;;
+
+(define %gzip-program "@GZIP@")
+(define %zstd-program "@ZSTD@")
+
+
;;;
;;; Guile shenanigans.
;;;
diff --git a/po/POTFILES.in b/po/POTFILES.in
index f53e6b0..9878406 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -6,6 +6,7 @@ modules/shepherd/scripts/herd.scm
modules/shepherd/scripts/reboot.scm
modules/shepherd/support.scm
modules/shepherd/service.scm
+modules/shepherd/service/log-rotation.scm
modules/shepherd/service/monitoring.scm
modules/shepherd/service/repl.scm
modules/shepherd/service/timer.scm
diff --git a/tests/services/log-rotation-internal.scm
b/tests/services/log-rotation-internal.scm
new file mode 100644
index 0000000..1679dc7
--- /dev/null
+++ b/tests/services/log-rotation-internal.scm
@@ -0,0 +1,74 @@
+;; GNU Shepherd --- Test the log rotation service.
+;; Copyright © 2024 Ludovic Courtès <ludo@gnu.org>
+;;
+;; This file is part of the GNU Shepherd.
+;;
+;; The GNU Shepherd is free software; you can redistribute it and/or modify it
+;; under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 3 of the License, or (at
+;; your option) any later version.
+;;
+;; The GNU Shepherd is distributed in the hope that it will be useful, but
+;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;; GNU General Public License for more details.
+;;
+;; You should have received a copy of the GNU General Public License
+;; along with the GNU Shepherd. If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (test-log-rotation-internal)
+ #:use-module (shepherd service log-rotation)
+ #:use-module (srfi srfi-64)
+ #:use-module (ice-9 ftw)
+ #:use-module (ice-9 match))
+
+(define (call-with-temporary-directory proc)
+ (let ((directory (mkdtemp "/tmp/shepherd-logs-XXXXXX")))
+ (dynamic-wind
+ (const #t)
+ (lambda ()
+ (proc directory))
+ (lambda ()
+ (for-each (lambda (file)
+ (delete-file (in-vicinity directory file)))
+ (scandir directory
+ (lambda (file)
+ (not (member file '("." ".."))))))))))
+
+(define (create-file file)
+ (close-port (open-file file "w0")))
+
+
+(test-begin "log-rotation-internal")
+
+;; Ensure that the right files are rotated in the right order.
+(test-equal "rotate-past-logs"
+ '(("foo.log.3.gz" -> "foo.log.4.gz")
+ ("foo.log.2" -> "foo.log.3")
+ ("foo.log.1" -> "foo.log.2")
+ ("bar.log.10.zst" -> "bar.log.11.zst")
+ ("bar.log.1.zst" -> "bar.log.2.zst"))
+ (call-with-temporary-directory
+ (lambda (directory)
+ (let ((result '()))
+ (define (record-rotation old new)
+ (set! result (cons (list old '-> new) result)))
+
+ (for-each (lambda (base)
+ (create-file (in-vicinity directory base)))
+ '("foo.log"
+ "foo.log.1" "foo.log.2" "foo.log.3.gz"
+ "bar.log" "bar.log.1.zst" "bar.log.10.zst"))
+
+ (rotate-past-logs (in-vicinity directory "foo.log")
+ record-rotation)
+ (rotate-past-logs (in-vicinity directory "bar.log")
+ record-rotation)
+ (map (match-lambda
+ ((old '-> new)
+ (and (string=? (dirname old) directory)
+ (string=? (dirname new) directory)
+ (list (basename old) '-> (basename new)))))
+ (reverse result))))))
+
+(test-end "log-rotation-internal")
diff --git a/tests/services/log-rotation.sh b/tests/services/log-rotation.sh
new file mode 100644
index 0000000..10115f3
--- /dev/null
+++ b/tests/services/log-rotation.sh
@@ -0,0 +1,100 @@
+# GNU Shepherd --- Test log rotation.
+# Copyright © 2024 Ludovic Courtès <ludo@gnu.org>
+#
+# This file is part of the GNU Shepherd.
+#
+# The GNU Shepherd is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or (at
+# your option) any later version.
+#
+# The GNU Shepherd is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with the GNU Shepherd. If not, see <http://www.gnu.org/licenses/>.
+
+shepherd --version
+herd --version
+
+socket="t-socket-$$"
+conf="t-conf-$$"
+log="t-log-$$"
+pid="t-pid-$$"
+service_log1="$PWD/t-service-log1-$$"
+service_log2="$PWD/t-service-log2-$$"
+
+herd="herd -s $socket"
+
+trap "cat $log || true; rm -f $socket $conf $log $service_log1* $service_log2*;
+ test -f $pid && kill \`cat $pid\` || true; rm -f $pid" EXIT
+
+cat > "$conf" <<EOF
+(use-modules (shepherd service log-rotation))
+
+(define command
+ '("$SHELL" "-c" "while true; do echo logging things; sleep 0.2; done"))
+
+(%default-rotation-size-threshold 0)
+
+(define services
+ (list (service '(one)
+ #:start (make-forkexec-constructor
+ command
+ #:log-file "$service_log1")
+ #:stop (make-kill-destructor))
+ (service '(two)
+ #:start (make-forkexec-constructor
+ '("sleep" "600")
+ #:log-file "$service_log2")
+ #:stop (make-kill-destructor))
+ (log-rotation-service #:rotation-size-threshold 0)))
+
+(register-services services)
+EOF
+
+rm -f "$pid"
+shepherd -I -s "$socket" -c "$conf" -l "$log" --pid="$pid" &
+
+# Wait till it's ready.
+while ! test -f "$pid" ; do sleep 0.3 ; done
+
+$herd start one
+$herd start two
+$herd start log-rotation
+
+sleep 0.5
+
+test -f "$service_log1"
+test -f "$service_log2"
+
+# First rotation.
+$herd trigger log-rotation
+
+until grep "Rotated " "$log"; do sleep 0.5; done
+
+test -f "$service_log1"
+test -f "$service_log1.1.gz"
+test -f "$service_log2.1.gz" && false
+test -f "$service_log2"
+
+# Second rotation.
+$herd trigger log-rotation
+
+until test -f "$service_log1.2.gz"; do sleep 0.5; done
+until test -f "$service_log1.1.gz"; do sleep 0.5; done
+test -f "$service_log1"
+test -f "$service_log2.1.gz" && false
+
+# Third rotation, with deletion of old log file.
+touch -d "2017-10-01" "$service_log1.2.gz"
+$herd trigger log-rotation
+
+until grep "Deleting .*$service_log1.2.gz" "$log"; do sleep 0.2; done
+until test -f "$service_log1.2.gz"; do sleep 0.2; done
+until test -f "$service_log1.1.gz"; do sleep 0.2; done
+test -f "$service_log1.3.gz" && false
+
+$herd stop log-rotation