master 707ee6a: EMBA infrastructure improvements for Emacs build testing.

[Teodor Zlatanov]

branch: master
commit 707ee6afe235e1b0f39900d8def0e770003de2db
Author: Ted Zlatanov <tzz@lifelogs.com>
Commit: Ted Zlatanov <tzz@lifelogs.com>

    EMBA infrastructure improvements for Emacs build testing.
    
    * .gitlab-ci.yml: Use job templates and rules. Split tests into
    fast/normal/slow. Make Docker images for each tested platform
    (inotify, filenotify-gio, gnustep). Increase timeout.
    * test/Makefile.in (check-lisp, check-net): Add new testing
    targets.
    * test/README: Document them.
    * test/file-organization.org: Mention test/infra.
    * test/infra/Dockerfile.emba: Add special Docker recipes for EMBA
    testing.
---
 .gitlab-ci.yml             | 196 ++++++++++++++++++++++++++++-----------------
 test/Makefile.in           |   6 ++
 test/README                |   6 ++
 test/file-organization.org |   5 ++
 test/infra/Dockerfile.emba |  71 ++++++++++++++++
 5 files changed, 212 insertions(+), 72 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bc18137..eb88476 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,89 +24,141 @@
 # Maintainer: Ted Zlatanov <tzz@lifelogs.com>
 # URL: https://emba.gnu.org/emacs/emacs
 
-image: debian:stretch
+# Never run merge request pipelines, they usually duplicate push pipelines
+# see 
https://docs.gitlab.com/ee/ci/yaml/README.html#common-if-clauses-for-rules
+workflow:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: never
+    - when: always
 
 variables:
   GIT_STRATEGY: fetch
   EMACS_EMBA_CI: 1
 
-before_script:
-  - apt update -qq
-  - DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y -qq 
-o=Dpkg::Use-Pty=0 libc-dev gcc g++ make autoconf automake libncurses-dev 
gnutls-dev git
+default:
+  image: docker:19.03.12
+  timeout: 3 hours
+  before_script:
+    - docker info
 
-stages:
-  - test
+.job-template:
+  # these will be cached across builds
+  cache:
+    key: ${CI_COMMIT_REF_SLUG}
+    paths: []
+    policy: pull-push
+  # these will be saved for followup builds
+  artifacts:
+    expire_in: 24 hrs
+    paths: []
+      # - "test/**/*.log"
+      # - "**/*.log"
 
-test-all:
-  # This tests also file monitor libraries inotify and inotifywatch.
-  stage: test
-  only:
-    changes:
-      - "Makefile.in"
-      - .gitlab-ci.yml
-      - aclocal.m4
-      - autogen.sh
-      - configure.ac
-      - lib/*.{h,c}
-      - lisp/*.el
-      - lisp/**/*.el
-      - src/*.{h,c}
-      - test/lisp/*.el
-      - test/lisp/**/*.el
-      - test/src/*.el
-  except:
-    changes:
-      # gfilemonitor, kqueue
-      - src/gfilenotify.c
-      - src/kqueue.c
-      # MS Windows
-      - lisp/w32*.el
-      - lisp/term/w32*.el
-      - src/w32*.{h,c}
-      # GNUstep
-      - lisp/term/ns-win.el
-      - src/ns*.{h,m}
-      - src/macfont.{h,m}
+.test-template:
+  rules:
+    - changes:
+        - "**/Makefile.in"
+        - .gitlab-ci.yml
+        - aclocal.m4
+        - autogen.sh
+        - configure.ac
+        - lib/*.{h,c}
+        - lisp/**/*.el
+        - src/*.{h,c}
+        - test/infra/*
+        - test/lisp/**/*.el
+        - test/src/*.el
+    - changes:
+        # gfilemonitor, kqueue
+        - src/gfilenotify.c
+        - src/kqueue.c
+        # MS Windows
+        - "**/w32*"
+        # GNUstep
+        - lisp/term/ns-win.el
+        - src/ns*.{h,m}
+        - src/macfont.{h,m}
+      when: never
+
+  # using the variables for each job
   script:
-    - DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y 
-qq -o=Dpkg::Use-Pty=0 inotify-tools
-    - ./autogen.sh autoconf
-    - ./configure --without-makeinfo
-    - make bootstrap
-    - make check-expensive
+    - docker build --target ${target} -t ${target}:${CI_COMMIT_REF_SLUG} -t 
${target}:${CI_COMMIT_SHA} -f test/infra/Dockerfile.emba .
+    # TODO: with make -j4 several of the tests were failing, for example 
shadowfile-tests, but passed without it
+    - docker run -i --rm -e EMACS_EMBA_CI=${EMACS_EMBA_CI} 
${target}:${CI_COMMIT_SHA} make ${make_params}
+
+stages:
+  - fast
+  - normal
+  - slow
+
+test-fast:
+  stage: fast
+  extends: [.job-template, .test-template]
+  variables:
+    target: emacs-inotify
+    make_params: "-C test check"
+
+test-lisp:
+  stage: normal
+  extends: [.job-template, .test-template]
+  variables:
+    target: emacs-inotify
+    make_params: "-C test check-lisp"
+
+test-net:
+  stage: normal
+  extends: [.job-template, .test-template]
+  variables:
+    target: emacs-inotify
+    make_params: "-C test check-net"
 
 test-filenotify-gio:
-  stage: test
   # This tests file monitor libraries gfilemonitor and gio.
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - lisp/autorevert.el
-      - lisp/filenotify.el
-      - lisp/net/tramp-sh.el
-      - src/gfilenotify.c
-      - test/lisp/autorevert-tests.el
-      - test/lisp/filenotify-tests.el
-  script:
-    - DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y 
-qq -o=Dpkg::Use-Pty=0 libglib2.0-dev libglib2.0-bin libglib2.0-0
-    - ./autogen.sh autoconf
-    - ./configure --without-makeinfo --with-file-notification=gfile
-    - make bootstrap
-    - make -k -C test autorevert-tests filenotify-tests
+  stage: normal
+  extends: [.job-template, .test-template]
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      changes:
+        - "**/Makefile.in"
+        - .gitlab-ci.yml
+        - lisp/autorevert.el
+        - lisp/filenotify.el
+        - lisp/net/tramp-sh.el
+        - src/gfilenotify.c
+        - test/infra/*
+        - test/lisp/autorevert-tests.el
+        - test/lisp/filenotify-tests.el
+  variables:
+    target: emacs-filenotify-gio
+    make_params: "-k -C test autorevert-tests filenotify-tests"
 
 test-gnustep:
-  stage: test
   # This tests the GNUstep build process
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - configure.ac
-      - src/ns*.{h,m}
-      - src/macfont.{h,m}
-      - lisp/term/ns-win.el
-      - nextstep/**/*
-  script:
-    - DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y 
-qq -o=Dpkg::Use-Pty=0 gnustep-devel
-    - ./autogen.sh autoconf
-    - ./configure --without-makeinfo --with-ns
-    - make bootstrap
-    - make install
+  stage: normal
+  extends: [.job-template, .test-template]
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      changes:
+        - "**/Makefile.in"
+        - .gitlab-ci.yml
+        - configure.ac
+        - src/ns*.{h,m}
+        - src/macfont.{h,m}
+        - lisp/term/ns-win.el
+        - nextstep/**/*
+        - test/infra/*
+  variables:
+    target: emacs-gnustep
+    make_params: install
+
+test-all:
+  # This tests also file monitor libraries inotify and inotifywatch.
+  stage: slow
+  extends: [.job-template, .test-template]
+  rules:
+    # note there's no "changes" section, so this always runs on a schedule
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+  variables:
+    target: emacs-inotify
+    make_params: check-expensive
diff --git a/test/Makefile.in b/test/Makefile.in
index fc40dad..2d595d9 100644
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -246,6 +246,12 @@ endef
 
 $(foreach test,${TESTS},$(eval $(call test_template,${test})))
 
+# Get the tests for only a specific directory
+NET_TESTS := $(patsubst %.el,%,$(wildcard lisp/net/*.el))
+LISP_TESTS := $(patsubst %.el,%,$(wildcard lisp/*.el))
+check-net: ${NET_TESTS}
+check-lisp: ${LISP_TESTS}
+
 ifeq (@HAVE_MODULES@, yes)
 # -fPIC is a no-op on Windows, but causes a compiler warning
 ifeq ($(SO),.dll)
diff --git a/test/README b/test/README
index ec566cb..38f4a10 100644
--- a/test/README
+++ b/test/README
@@ -39,6 +39,12 @@ The Makefile in this directory supports the following 
targets:
 * make check-all
   Like "make check", but run all tests.
 
+* make check-lisp
+  Like "make check", but run only the tests in test/lisp/*.el
+
+* make check-net
+  Like "make check", but run only the tests in test/lisp/net/*.el
+
 * make <filename>  -or-  make <filename>.log
   Run all tests declared in <filename>.el.  This includes expensive
   tests.  In the former case the output is shown on the terminal, in
diff --git a/test/file-organization.org b/test/file-organization.org
index 64c0755..efc3545 100644
--- a/test/file-organization.org
+++ b/test/file-organization.org
@@ -57,3 +57,8 @@ directory called ~test/lisp/progmodes/flymake-resources~.
 No guidance is given for the organization of resource files inside the
 ~-resources~ directory; files can be organized at the author's
 discretion.
+
+** Testing Infrastructure Files
+
+Files used to support testing infrastructure such as EMBA should be
+placed in ~infra~.
diff --git a/test/infra/Dockerfile.emba b/test/infra/Dockerfile.emba
new file mode 100644
index 0000000..dd41982
--- /dev/null
+++ b/test/infra/Dockerfile.emba
@@ -0,0 +1,71 @@
+# Copyright (C) 2021 Free Software Foundation, Inc.
+#
+#  This file is part of GNU Emacs.
+#
+#  GNU Emacs is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  GNU Emacs is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+# GNU Emacs support for the GitLab-specific build of Docker images.
+
+# The presence of this file does not imply any FSF/GNU endorsement of
+# Docker or any other particular tool.  Also, it is intended for
+# evaluation purposes, thus possibly temporary.
+
+# Maintainer: Ted Zlatanov <tzz@lifelogs.com>
+# URL: https://emba.gnu.org/emacs/emacs
+
+FROM debian:stretch as emacs-base
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends -o=Dpkg::Use-Pty=0 \
+      libc-dev gcc g++ make autoconf automake libncurses-dev gnutls-dev git \
+    && rm -rf /var/lib/apt/lists/*
+
+FROM emacs-base as emacs-inotify
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends -o=Dpkg::Use-Pty=0 
inotify-tools \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY . /checkout
+WORKDIR /checkout
+RUN ./autogen.sh autoconf
+RUN ./configure --without-makeinfo
+RUN make bootstrap
+RUN make -j4
+
+FROM emacs-base as emacs-filenotify-gio
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends -o=Dpkg::Use-Pty=0 
libglib2.0-dev libglib2.0-bin libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY . /checkout
+WORKDIR /checkout
+RUN ./autogen.sh autoconf
+RUN ./configure --without-makeinfo --with-file-notification=gfile
+RUN make bootstrap
+RUN make -j4
+
+FROM emacs-base as emacs-gnustep
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends -o=Dpkg::Use-Pty=0 
gnustep-devel \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY . /checkout
+WORKDIR /checkout
+RUN ./autogen.sh autoconf
+RUN ./configure --without-makeinfo --with-ns
+RUN make bootstrap
+RUN make -j4