45/118: nix-copy-closure -s: Do substitutions via ‘nix-store --serve’

[Ludovic Courtès]

civodul pushed a commit to branch nix
in repository guix.

commit 7bb632b02464febd8806ef4bd3fa0ac107f52650
Author: Eelco Dolstra <address@hidden>
Date:   Thu Jul 10 20:43:04 2014 +0200

    nix-copy-closure -s: Do substitutions via ‘nix-store --serve’
    
    This means we no longer need an SSH master connection, since we only
    execute a single command on the remote host.
---
 perl/lib/Nix/CopyClosure.pm              |   19 +++++++++++--------
 scripts/nix-copy-closure.in              |    5 ++---
 src/download-via-ssh/download-via-ssh.cc |    1 +
 src/nix-store/nix-store.cc               |   29 +++++++++++++++++++++++++++++
 4 files changed, 43 insertions(+), 11 deletions(-)

diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm
index 53da720..779d743 100644
--- a/perl/lib/Nix/CopyClosure.pm
+++ b/perl/lib/Nix/CopyClosure.pm
@@ -29,16 +29,12 @@ sub copyTo {
     my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor,
         $includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = 
@_;
 
+    $useSubstitutes = 0 if $dryRun;
+
     # Get the closure of this path.
     my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs,
         map { followLinksToStorePath $_ } @{$storePaths})));
 
-    # Optionally use substitutes on the remote host.
-    if (!$dryRun && $useSubstitutes) {
-        system "ssh $sshHost @{$sshOpts} nix-store -r --ignore-unknown 
@closure";
-        # Ignore exit status because this is just an optimisation.
-    }
-
     # Start ‘nix-store --serve’ on the remote host.
     my ($from, $to);
     my $pid = open2($from, $to, "ssh $sshHost @{$sshOpts} nix-store --serve 
--write");
@@ -60,8 +56,9 @@ sub copyTo {
 
     # Send the "query valid paths" command with the "lock" option
     # enabled. This prevents a race where the remote host
-    # garbage-collect paths that are already there.
-    syswrite($to, pack("L<x4L<x4L<x4", 1, 1, scalar @closure)) or die;
+    # garbage-collect paths that are already there. Optionally, ask
+    # the remote host to substitute missing paths.
+    syswrite($to, pack("L<x4L<x4L<x4L<x4", 1, 1, $useSubstitutes, scalar 
@closure)) or die;
     writeString($_, $to) foreach @closure;
 
     # Get back the set of paths that are already valid on the remote host.
@@ -119,6 +116,12 @@ sub oldCopyTo {
     my ($closure, $sshHost, $sshOpts, $storePaths, $compressor, $decompressor,
         $includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = 
@_;
 
+    # Optionally use substitutes on the remote host.
+    if (!$dryRun && $useSubstitutes) {
+        system "ssh $sshHost @{$sshOpts} nix-store -r --ignore-unknown 
@$closure";
+        # Ignore exit status because this is just an optimisation.
+    }
+
     # Ask the remote host which paths are invalid.  Because of limits
     # to the command line length, do this in chunks.  Eventually,
     # we'll want to use ‘--from-stdin’, but we can't rely on the
diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in
index abd3760..05faa63 100755
--- a/scripts/nix-copy-closure.in
+++ b/scripts/nix-copy-closure.in
@@ -81,9 +81,6 @@ while (@ARGV) {
 die "$0: you did not specify a host name\n" unless defined $sshHost;
 
 
-openSSHConnection $sshHost or die "$0: unable to start SSH\n";
-
-
 if ($toMode) { # Copy TO the remote machine.
     Nix::CopyClosure::copyTo(
         $sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor,
@@ -92,6 +89,8 @@ if ($toMode) { # Copy TO the remote machine.
 
 else { # Copy FROM the remote machine.
 
+    openSSHConnection $sshHost or die "$0: unable to start SSH\n";
+
     # Query the closure of the given store paths on the remote
     # machine.  Paths are assumed to be store paths; there is no
     # resolution (following of symlinks).
diff --git a/src/download-via-ssh/download-via-ssh.cc 
b/src/download-via-ssh/download-via-ssh.cc
index 6834634..0fa0897 100644
--- a/src/download-via-ssh/download-via-ssh.cc
+++ b/src/download-via-ssh/download-via-ssh.cc
@@ -59,6 +59,7 @@ static void query(std::pair<FdSink, FdSource> & pipes)
         if (cmd == "have") {
             writeInt(cmdQueryValidPaths, pipes.first);
             writeInt(0, pipes.first); // don't lock
+            writeInt(0, pipes.first); // don't substitute
             writeStrings(tokenized, pipes.first);
             pipes.first.flush();
             PathSet paths = readStrings<PathSet>(pipes.second);
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 28b205b..f31eb0e 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -896,16 +896,42 @@ static void opServe(Strings opFlags, Strings opArgs)
         }
 
         switch (cmd) {
+
             case cmdQueryValidPaths: {
                 bool lock = readInt(in);
+                bool substitute = readInt(in);
                 PathSet paths = readStorePaths<PathSet>(in);
                 if (lock && writeAllowed)
                     for (auto & path : paths)
                         store->addTempRoot(path);
+
+                /* If requested, substitute missing paths. This
+                   implements nix-copy-closure's --use-substitutes
+                   flag. */
+                if (substitute && writeAllowed) {
+                    /* Filter out .drv files (we don't want to build 
anything). */
+                    PathSet paths2;
+                    for (auto & path : paths)
+                        if (!isDerivation(path)) paths2.insert(path);
+                    unsigned long long downloadSize, narSize;
+                    PathSet willBuild, willSubstitute, unknown;
+                    queryMissing(*store, PathSet(paths2.begin(), paths2.end()),
+                        willBuild, willSubstitute, unknown, downloadSize, 
narSize);
+                    /* FIXME: should use ensurePath(), but it only
+                       does one path at a time. */
+                    if (!willSubstitute.empty())
+                        try {
+                            store->buildPaths(willSubstitute);
+                        } catch (Error & e) {
+                            printMsg(lvlError, format("warning: %1%") % 
e.msg());
+                        }
+                }
+
                 writeStrings(store->queryValidPaths(paths), out);
                 out.flush();
                 break;
             }
+
             case cmdQueryPathInfos: {
                 PathSet paths = readStorePaths<PathSet>(in);
                 // !!! Maybe we want a queryPathInfos?
@@ -924,10 +950,12 @@ static void opServe(Strings opFlags, Strings opArgs)
                 out.flush();
                 break;
             }
+
             case cmdDumpStorePath:
                 dumpPath(readStorePath(in), out);
                 out.flush();
                 break;
+
             case cmdImportPaths: {
                 if (!writeAllowed) throw Error("importing paths not allowed");
                 string compression = readString(in);
@@ -966,6 +994,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
                 break;
             }
+
             default:
                 throw Error(format("unknown serve command %1%") % cmd);
         }