# # # add_dir "tests/delta_directions" # # add_file "tests/delta_directions/__driver__.lua" # content [4f32096c299d54b83a2f6d83079af05171501107] # # patch "NEWS" # from [6746ecaa1c40d162e80b9342d2de5eb7783440dd] # to [bfb023a41b4fc9dc17a9526e3d58da604e699895] # # patch "database.cc" # from [ddc702beb36a7317da57480b789f64dd0c2f041a] # to [d3ff536dbfd8f3e8fb8b81417acc7bae06dcd3df] # # patch "graph.cc" # from [0e4930c4797e6f2b6c163b12e28a8d3876bedbec] # to [bdda595f7f9c3b6f1513389550f316bf3ebe688d] # # patch "monotone.texi" # from [3446d31f637ac3426b7ff20ffaadde1b2b2ba0de] # to [c473c9643b4b96ff4549fe74edde642249c3db13] # ============================================================ --- tests/delta_directions/__driver__.lua 4f32096c299d54b83a2f6d83079af05171501107 +++ tests/delta_directions/__driver__.lua 4f32096c299d54b83a2f6d83079af05171501107 @@ -0,0 +1,61 @@ +include("/common/netsync.lua") +mtn_setup() +netsync.setup() + +check(mtn2("set", "database", "delta-direction", "both"), 0) +check(mtn3("set", "database", "delta-direction", "forward"), 0) + + +addfile("foo", "bar") +commit() + +srv = netsync.start() +srv:pull("*", 2) +srv:stop() + +srv = netsync.start(3) +srv:push("*", 2) +srv:stop() + +---------------------------------- +addfile("uvw", "xyz") +writefile("foo", "baz") +check(mtn2("commit", "-mx"), 0, false, false) + +srv = netsync.start(3) +srv:push("*", 2) +srv:stop() + +srv = netsync.start(2) +srv:pull("*", 1) +srv:stop() + +---------------------------------- +addfile("abc", "def") +writefile("uvw", "rst") +writefile("foo", "fnord") +check(mtn3("commit", "-mx"), 0, false, false) + +srv = netsync.start(3) +srv:pull("*", 2) +srv:stop() + +srv = netsync.start() +srv:push("*", 2) +srv:stop() + +---------------------------------------------------- +check(mtn("db", "info"), 0, true) +rename("stdout", "info-1") +check(mtn2("db", "info"), 0, true) +rename("stdout", "info-2") +check(mtn3("db", "info"), 0, true) +rename("stdout", "info-3") + +check(qgrep("revisions *: *3$", "info-1")) +check(qgrep("revisions *: *3$", "info-2")) +check(qgrep("revisions *: *3$", "info-3")) + +check(qgrep("file deltas *: *3$", "info-1")) +check(qgrep("file deltas *: *6$", "info-2")) +check(qgrep("file deltas *: *3$", "info-3")) \ No newline at end of file ============================================================ --- NEWS 6746ecaa1c40d162e80b9342d2de5eb7783440dd +++ NEWS bfb023a41b4fc9dc17a9526e3d58da604e699895 @@ -21,6 +21,14 @@ been a long-standing request of various redistributors of binary packages, who prefer the use of globally shared libraries. + - There is a new db var "database delta-direction", which can have + values "reverse" (default), "forward", and "both". This controls + what kind of deltas are stored for new file versions. Forward + deltas are very fast for netsync, but slow for most other uses. + Set this to "both" (or "forward" if you're short on disk space) on + an empty db and pull everything into it, to get a database which + will be much faster for server usage (especially initial pulls). + Bugs fixed - In 0.42, a netsync writer would attemts to queue up all outgoing ============================================================ --- database.cc ddc702beb36a7317da57480b789f64dd0c2f041a +++ database.cc d3ff536dbfd8f3e8fb8b81417acc7bae06dcd3df @@ -1743,14 +1743,13 @@ database::put_file_delta(file_id const & file_id const & base, file_delta const & del) { - // nb: delta schema is (id, base, delta) I(!null_id(ident)); I(!null_id(base)); gzip del_packed; encode_gzip(del.inner(), del_packed); - imp->execute(query("INSERT INTO file_deltas VALUES (?, ?, ?)") + imp->execute(query("INSERT INTO file_deltas (id, base, delta) VALUES (?, ?, ?)") % blob(ident.inner()()) % blob(base.inner()()) % blob(del_packed())); @@ -2187,8 +2186,6 @@ database::put_file_version(file_id const file_delta const & del) { I(!(old_id == new_id)); - file_data old_data, new_data; - file_delta reverse_delta; if (!file_version_exists(old_id)) { @@ -2197,6 +2194,24 @@ database::put_file_version(file_id const return; } + var_value delta_direction("reverse"); + var_key key(var_domain("database"), var_name("delta-direction")); + if (var_exists(key)) + { + get_var(key, delta_direction); + } + bool make_reverse_deltas(delta_direction() == "reverse" || + delta_direction() == "both"); + bool make_forward_deltas(delta_direction() == "forward" || + delta_direction() == "both"); + if (!make_reverse_deltas && !make_forward_deltas) + { + W(F("Unknown delta direction '%s'; assuming 'reverse'. Valid " + "values are 'reverse', 'forward', 'both'.") % delta_direction); + make_reverse_deltas = true; + } + + file_data old_data, new_data; get_file_version(old_id, old_data); { data tmp; @@ -2204,6 +2219,7 @@ database::put_file_version(file_id const new_data = file_data(tmp); } + file_delta reverse_delta; { string tmp; invert_xdelta(old_data.inner()(), del.inner()(), tmp); @@ -2217,23 +2233,36 @@ database::put_file_version(file_id const } transaction_guard guard(*this); - if (file_or_manifest_base_exists(old_id, "files")) + if (make_reverse_deltas) { - // descendent of a head version replaces the head, therefore old head - // must be disposed of - imp->drop_or_cancel_file(old_id); + if (!file_or_manifest_base_exists(new_id, "files")) + { + imp->schedule_delayed_file(new_id, new_data); + } + if (!imp->delta_exists(old_id, new_id, "file_deltas")) + { + put_file_delta(old_id, new_id, reverse_delta); + } } - if (!file_or_manifest_base_exists(new_id, "files")) + if (make_forward_deltas) { - imp->schedule_delayed_file(new_id, new_data); + if (!imp->delta_exists(new_id, old_id, "file_deltas")) + { + put_file_delta(new_id, old_id, del); + } + } + else + { imp->drop(new_id.inner(), "file_deltas"); } - - if (!imp->delta_exists(old_id, new_id, "file_deltas")) + if (file_or_manifest_base_exists(old_id, "files")) { - put_file_delta(old_id, new_id, reverse_delta); - guard.commit(); + // descendent of a head version replaces the head, therefore old head + // must be disposed of + if (delta_exists(old_id.inner(), "file_deltas")) + imp->drop_or_cancel_file(old_id); } + guard.commit(); } void @@ -2523,19 +2552,22 @@ database::deltify_revision(revision_id c j = edge_changes(i).deltas_applied.begin(); j != edge_changes(i).deltas_applied.end(); ++j) { - if (file_or_manifest_base_exists(delta_entry_src(j), "files") && - file_version_exists(delta_entry_dst(j))) + file_id old_id(delta_entry_src(j)); + file_id new_id(delta_entry_dst(j)); + // if not yet deltified + if (file_or_manifest_base_exists(old_id, "files") && + file_version_exists(new_id)) { file_data old_data; file_data new_data; - get_file_version(delta_entry_src(j), old_data); - get_file_version(delta_entry_dst(j), new_data); + get_file_version(old_id, old_data); + get_file_version(new_id, new_data); delta delt; diff(old_data.inner(), new_data.inner(), delt); file_delta del(delt); - imp->drop_or_cancel_file(delta_entry_dst(j)); - imp->drop(delta_entry_dst(j).inner(), "file_deltas"); - put_file_version(delta_entry_src(j), delta_entry_dst(j), del); + imp->drop_or_cancel_file(new_id); + imp->drop(new_id.inner(), "file_deltas"); + put_file_version(old_id, new_id, del); } } } ============================================================ --- graph.cc 0e4930c4797e6f2b6c163b12e28a8d3876bedbec +++ graph.cc bdda595f7f9c3b6f1513389550f316bf3ebe688d @@ -63,16 +63,17 @@ get_reconstruction_path(id const & start // moment. this imperative version only loads the descendents of the // reconstruction node, so it much cheaper in terms of memory. + set seen_nodes; vector< shared_ptr > live_paths; { shared_ptr pth0 = shared_ptr(new reconstruction_path()); pth0->push_back(start); live_paths.push_back(pth0); + seen_nodes.insert(start); } shared_ptr selected_path; - set seen_nodes; while (!selected_path) { @@ -126,7 +127,8 @@ get_reconstruction_path(id const & start } // check for a cycle... not that anything would break if // there were one, but it's nice to let us know we have a bug - for (reconstruction_path::const_iterator k = pthN->begin(); k != pthN->end(); ++k) + for (reconstruction_path::const_iterator k = pthN->begin(); + k != pthN->end(); ++k) I(*k != *j); pthN->push_back(*j); next_paths.push_back(pthN); ============================================================ --- monotone.texi 3446d31f637ac3426b7ff20ffaadde1b2b2ba0de +++ monotone.texi c473c9643b4b96ff4549fe74edde642249c3db13 @@ -3521,6 +3521,15 @@ @heading Existing vars The default server for netsync operations to use. Automatically set by first use of netsync, and by any netsync that uses the @option{--set-default} option. address@hidden delta-direction +This tells monotone whether to store @value{reverse} deltas (the default), address@hidden deltas, or @value{both} kinds of deltas for reconstructing +versions of files. Reverse deltas are faster when inspecting recent files, +while forward deltas are much faster for sending over the network. This +should probably be set to @value{both} for a server database, unless disk +space is limited. + +Changing this value does not affect deltas that have already been stored. @end table @item known-servers