From 472a0e82a52a3d5d841e1dfad6b13e26082a5750 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sat, 13 Nov 2021 21:47:15 +0100 Subject: daemon: Do not deduplicate files smaller than 8 KiB. Files smaller than 8 KiB typically represent ~70% of the entries in /gnu/store/.links but only contribute to ~4% of the space savings afforded by deduplication. Not considering these files for deduplication speeds up file insertion in the store and, more importantly, leaves 'removeUnusedLinks' with fewer entries to traverse, thereby speeding it up proportionally. Partly fixes . * config-daemon.ac: Remove symlink hard link check and CAN_LINK_SYMLINK definition. * guix/store/deduplication.scm (%deduplication-minimum-size): New variable. (deduplicate)[loop]: Do not recurse when FILE's size is below %DEDUPLICATION-MINIMUM-SIZE. (dump-port): New procedure. (dump-file/deduplicate)[hash]: Turn into... [dump-and-compute-hash]: ... this thunk. Call 'deduplicate' only when SIZE is greater than %DEDUPLICATION-MINIMUM-SIZE; otherwise call 'dump-port'. * nix/libstore/gc.cc (LocalStore::removeUnusedLinks): Drop files where st.st_size < deduplicationMinSize. * nix/libstore/local-store.hh (deduplicationMinSize): New declaration. * nix/libstore/optimise-store.cc (deduplicationMinSize): New variable. (LocalStore::optimisePath_): Return when PATH is a symlink or smaller than 'deduplicationMinSize'. * tests/derivations.scm ("identical files are deduplicated"): Produce files bigger than %DEDUPLICATION-MINIMUM-SIZE. * tests/nar.scm ("restore-file-set with directories (signed, valid)"): Likewise. * tests/store-deduplication.scm ("deduplicate, below %deduplication-minimum-size"): New test. ("deduplicate", "deduplicate, ENOSPC"): Produce files bigger than %DEDUPLICATION-MINIMUM-SIZE. * tests/store.scm ("substitute, deduplication"): Likewise. --- nix/libstore/gc.cc | 4 +++- nix/libstore/local-store.hh | 3 +++ nix/libstore/optimise-store.cc | 15 +++++++++------ 3 files changed, 15 insertions(+), 7 deletions(-) (limited to 'nix') diff --git a/nix/libstore/gc.cc b/nix/libstore/gc.cc index e1d0765154..16519116e4 100644 --- a/nix/libstore/gc.cc +++ b/nix/libstore/gc.cc @@ -606,7 +606,9 @@ void LocalStore::removeUnusedLinks(const GCState & state) throw SysError(format("statting `%1%'") % path); #endif - if (st.st_nlink != 1) { + /* Drop links for files smaller than 'deduplicationMinSize', even if + they have more than one hard link. */ + if (st.st_nlink != 1 && st.st_size >= deduplicationMinSize) { actualSize += st.st_size; unsharedSize += (st.st_nlink - 1) * st.st_size; continue; diff --git a/nix/libstore/local-store.hh b/nix/libstore/local-store.hh index 9ba37219da..20d3c3c893 100644 --- a/nix/libstore/local-store.hh +++ b/nix/libstore/local-store.hh @@ -292,4 +292,7 @@ void canonicaliseTimestampAndPermissions(const Path & path); MakeError(PathInUse, Error); +/* Size below which a file is not considered for deduplication. */ +extern const size_t deduplicationMinSize; + } diff --git a/nix/libstore/optimise-store.cc b/nix/libstore/optimise-store.cc index eb303ab4c3..9fd6f3cb35 100644 --- a/nix/libstore/optimise-store.cc +++ b/nix/libstore/optimise-store.cc @@ -15,6 +15,9 @@ namespace nix { +/* Any file smaller than this is not considered for deduplication. + Keep in sync with (guix store deduplication). */ +const size_t deduplicationMinSize = 8192; static void makeWritable(const Path & path) { @@ -105,12 +108,12 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa return; } - /* We can hard link regular files and maybe symlinks. */ - if (!S_ISREG(st.st_mode) -#if CAN_LINK_SYMLINK - && !S_ISLNK(st.st_mode) -#endif - ) return; + /* We can hard link regular files (and maybe symlinks), but do that only + for files larger than some threshold. This avoids adding too many + entries to '.links', which would slow down 'removeUnusedLinks' while + saving little space. */ + if (!S_ISREG(st.st_mode) || ((size_t) st.st_size) < deduplicationMinSize) + return; /* Sometimes SNAFUs can cause files in the store to be modified, in particular when running programs as root under -- cgit v1.2.3