all messages for Guix-related lists mirrored at yhetil.org
 help / color / mirror / code / Atom feed
blob 2662253b9150cfae59b006a80bb7b21c763d0fee 8712 bytes (raw)
name: nix/libstore/optimise-store.cc 	 # note: path name is non-authoritative(*)

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
 
#include "config.h"

#include "util.hh"
#include "local-store.hh"
#include "globals.hh"

#include <cstdlib>
#include <cstring>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include <stdio.h>


namespace nix {


static void makeWritable(const Path & path)
{
    struct stat st;
    if (lstat(path.c_str(), &st))
        throw SysError(format("getting attributes of path `%1%'") % path);
    if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
        throw SysError(format("changing writability of `%1%'") % path);
}


struct MakeReadOnly
{
    Path path;
    MakeReadOnly(const Path & path) : path(path) { }
    ~MakeReadOnly()
    {
        try {
            /* This will make the path read-only. */
            if (path != "") canonicaliseTimestampAndPermissions(path);
        } catch (...) {
            ignoreException();
        }
    }
};


LocalStore::InodeHash LocalStore::loadInodeHash()
{
    printMsg(lvlDebug, "loading hash inodes in memory");
    InodeHash inodeHash;

    AutoCloseDir dir = opendir(linksDir.c_str());
    if (!dir) throw SysError(format("opening directory `%1%'") % linksDir);

    struct dirent * dirent;
    while (errno = 0, dirent = readdir(dir)) { /* sic */
        checkInterrupt();
        // We don't care if we hit non-hash files, anything goes
        inodeHash.insert(dirent->d_ino);
    }
    if (errno) throw SysError(format("reading directory `%1%'") % linksDir);

    printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size());

    return inodeHash;
}


Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash)
{
    Strings names;

    AutoCloseDir dir = opendir(path.c_str());
    if (!dir) throw SysError(format("opening directory `%1%'") % path);

    struct dirent * dirent;
    while (errno = 0, dirent = readdir(dir)) { /* sic */
        checkInterrupt();

        if (inodeHash.count(dirent->d_ino)) {
            printMsg(lvlDebug, format("`%1%' is already linked") % dirent->d_name);
            continue;
        }

        string name = dirent->d_name;
        if (name == "." || name == "..") continue;
        names.push_back(name);
    }
    if (errno) throw SysError(format("reading directory `%1%'") % path);

    return names;
}


void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash)
{
    checkInterrupt();

    struct stat st;
    if (lstat(path.c_str(), &st))
        throw SysError(format("getting attributes of path `%1%'") % path);

    if (S_ISDIR(st.st_mode)) {
        Strings names = readDirectoryIgnoringInodes(path, inodeHash);
        foreach (Strings::iterator, i, names)
            optimisePath_(stats, path + "/" + *i, inodeHash);
        return;
    }

    /* We can hard link regular files and maybe symlinks. */
    if (!S_ISREG(st.st_mode)
#if CAN_LINK_SYMLINK
        && !S_ISLNK(st.st_mode)
#endif
        ) return;

    /* Sometimes SNAFUs can cause files in the store to be
       modified, in particular when running programs as root under
       Guix System (example: $fontconfig/var/cache being modified).  Skip
       those files.  FIXME: check the modification time. */
    if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
        printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path);
        return;
    }

    /* This can still happen on top-level files. */
    if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
        printMsg(lvlDebug, format("`%1%' is already linked, with %2% other file(s).") % path % (st.st_nlink - 2));
        return;
    }

    /* Hash the file.  Note that hashPath() returns the hash over the
       NAR serialisation, which includes the execute bit on the file.
       Thus, executable and non-executable files with the same
       contents *won't* be linked (which is good because otherwise the
       permissions would be screwed up).

       Also note that if `path' is a symlink, then we're hashing the
       contents of the symlink (i.e. the result of readlink()), not
       the contents of the target (which may not even exist). */
    Hash hash = hashPath(htSHA256, path).first;
    printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));

    /* Check if this is a known hash. */
    Path linkPath = linksDir + "/" + printHash32(hash);

 retry:
    if (!pathExists(linkPath)) {
        /* Nope, create a hard link in the links directory. */
        if (link(path.c_str(), linkPath.c_str()) == 0) {
            inodeHash.insert(st.st_ino);
            return;
        }

	switch (errno) {
	case EEXIST:
	    /* Fall through if another process created ‘linkPath’ before
	       we did. */
	    break;

	case ENOSPC:
	    /* On ext4, that probably means the directory index is full.  When
	       that happens, it's fine to ignore it: we just effectively
	       disable deduplication of this file.  */
	    printMsg(lvlInfo, format("cannot link `%1%' to `%2%': %3%")
		     % linkPath % path % strerror(ENOSPC));
	    return;

	default:
            throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path);
	}
    }

    /* Yes!  We've seen a file with the same contents.  Replace the
       current file with a hard link to that file. */
    struct stat stLink;
    if (lstat(linkPath.c_str(), &stLink))
        throw SysError(format("getting attributes of path `%1%'") % linkPath);

    if (st.st_ino == stLink.st_ino) {
        printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath);
        return;
    }

    if (st.st_size != stLink.st_size) {
        printMsg(lvlError, format("removing corrupted link `%1%’") % linkPath);
        unlink(linkPath.c_str());
        goto retry;
    }

    printMsg(lvlTalkative, format("linking `%1%’ to `%2%’") % path % linkPath);

    /* Make the containing directory writable, but only if it's not
       the store itself (we don't want or need to mess with its
       permissions). */
    bool mustToggle = !isStorePath(path);
    if (mustToggle) makeWritable(dirOf(path));

    /* When we're done, make the directory read-only again and reset
       its timestamp back to 0. */
    MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");

    Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
        % settings.nixStore % getpid() % rand()).str();

    if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
        if (errno == EMLINK) {
            /* Too many links to the same file (>= 32000 on most file
               systems).  This is likely to happen with empty files.
               Just shrug and ignore. */
            if (st.st_size)
                printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
            return;
        }
	    throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath);
	}

    /* Atomically replace the old file with the new hard link. */
    if (rename(tempLink.c_str(), path.c_str()) == -1) {
	int renameErrno = errno;
        if (unlink(tempLink.c_str()) == -1)
            printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
        if (renameErrno == EMLINK) {
            /* Some filesystems generate too many links on the rename,
               rather than on the original link.  (Probably it
               temporarily increases the st_nlink field before
               decreasing it again.) */
            if (st.st_size)
                printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
            return;
        }
        throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path);
    }

    stats.filesLinked++;
    stats.bytesFreed += st.st_size;
    stats.blocksFreed += st.st_blocks;
}


void LocalStore::optimiseStore(OptimiseStats & stats)
{
    PathSet paths = queryAllValidPaths();
    InodeHash inodeHash = loadInodeHash();

    foreach (PathSet::iterator, i, paths) {
        addTempRoot(*i);
        if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
        startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
        optimisePath_(stats, *i, inodeHash);
    }
}

static string showBytes(unsigned long long bytes)
{
    return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
}

void LocalStore::optimiseStore()
{
    OptimiseStats stats;

    optimiseStore(stats);

    printMsg(lvlError,
        format("%1% freed by hard-linking %2% files")
        % showBytes(stats.bytesFreed)
        % stats.filesLinked);
}

void LocalStore::optimisePath(const Path & path)
{
    OptimiseStats stats;
    InodeHash inodeHash;

    if (settings.autoOptimiseStore) optimisePath_(stats, path, inodeHash);
}


}

debug log:

solving 2662253b91 ...
found 2662253b91 in https://yhetil.org/guix/20210518163405.3618-1-me@tobias.gr/
found eb303ab4c3 in https://git.savannah.gnu.org/cgit/guix.git
preparing index
index prepared:
100644 eb303ab4c35dadd5ffc3654e36590a4682f4639e	nix/libstore/optimise-store.cc

applying [1/1] https://yhetil.org/guix/20210518163405.3618-1-me@tobias.gr/
diff --git a/nix/libstore/optimise-store.cc b/nix/libstore/optimise-store.cc
index eb303ab4c3..2662253b91 100644

Checking patch nix/libstore/optimise-store.cc...
Applied patch nix/libstore/optimise-store.cc cleanly.

index at:
100644 2662253b9150cfae59b006a80bb7b21c763d0fee	nix/libstore/optimise-store.cc

(*) Git path names are given by the tree(s) the blob belongs to.
    Blobs themselves have no identifier aside from the hash of its contents.^

Code repositories for project(s) associated with this external index

	https://git.savannah.gnu.org/cgit/guix.git

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.