filesystem-disk-test.c++ (31458B)
1 // Copyright (c) 2016 Sandstorm Development Group, Inc. and contributors 2 // Licensed under the MIT License: 3 // 4 // Permission is hereby granted, free of charge, to any person obtaining a copy 5 // of this software and associated documentation files (the "Software"), to deal 6 // in the Software without restriction, including without limitation the rights 7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 // copies of the Software, and to permit persons to whom the Software is 9 // furnished to do so, subject to the following conditions: 10 // 11 // The above copyright notice and this permission notice shall be included in 12 // all copies or substantial portions of the Software. 13 // 14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 // THE SOFTWARE. 21 22 #include "filesystem.h" 23 #include "test.h" 24 #include "encoding.h" 25 #include <stdlib.h> 26 #if _WIN32 27 #include <windows.h> 28 #include "windows-sanity.h" 29 #else 30 #include <unistd.h> 31 #include <errno.h> 32 #include <sys/types.h> 33 #include <sys/stat.h> 34 #include <fcntl.h> 35 #include <dirent.h> 36 #endif 37 38 namespace kj { 39 namespace { 40 41 bool isWine() KJ_UNUSED; 42 43 #if _WIN32 44 45 bool detectWine() { 46 HMODULE hntdll = GetModuleHandle("ntdll.dll"); 47 if(hntdll == NULL) return false; 48 return GetProcAddress(hntdll, "wine_get_version") != nullptr; 49 } 50 51 bool isWine() { 52 static bool result = detectWine(); 53 return result; 54 } 55 56 template <typename Func> 57 static auto newTemp(Func&& create) 58 -> Decay<decltype(*kj::_::readMaybe(create(Array<wchar_t>())))> { 59 wchar_t wtmpdir[MAX_PATH + 1]; 60 DWORD len = GetTempPathW(kj::size(wtmpdir), wtmpdir); 61 KJ_ASSERT(len < kj::size(wtmpdir)); 62 auto tmpdir = decodeWideString(arrayPtr(wtmpdir, len)); 63 64 static uint counter = 0; 65 for (;;) { 66 auto path = kj::str(tmpdir, "kj-filesystem-test.", GetCurrentProcessId(), ".", counter++); 67 KJ_IF_MAYBE(result, create(encodeWideString(path, true))) { 68 return kj::mv(*result); 69 } 70 } 71 } 72 73 static Own<File> newTempFile() { 74 return newTemp([](Array<wchar_t> candidatePath) -> Maybe<Own<File>> { 75 HANDLE handle; 76 KJ_WIN32_HANDLE_ERRORS(handle = CreateFileW( 77 candidatePath.begin(), 78 FILE_GENERIC_READ | FILE_GENERIC_WRITE, 79 0, 80 NULL, 81 CREATE_NEW, 82 FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE, 83 NULL)) { 84 case ERROR_ALREADY_EXISTS: 85 case ERROR_FILE_EXISTS: 86 return nullptr; 87 default: 88 KJ_FAIL_WIN32("CreateFileW", error); 89 } 90 return newDiskFile(AutoCloseHandle(handle)); 91 }); 92 } 93 94 static Array<wchar_t> join16(ArrayPtr<const wchar_t> path, const wchar_t* file) { 95 // Assumes `path` ends with a NUL terminator (and `file` is of course NUL terminated as well). 96 97 size_t len = wcslen(file) + 1; 98 auto result = kj::heapArray<wchar_t>(path.size() + len); 99 memcpy(result.begin(), path.begin(), path.asBytes().size() - sizeof(wchar_t)); 100 result[path.size() - 1] = '\\'; 101 memcpy(result.begin() + path.size(), file, len * sizeof(wchar_t)); 102 return result; 103 } 104 105 class TempDir { 106 public: 107 TempDir(): filename(newTemp([](Array<wchar_t> candidatePath) -> Maybe<Array<wchar_t>> { 108 KJ_WIN32_HANDLE_ERRORS(CreateDirectoryW(candidatePath.begin(), NULL)) { 109 case ERROR_ALREADY_EXISTS: 110 case ERROR_FILE_EXISTS: 111 return nullptr; 112 default: 113 KJ_FAIL_WIN32("CreateDirectoryW", error); 114 } 115 return kj::mv(candidatePath); 116 })) {} 117 118 Own<Directory> get() { 119 HANDLE handle; 120 KJ_WIN32(handle = CreateFileW( 121 filename.begin(), 122 GENERIC_READ, 123 FILE_SHARE_READ | FILE_SHARE_WRITE, 124 NULL, 125 OPEN_EXISTING, 126 FILE_FLAG_BACKUP_SEMANTICS, // apparently, this flag is required for directories 127 NULL)); 128 return newDiskDirectory(AutoCloseHandle(handle)); 129 } 130 131 ~TempDir() noexcept(false) { 132 recursiveDelete(filename); 133 } 134 135 private: 136 Array<wchar_t> filename; 137 138 static void recursiveDelete(ArrayPtr<const wchar_t> path) { 139 // Recursively delete the temp dir, verifying that no .kj-tmp. files were left over. 140 // 141 // Mostly copied from rmrfChildren() in filesystem-win32.c++. 142 143 auto glob = join16(path, L"\\*"); 144 145 WIN32_FIND_DATAW data; 146 HANDLE handle = FindFirstFileW(glob.begin(), &data); 147 if (handle == INVALID_HANDLE_VALUE) { 148 auto error = GetLastError(); 149 if (error == ERROR_FILE_NOT_FOUND) return; 150 KJ_FAIL_WIN32("FindFirstFile", error, path) { return; } 151 } 152 KJ_DEFER(KJ_WIN32(FindClose(handle)) { break; }); 153 154 do { 155 // Ignore "." and "..", ugh. 156 if (data.cFileName[0] == L'.') { 157 if (data.cFileName[1] == L'\0' || 158 (data.cFileName[1] == L'.' && data.cFileName[2] == L'\0')) { 159 continue; 160 } 161 } 162 163 String utf8Name = decodeWideString(arrayPtr(data.cFileName, wcslen(data.cFileName))); 164 KJ_EXPECT(!utf8Name.startsWith(".kj-tmp."), "temp file not cleaned up", utf8Name); 165 166 auto child = join16(path, data.cFileName); 167 if ((data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) && 168 !(data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { 169 recursiveDelete(child); 170 } else { 171 KJ_WIN32(DeleteFileW(child.begin())); 172 } 173 } while (FindNextFileW(handle, &data)); 174 175 auto error = GetLastError(); 176 if (error != ERROR_NO_MORE_FILES) { 177 KJ_FAIL_WIN32("FindNextFile", error, path) { return; } 178 } 179 180 uint retryCount = 0; 181 retry: 182 KJ_WIN32_HANDLE_ERRORS(RemoveDirectoryW(path.begin())) { 183 case ERROR_DIR_NOT_EMPTY: 184 if (retryCount++ < 10) { 185 Sleep(10); 186 goto retry; 187 } 188 KJ_FALLTHROUGH; 189 default: 190 KJ_FAIL_WIN32("RemoveDirectory", error) { break; } 191 } 192 } 193 }; 194 195 #else 196 197 bool isWine() { return false; } 198 199 #if __APPLE__ || __CYGWIN__ 200 #define HOLES_NOT_SUPPORTED 1 201 #endif 202 203 #if __ANDROID__ 204 #define VAR_TMP "/data/local/tmp" 205 #else 206 #define VAR_TMP "/var/tmp" 207 #endif 208 209 static Own<File> newTempFile() { 210 char filename[] = VAR_TMP "/kj-filesystem-test.XXXXXX"; 211 int fd; 212 KJ_SYSCALL(fd = mkstemp(filename)); 213 KJ_DEFER(KJ_SYSCALL(unlink(filename))); 214 return newDiskFile(AutoCloseFd(fd)); 215 } 216 217 class TempDir { 218 public: 219 TempDir(): filename(heapString(VAR_TMP "/kj-filesystem-test.XXXXXX")) { 220 if (mkdtemp(filename.begin()) == nullptr) { 221 KJ_FAIL_SYSCALL("mkdtemp", errno, filename); 222 } 223 } 224 225 Own<Directory> get() { 226 int fd; 227 KJ_SYSCALL(fd = open(filename.cStr(), O_RDONLY)); 228 return newDiskDirectory(AutoCloseFd(fd)); 229 } 230 231 ~TempDir() noexcept(false) { 232 recursiveDelete(filename); 233 } 234 235 private: 236 String filename; 237 238 static void recursiveDelete(StringPtr path) { 239 // Recursively delete the temp dir, verifying that no .kj-tmp. files were left over. 240 241 { 242 DIR* dir = opendir(path.cStr()); 243 KJ_ASSERT(dir != nullptr); 244 KJ_DEFER(closedir(dir)); 245 246 for (;;) { 247 auto entry = readdir(dir); 248 if (entry == nullptr) break; 249 250 StringPtr name = entry->d_name; 251 if (name == "." || name == "..") continue; 252 253 auto subPath = kj::str(path, '/', entry->d_name); 254 255 KJ_EXPECT(!name.startsWith(".kj-tmp."), "temp file not cleaned up", subPath); 256 257 struct stat stats; 258 KJ_SYSCALL(lstat(subPath.cStr(), &stats)); 259 260 if (S_ISDIR(stats.st_mode)) { 261 recursiveDelete(subPath); 262 } else { 263 KJ_SYSCALL(unlink(subPath.cStr())); 264 } 265 } 266 } 267 268 KJ_SYSCALL(rmdir(path.cStr())); 269 } 270 }; 271 272 #endif // _WIN32, else 273 274 KJ_TEST("DiskFile") { 275 auto file = newTempFile(); 276 277 KJ_EXPECT(file->readAllText() == ""); 278 279 // mmaping empty file should work 280 KJ_EXPECT(file->mmap(0, 0).size() == 0); 281 KJ_EXPECT(file->mmapPrivate(0, 0).size() == 0); 282 KJ_EXPECT(file->mmapWritable(0, 0)->get().size() == 0); 283 284 file->writeAll("foo"); 285 KJ_EXPECT(file->readAllText() == "foo"); 286 287 file->write(3, StringPtr("bar").asBytes()); 288 KJ_EXPECT(file->readAllText() == "foobar"); 289 290 file->write(3, StringPtr("baz").asBytes()); 291 KJ_EXPECT(file->readAllText() == "foobaz"); 292 293 file->write(9, StringPtr("qux").asBytes()); 294 KJ_EXPECT(file->readAllText() == kj::StringPtr("foobaz\0\0\0qux", 12)); 295 296 file->truncate(6); 297 KJ_EXPECT(file->readAllText() == "foobaz"); 298 299 file->truncate(18); 300 KJ_EXPECT(file->readAllText() == kj::StringPtr("foobaz\0\0\0\0\0\0\0\0\0\0\0\0", 18)); 301 302 // empty mappings work, even if useless 303 KJ_EXPECT(file->mmap(0, 0).size() == 0); 304 KJ_EXPECT(file->mmapPrivate(0, 0).size() == 0); 305 KJ_EXPECT(file->mmapWritable(0, 0)->get().size() == 0); 306 KJ_EXPECT(file->mmap(2, 0).size() == 0); 307 KJ_EXPECT(file->mmapPrivate(2, 0).size() == 0); 308 KJ_EXPECT(file->mmapWritable(2, 0)->get().size() == 0); 309 310 { 311 auto mapping = file->mmap(0, 18); 312 auto privateMapping = file->mmapPrivate(0, 18); 313 auto writableMapping = file->mmapWritable(0, 18); 314 315 KJ_EXPECT(mapping.size() == 18); 316 KJ_EXPECT(privateMapping.size() == 18); 317 KJ_EXPECT(writableMapping->get().size() == 18); 318 319 KJ_EXPECT(writableMapping->get().begin() != mapping.begin()); 320 KJ_EXPECT(privateMapping.begin() != mapping.begin()); 321 KJ_EXPECT(writableMapping->get().begin() != privateMapping.begin()); 322 323 KJ_EXPECT(kj::str(mapping.slice(0, 6).asChars()) == "foobaz"); 324 KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "foobaz"); 325 KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "foobaz"); 326 327 privateMapping[0] = 'F'; 328 KJ_EXPECT(kj::str(mapping.slice(0, 6).asChars()) == "foobaz"); 329 KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "foobaz"); 330 KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "Foobaz"); 331 332 writableMapping->get()[1] = 'D'; 333 writableMapping->changed(writableMapping->get().slice(1, 2)); 334 KJ_EXPECT(kj::str(mapping.slice(0, 6).asChars()) == "fDobaz"); 335 KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "fDobaz"); 336 KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "Foobaz"); 337 338 file->write(0, StringPtr("qux").asBytes()); 339 KJ_EXPECT(kj::str(mapping.slice(0, 6).asChars()) == "quxbaz"); 340 KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "quxbaz"); 341 KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "Foobaz"); 342 343 file->write(12, StringPtr("corge").asBytes()); 344 KJ_EXPECT(kj::str(mapping.slice(12, 17).asChars()) == "corge"); 345 346 #if !_WIN32 && !__CYGWIN__ // Windows doesn't allow the file size to change while mapped. 347 // Can shrink. 348 file->truncate(6); 349 KJ_EXPECT(kj::str(mapping.slice(12, 17).asChars()) == kj::StringPtr("\0\0\0\0\0", 5)); 350 351 // Can regrow. 352 file->truncate(18); 353 KJ_EXPECT(kj::str(mapping.slice(12, 17).asChars()) == kj::StringPtr("\0\0\0\0\0", 5)); 354 355 // Can even regrow past previous capacity. 356 file->truncate(100); 357 #endif 358 } 359 360 file->truncate(6); 361 362 KJ_EXPECT(file->readAllText() == "quxbaz"); 363 file->zero(3, 3); 364 KJ_EXPECT(file->readAllText() == StringPtr("qux\0\0\0", 6)); 365 } 366 367 KJ_TEST("DiskFile::copy()") { 368 auto source = newTempFile(); 369 source->writeAll("foobarbaz"); 370 371 auto dest = newTempFile(); 372 dest->writeAll("quxcorge"); 373 374 KJ_EXPECT(dest->copy(3, *source, 6, kj::maxValue) == 3); 375 KJ_EXPECT(dest->readAllText() == "quxbazge"); 376 377 KJ_EXPECT(dest->copy(0, *source, 3, 4) == 4); 378 KJ_EXPECT(dest->readAllText() == "barbazge"); 379 380 KJ_EXPECT(dest->copy(0, *source, 128, kj::maxValue) == 0); 381 382 KJ_EXPECT(dest->copy(4, *source, 3, 0) == 0); 383 384 String bigString = strArray(repeat("foobar", 10000), ""); 385 source->truncate(bigString.size() + 1000); 386 source->write(123, bigString.asBytes()); 387 388 dest->copy(321, *source, 123, bigString.size()); 389 KJ_EXPECT(dest->readAllText().slice(321) == bigString); 390 } 391 392 KJ_TEST("DiskDirectory") { 393 TempDir tempDir; 394 auto dir = tempDir.get(); 395 396 KJ_EXPECT(dir->listNames() == nullptr); 397 KJ_EXPECT(dir->listEntries() == nullptr); 398 KJ_EXPECT(!dir->exists(Path("foo"))); 399 KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr); 400 KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::MODIFY) == nullptr); 401 402 { 403 auto file = dir->openFile(Path("foo"), WriteMode::CREATE); 404 file->writeAll("foobar"); 405 } 406 407 KJ_EXPECT(dir->exists(Path("foo"))); 408 409 { 410 auto stats = dir->lstat(Path("foo")); 411 KJ_EXPECT(stats.type == FsNode::Type::FILE); 412 KJ_EXPECT(stats.size == 6); 413 } 414 415 { 416 auto list = dir->listNames(); 417 KJ_ASSERT(list.size() == 1); 418 KJ_EXPECT(list[0] == "foo"); 419 } 420 421 { 422 auto list = dir->listEntries(); 423 KJ_ASSERT(list.size() == 1); 424 KJ_EXPECT(list[0].name == "foo"); 425 KJ_EXPECT(list[0].type == FsNode::Type::FILE); 426 } 427 428 KJ_EXPECT(dir->openFile(Path("foo"))->readAllText() == "foobar"); 429 430 KJ_EXPECT(dir->tryOpenFile(Path({"foo", "bar"}), WriteMode::MODIFY) == nullptr); 431 KJ_EXPECT(dir->tryOpenFile(Path({"bar", "baz"}), WriteMode::MODIFY) == nullptr); 432 KJ_EXPECT_THROW_RECOVERABLE_MESSAGE("parent is not a directory", 433 dir->tryOpenFile(Path({"bar", "baz"}), WriteMode::CREATE)); 434 435 { 436 auto file = dir->openFile(Path({"bar", "baz"}), WriteMode::CREATE | WriteMode::CREATE_PARENT); 437 file->writeAll("bazqux"); 438 } 439 440 KJ_EXPECT(dir->openFile(Path({"bar", "baz"}))->readAllText() == "bazqux"); 441 442 { 443 auto stats = dir->lstat(Path("bar")); 444 KJ_EXPECT(stats.type == FsNode::Type::DIRECTORY); 445 } 446 447 { 448 auto list = dir->listNames(); 449 KJ_ASSERT(list.size() == 2); 450 KJ_EXPECT(list[0] == "bar"); 451 KJ_EXPECT(list[1] == "foo"); 452 } 453 454 { 455 auto list = dir->listEntries(); 456 KJ_ASSERT(list.size() == 2); 457 KJ_EXPECT(list[0].name == "bar"); 458 KJ_EXPECT(list[0].type == FsNode::Type::DIRECTORY); 459 KJ_EXPECT(list[1].name == "foo"); 460 KJ_EXPECT(list[1].type == FsNode::Type::FILE); 461 } 462 463 { 464 auto subdir = dir->openSubdir(Path("bar")); 465 466 KJ_EXPECT(subdir->openFile(Path("baz"))->readAllText() == "bazqux"); 467 } 468 469 auto subdir = dir->openSubdir(Path("corge"), WriteMode::CREATE); 470 471 subdir->openFile(Path("grault"), WriteMode::CREATE)->writeAll("garply"); 472 473 KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "garply"); 474 475 dir->openFile(Path({"corge", "grault"}), WriteMode::CREATE | WriteMode::MODIFY) 476 ->write(0, StringPtr("rag").asBytes()); 477 KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "ragply"); 478 479 KJ_EXPECT(dir->openSubdir(Path("corge"))->listNames().size() == 1); 480 481 { 482 auto replacer = 483 dir->replaceFile(Path({"corge", "grault"}), WriteMode::CREATE | WriteMode::MODIFY); 484 replacer->get().writeAll("rag"); 485 486 // temp file not in list 487 KJ_EXPECT(dir->openSubdir(Path("corge"))->listNames().size() == 1); 488 489 // Don't commit. 490 } 491 KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "ragply"); 492 493 { 494 auto replacer = 495 dir->replaceFile(Path({"corge", "grault"}), WriteMode::CREATE | WriteMode::MODIFY); 496 replacer->get().writeAll("rag"); 497 498 // temp file not in list 499 KJ_EXPECT(dir->openSubdir(Path("corge"))->listNames().size() == 1); 500 501 replacer->commit(); 502 KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "rag"); 503 } 504 505 KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "rag"); 506 507 { 508 auto appender = dir->appendFile(Path({"corge", "grault"}), WriteMode::MODIFY); 509 appender->write("waldo", 5); 510 appender->write("fred", 4); 511 } 512 513 KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "ragwaldofred"); 514 515 KJ_EXPECT(dir->exists(Path("foo"))); 516 dir->remove(Path("foo")); 517 KJ_EXPECT(!dir->exists(Path("foo"))); 518 KJ_EXPECT(!dir->tryRemove(Path("foo"))); 519 520 KJ_EXPECT(dir->exists(Path({"bar", "baz"}))); 521 dir->remove(Path({"bar", "baz"})); 522 KJ_EXPECT(!dir->exists(Path({"bar", "baz"}))); 523 KJ_EXPECT(dir->exists(Path("bar"))); 524 KJ_EXPECT(!dir->tryRemove(Path({"bar", "baz"}))); 525 526 #if _WIN32 527 // On Windows, we can't delete a directory while we still have it open. 528 subdir = nullptr; 529 #endif 530 531 KJ_EXPECT(dir->exists(Path("corge"))); 532 KJ_EXPECT(dir->exists(Path({"corge", "grault"}))); 533 dir->remove(Path("corge")); 534 KJ_EXPECT(!dir->exists(Path("corge"))); 535 KJ_EXPECT(!dir->exists(Path({"corge", "grault"}))); 536 KJ_EXPECT(!dir->tryRemove(Path("corge"))); 537 } 538 539 #if !_WIN32 // Creating symlinks on Win32 requires admin privileges prior to Windows 10. 540 KJ_TEST("DiskDirectory symlinks") { 541 TempDir tempDir; 542 auto dir = tempDir.get(); 543 544 dir->symlink(Path("foo"), "bar/qux/../baz", WriteMode::CREATE); 545 546 KJ_EXPECT(!dir->trySymlink(Path("foo"), "bar/qux/../baz", WriteMode::CREATE)); 547 548 { 549 auto stats = dir->lstat(Path("foo")); 550 KJ_EXPECT(stats.type == FsNode::Type::SYMLINK); 551 } 552 553 KJ_EXPECT(dir->readlink(Path("foo")) == "bar/qux/../baz"); 554 555 // Broken link into non-existing directory cannot be opened in any mode. 556 KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr); 557 KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::CREATE) == nullptr); 558 KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::MODIFY) == nullptr); 559 KJ_EXPECT_THROW_RECOVERABLE_MESSAGE("parent is not a directory", 560 dir->tryOpenFile(Path("foo"), WriteMode::CREATE | WriteMode::MODIFY)); 561 KJ_EXPECT_THROW_RECOVERABLE_MESSAGE("parent is not a directory", 562 dir->tryOpenFile(Path("foo"), 563 WriteMode::CREATE | WriteMode::MODIFY | WriteMode::CREATE_PARENT)); 564 565 // Create the directory. 566 auto subdir = dir->openSubdir(Path("bar"), WriteMode::CREATE); 567 subdir->openSubdir(Path("qux"), WriteMode::CREATE); 568 569 // Link still points to non-existing file so cannot be open in most modes. 570 KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr); 571 KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::CREATE) == nullptr); 572 KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::MODIFY) == nullptr); 573 574 // But... CREATE | MODIFY works. 575 dir->openFile(Path("foo"), WriteMode::CREATE | WriteMode::MODIFY) 576 ->writeAll("foobar"); 577 578 KJ_EXPECT(dir->openFile(Path({"bar", "baz"}))->readAllText() == "foobar"); 579 KJ_EXPECT(dir->openFile(Path("foo"))->readAllText() == "foobar"); 580 KJ_EXPECT(dir->openFile(Path("foo"), WriteMode::MODIFY)->readAllText() == "foobar"); 581 582 // operations that modify the symlink 583 dir->symlink(Path("foo"), "corge", WriteMode::MODIFY); 584 KJ_EXPECT(dir->openFile(Path({"bar", "baz"}))->readAllText() == "foobar"); 585 KJ_EXPECT(dir->readlink(Path("foo")) == "corge"); 586 KJ_EXPECT(!dir->exists(Path("foo"))); 587 KJ_EXPECT(dir->lstat(Path("foo")).type == FsNode::Type::SYMLINK); 588 KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr); 589 590 dir->remove(Path("foo")); 591 KJ_EXPECT(!dir->exists(Path("foo"))); 592 KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr); 593 } 594 #endif 595 596 KJ_TEST("DiskDirectory link") { 597 TempDir tempDirSrc; 598 TempDir tempDirDst; 599 600 auto src = tempDirSrc.get(); 601 auto dst = tempDirDst.get(); 602 603 src->openFile(Path("foo"), WriteMode::CREATE | WriteMode::CREATE_PARENT) 604 ->writeAll("foobar"); 605 606 dst->transfer(Path("link"), WriteMode::CREATE, *src, Path("foo"), TransferMode::LINK); 607 608 KJ_EXPECT(dst->openFile(Path("link"))->readAllText() == "foobar"); 609 610 // Writing the old location modifies the new. 611 src->openFile(Path("foo"), WriteMode::MODIFY)->writeAll("bazqux"); 612 KJ_EXPECT(dst->openFile(Path("link"))->readAllText() == "bazqux"); 613 614 // Replacing the old location doesn't modify the new. 615 { 616 auto replacer = src->replaceFile(Path("foo"), WriteMode::MODIFY); 617 replacer->get().writeAll("corge"); 618 replacer->commit(); 619 } 620 KJ_EXPECT(src->openFile(Path("foo"))->readAllText() == "corge"); 621 KJ_EXPECT(dst->openFile(Path("link"))->readAllText() == "bazqux"); 622 } 623 624 KJ_TEST("DiskDirectory copy") { 625 TempDir tempDirSrc; 626 TempDir tempDirDst; 627 628 auto src = tempDirSrc.get(); 629 auto dst = tempDirDst.get(); 630 631 src->openFile(Path({"foo", "bar"}), WriteMode::CREATE | WriteMode::CREATE_PARENT) 632 ->writeAll("foobar"); 633 src->openFile(Path({"foo", "baz", "qux"}), WriteMode::CREATE | WriteMode::CREATE_PARENT) 634 ->writeAll("bazqux"); 635 636 dst->transfer(Path("link"), WriteMode::CREATE, *src, Path("foo"), TransferMode::COPY); 637 638 KJ_EXPECT(src->openFile(Path({"foo", "bar"}))->readAllText() == "foobar"); 639 KJ_EXPECT(src->openFile(Path({"foo", "baz", "qux"}))->readAllText() == "bazqux"); 640 KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar"); 641 KJ_EXPECT(dst->openFile(Path({"link", "baz", "qux"}))->readAllText() == "bazqux"); 642 643 KJ_EXPECT(dst->exists(Path({"link", "bar"}))); 644 src->remove(Path({"foo", "bar"})); 645 KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar"); 646 } 647 648 KJ_TEST("DiskDirectory copy-replace") { 649 TempDir tempDirSrc; 650 TempDir tempDirDst; 651 652 auto src = tempDirSrc.get(); 653 auto dst = tempDirDst.get(); 654 655 src->openFile(Path({"foo", "bar"}), WriteMode::CREATE | WriteMode::CREATE_PARENT) 656 ->writeAll("foobar"); 657 src->openFile(Path({"foo", "baz", "qux"}), WriteMode::CREATE | WriteMode::CREATE_PARENT) 658 ->writeAll("bazqux"); 659 660 dst->openFile(Path({"link", "corge"}), WriteMode::CREATE | WriteMode::CREATE_PARENT) 661 ->writeAll("abcd"); 662 663 // CREATE fails. 664 KJ_EXPECT(!dst->tryTransfer(Path("link"), WriteMode::CREATE, 665 *src, Path("foo"), TransferMode::COPY)); 666 667 // Verify nothing changed. 668 KJ_EXPECT(dst->openFile(Path({"link", "corge"}))->readAllText() == "abcd"); 669 KJ_EXPECT(!dst->exists(Path({"foo", "bar"}))); 670 671 // Now try MODIFY. 672 dst->transfer(Path("link"), WriteMode::MODIFY, *src, Path("foo"), TransferMode::COPY); 673 674 KJ_EXPECT(src->openFile(Path({"foo", "bar"}))->readAllText() == "foobar"); 675 KJ_EXPECT(src->openFile(Path({"foo", "baz", "qux"}))->readAllText() == "bazqux"); 676 KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar"); 677 KJ_EXPECT(dst->openFile(Path({"link", "baz", "qux"}))->readAllText() == "bazqux"); 678 KJ_EXPECT(!dst->exists(Path({"link", "corge"}))); 679 680 KJ_EXPECT(dst->exists(Path({"link", "bar"}))); 681 src->remove(Path({"foo", "bar"})); 682 KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar"); 683 } 684 685 KJ_TEST("DiskDirectory move") { 686 TempDir tempDirSrc; 687 TempDir tempDirDst; 688 689 auto src = tempDirSrc.get(); 690 auto dst = tempDirDst.get(); 691 692 src->openFile(Path({"foo", "bar"}), WriteMode::CREATE | WriteMode::CREATE_PARENT) 693 ->writeAll("foobar"); 694 src->openFile(Path({"foo", "baz", "qux"}), WriteMode::CREATE | WriteMode::CREATE_PARENT) 695 ->writeAll("bazqux"); 696 697 dst->transfer(Path("link"), WriteMode::CREATE, *src, Path("foo"), TransferMode::MOVE); 698 699 KJ_EXPECT(!src->exists(Path({"foo"}))); 700 KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar"); 701 KJ_EXPECT(dst->openFile(Path({"link", "baz", "qux"}))->readAllText() == "bazqux"); 702 } 703 704 KJ_TEST("DiskDirectory move-replace") { 705 TempDir tempDirSrc; 706 TempDir tempDirDst; 707 708 auto src = tempDirSrc.get(); 709 auto dst = tempDirDst.get(); 710 711 src->openFile(Path({"foo", "bar"}), WriteMode::CREATE | WriteMode::CREATE_PARENT) 712 ->writeAll("foobar"); 713 src->openFile(Path({"foo", "baz", "qux"}), WriteMode::CREATE | WriteMode::CREATE_PARENT) 714 ->writeAll("bazqux"); 715 716 dst->openFile(Path({"link", "corge"}), WriteMode::CREATE | WriteMode::CREATE_PARENT) 717 ->writeAll("abcd"); 718 719 // CREATE fails. 720 KJ_EXPECT(!dst->tryTransfer(Path("link"), WriteMode::CREATE, 721 *src, Path("foo"), TransferMode::MOVE)); 722 723 // Verify nothing changed. 724 KJ_EXPECT(dst->openFile(Path({"link", "corge"}))->readAllText() == "abcd"); 725 KJ_EXPECT(!dst->exists(Path({"foo", "bar"}))); 726 KJ_EXPECT(src->exists(Path({"foo"}))); 727 728 // Now try MODIFY. 729 dst->transfer(Path("link"), WriteMode::MODIFY, *src, Path("foo"), TransferMode::MOVE); 730 731 KJ_EXPECT(!src->exists(Path({"foo"}))); 732 KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar"); 733 KJ_EXPECT(dst->openFile(Path({"link", "baz", "qux"}))->readAllText() == "bazqux"); 734 } 735 736 KJ_TEST("DiskDirectory createTemporary") { 737 TempDir tempDir; 738 auto dir = tempDir.get(); 739 auto file = dir->createTemporary(); 740 file->writeAll("foobar"); 741 KJ_EXPECT(file->readAllText() == "foobar"); 742 KJ_EXPECT(dir->listNames() == nullptr); 743 } 744 745 #if !__CYGWIN__ // TODO(someday): Figure out why this doesn't work on Cygwin. 746 KJ_TEST("DiskDirectory replaceSubdir()") { 747 TempDir tempDir; 748 auto dir = tempDir.get(); 749 750 { 751 auto replacer = dir->replaceSubdir(Path("foo"), WriteMode::CREATE); 752 replacer->get().openFile(Path("bar"), WriteMode::CREATE)->writeAll("original"); 753 KJ_EXPECT(replacer->get().openFile(Path("bar"))->readAllText() == "original"); 754 KJ_EXPECT(!dir->exists(Path({"foo", "bar"}))); 755 756 replacer->commit(); 757 KJ_EXPECT(replacer->get().openFile(Path("bar"))->readAllText() == "original"); 758 KJ_EXPECT(dir->openFile(Path({"foo", "bar"}))->readAllText() == "original"); 759 } 760 761 { 762 // CREATE fails -- already exists. 763 auto replacer = dir->replaceSubdir(Path("foo"), WriteMode::CREATE); 764 replacer->get().openFile(Path("corge"), WriteMode::CREATE)->writeAll("bazqux"); 765 KJ_EXPECT(dir->listNames().size() == 1 && dir->listNames()[0] == "foo"); 766 KJ_EXPECT(!replacer->tryCommit()); 767 } 768 769 // Unchanged. 770 KJ_EXPECT(dir->openFile(Path({"foo", "bar"}))->readAllText() == "original"); 771 KJ_EXPECT(!dir->exists(Path({"foo", "corge"}))); 772 773 { 774 // MODIFY succeeds. 775 auto replacer = dir->replaceSubdir(Path("foo"), WriteMode::MODIFY); 776 replacer->get().openFile(Path("corge"), WriteMode::CREATE)->writeAll("bazqux"); 777 KJ_EXPECT(dir->listNames().size() == 1 && dir->listNames()[0] == "foo"); 778 replacer->commit(); 779 } 780 781 // Replaced with new contents. 782 KJ_EXPECT(!dir->exists(Path({"foo", "bar"}))); 783 KJ_EXPECT(dir->openFile(Path({"foo", "corge"}))->readAllText() == "bazqux"); 784 } 785 #endif // !__CYGWIN__ 786 787 KJ_TEST("DiskDirectory replace directory with file") { 788 TempDir tempDir; 789 auto dir = tempDir.get(); 790 791 dir->openFile(Path({"foo", "bar"}), WriteMode::CREATE | WriteMode::CREATE_PARENT) 792 ->writeAll("foobar"); 793 794 { 795 // CREATE fails -- already exists. 796 auto replacer = dir->replaceFile(Path("foo"), WriteMode::CREATE); 797 replacer->get().writeAll("bazqux"); 798 KJ_EXPECT(!replacer->tryCommit()); 799 } 800 801 // Still a directory. 802 KJ_EXPECT(dir->lstat(Path("foo")).type == FsNode::Type::DIRECTORY); 803 804 { 805 // MODIFY succeeds. 806 auto replacer = dir->replaceFile(Path("foo"), WriteMode::MODIFY); 807 replacer->get().writeAll("bazqux"); 808 replacer->commit(); 809 } 810 811 // Replaced with file. 812 KJ_EXPECT(dir->openFile(Path("foo"))->readAllText() == "bazqux"); 813 } 814 815 KJ_TEST("DiskDirectory replace file with directory") { 816 TempDir tempDir; 817 auto dir = tempDir.get(); 818 819 dir->openFile(Path("foo"), WriteMode::CREATE) 820 ->writeAll("foobar"); 821 822 { 823 // CREATE fails -- already exists. 824 auto replacer = dir->replaceSubdir(Path("foo"), WriteMode::CREATE); 825 replacer->get().openFile(Path("bar"), WriteMode::CREATE)->writeAll("bazqux"); 826 KJ_EXPECT(dir->listNames().size() == 1 && dir->listNames()[0] == "foo"); 827 KJ_EXPECT(!replacer->tryCommit()); 828 } 829 830 // Still a file. 831 KJ_EXPECT(dir->openFile(Path("foo"))->readAllText() == "foobar"); 832 833 { 834 // MODIFY succeeds. 835 auto replacer = dir->replaceSubdir(Path("foo"), WriteMode::MODIFY); 836 replacer->get().openFile(Path("bar"), WriteMode::CREATE)->writeAll("bazqux"); 837 KJ_EXPECT(dir->listNames().size() == 1 && dir->listNames()[0] == "foo"); 838 replacer->commit(); 839 } 840 841 // Replaced with directory. 842 KJ_EXPECT(dir->openFile(Path({"foo", "bar"}))->readAllText() == "bazqux"); 843 } 844 845 #if !defined(HOLES_NOT_SUPPORTED) && (CAPNP_DEBUG_TYPES || CAPNP_EXPENSIVE_TESTS) 846 // Not all filesystems support sparse files, and if they do, they don't necessarily support 847 // copying them in a way that preserves holes. We don't want the capnp test suite to fail just 848 // because it was run on the wrong filesystem. We could design the test to check first if the 849 // filesystem supports holes, but the code to do that would be almost the same as the code being 850 // tested... Instead, we've marked this test so it only runs when building this library using 851 // defines that only the Cap'n Proto maintainers use. So, we run the test ourselves but we don't 852 // make other people run it. 853 854 KJ_TEST("DiskFile holes") { 855 if (isWine()) { 856 // WINE doesn't support sparse files. 857 return; 858 } 859 860 TempDir tempDir; 861 auto dir = tempDir.get(); 862 863 auto file = dir->openFile(Path("holes"), WriteMode::CREATE); 864 865 #if _WIN32 866 FILE_SET_SPARSE_BUFFER sparseInfo; 867 memset(&sparseInfo, 0, sizeof(sparseInfo)); 868 sparseInfo.SetSparse = TRUE; 869 DWORD dummy; 870 KJ_WIN32(DeviceIoControl( 871 KJ_ASSERT_NONNULL(file->getWin32Handle()), 872 FSCTL_SET_SPARSE, &sparseInfo, sizeof(sparseInfo), 873 NULL, 0, &dummy, NULL)); 874 #endif 875 876 file->writeAll("foobar"); 877 file->write(1 << 20, StringPtr("foobar").asBytes()); 878 879 // Some filesystems, like BTRFS, report zero `spaceUsed` until synced. 880 file->datasync(); 881 882 // Allow for block sizes as low as 512 bytes and as high as 64k. 883 auto meta = file->stat(); 884 KJ_EXPECT(meta.spaceUsed >= 2 * 512, meta.spaceUsed); 885 KJ_EXPECT(meta.spaceUsed <= 2 * 65536); 886 887 byte buf[7]; 888 889 #if !_WIN32 // Win32 CopyFile() does NOT preserve sparseness. 890 { 891 // Copy doesn't fill in holes. 892 dir->transfer(Path("copy"), WriteMode::CREATE, Path("holes"), TransferMode::COPY); 893 auto copy = dir->openFile(Path("copy")); 894 KJ_EXPECT(copy->stat().spaceUsed == meta.spaceUsed); 895 KJ_EXPECT(copy->read(0, buf) == 7); 896 KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == "foobar"); 897 898 KJ_EXPECT(copy->read(1 << 20, buf) == 6); 899 KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == "foobar"); 900 901 KJ_EXPECT(copy->read(1 << 19, buf) == 7); 902 KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == StringPtr("\0\0\0\0\0\0", 6)); 903 } 904 #endif 905 906 file->truncate(1 << 21); 907 file->datasync(); 908 KJ_EXPECT(file->stat().spaceUsed == meta.spaceUsed); 909 KJ_EXPECT(file->read(1 << 20, buf) == 7); 910 KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == "foobar"); 911 912 #if !_WIN32 // Win32 CopyFile() does NOT preserve sparseness. 913 { 914 dir->transfer(Path("copy"), WriteMode::MODIFY, Path("holes"), TransferMode::COPY); 915 auto copy = dir->openFile(Path("copy")); 916 KJ_EXPECT(copy->stat().spaceUsed == meta.spaceUsed); 917 KJ_EXPECT(copy->read(0, buf) == 7); 918 KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == "foobar"); 919 920 KJ_EXPECT(copy->read(1 << 20, buf) == 7); 921 KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == "foobar"); 922 923 KJ_EXPECT(copy->read(1 << 19, buf) == 7); 924 KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == StringPtr("\0\0\0\0\0\0", 6)); 925 } 926 #endif 927 928 // Try punching a hole with zero(). 929 #if _WIN32 930 uint64_t blockSize = 4096; // TODO(someday): Actually ask the OS. 931 #else 932 struct stat stats; 933 KJ_SYSCALL(fstat(KJ_ASSERT_NONNULL(file->getFd()), &stats)); 934 uint64_t blockSize = stats.st_blksize; 935 #endif 936 file->zero(1 << 20, blockSize); 937 file->datasync(); 938 #if !_WIN32 939 // TODO(someday): This doesn't work on Windows. I don't know why. We're definitely using the 940 // proper ioctl. Oh well. 941 KJ_EXPECT(file->stat().spaceUsed < meta.spaceUsed); 942 #endif 943 KJ_EXPECT(file->read(1 << 20, buf) == 7); 944 KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == StringPtr("\0\0\0\0\0\0", 6)); 945 } 946 #endif 947 948 } // namespace 949 } // namespace kj