neptools

Modding tools to Neptunia games
git clone https://git.neptards.moe/neptards/neptools.git
Log | Files | Refs | Submodules | README | LICENSE

source.cpp (10735B)

      1 #include "source.hpp"
      2 
      3 #include "sink.hpp"
      4 
      5 #include <libshit/doctest.hpp>
      6 #include <libshit/except.hpp>
      7 #include <libshit/lua/function_call.hpp>
      8 #include <libshit/platform.hpp>
      9 #include <libshit/string_utils.hpp>
     10 
     11 #include <fstream>
     12 #include <iostream>
     13 
     14 #if !LIBSHIT_OS_IS_WINDOWS
     15 #  include <unistd.h>
     16 #endif
     17 
     18 #define LIBSHIT_LOG_NAME "source"
     19 #include <libshit/logger_helper.hpp>
     20 
     21 namespace Neptools
     22 {
     23   TEST_SUITE_BEGIN("Neptools::Source");
     24 
     25   namespace
     26   {
     27 
     28     template <typename T>
     29     struct UnixLike : public Source::Provider
     30     {
     31       UnixLike(Libshit::LowIo io, boost::filesystem::path file_name,
     32                FilePosition size)
     33         : Source::Provider{Libshit::Move(file_name), size},
     34           io{Libshit::Move(io)} {}
     35 
     36       void Destroy() noexcept;
     37 
     38       void Pread(FilePosition offs, Byte* buf, FileMemSize len) override;
     39       void EnsureChunk(FilePosition i);
     40 
     41       Libshit::LowIo io;
     42     };
     43 
     44     struct MmapProvider final : public UnixLike<MmapProvider>
     45     {
     46       MmapProvider(Libshit::LowIo&& fd, boost::filesystem::path file_name,
     47                    FilePosition size);
     48       ~MmapProvider() noexcept override { Destroy(); }
     49 
     50       static FileMemSize CHUNK_SIZE;
     51       void* ReadChunk(FilePosition offs, FileMemSize size);
     52       void DeleteChunk(size_t i);
     53     };
     54 
     55     struct UnixProvider final : public UnixLike<UnixProvider>
     56     {
     57       //using UnixLike::UnixLike;
     58       // workaround clang bug...
     59       UnixProvider(Libshit::LowIo&& io, boost::filesystem::path file_name,
     60                    FilePosition size)
     61         : UnixLike{Libshit::Move(io), Libshit::Move(file_name), size} {}
     62       ~UnixProvider() noexcept override { Destroy(); }
     63 
     64       static FileMemSize CHUNK_SIZE;
     65       void* ReadChunk(FilePosition offs, FileMemSize size);
     66       void DeleteChunk(size_t i);
     67     };
     68 
     69     struct StringProvider final : public Source::Provider
     70     {
     71       StringProvider(boost::filesystem::path file_name, std::string str)
     72         : Source::Provider(Libshit::Move(file_name), str.size()),
     73           str{Libshit::Move(str)}
     74       {
     75         LruPush(reinterpret_cast<const Byte*>(this->str.data()),
     76                 0, this->str.size());
     77       }
     78 
     79       void Pread(FilePosition, Byte*, FileMemSize) override
     80       { LIBSHIT_UNREACHABLE("StringProvider Pread"); }
     81 
     82       std::string str;
     83     };
     84 
     85     struct UniquePtrProvider final : public Source::Provider
     86     {
     87       UniquePtrProvider(boost::filesystem::path file_name,
     88                         std::unique_ptr<char[]> data, std::size_t len)
     89         : Source::Provider(Libshit::Move(file_name), len),
     90           data{Libshit::Move(data)}
     91       { LruPush(reinterpret_cast<const Byte*>(this->data.get()), 0, len); }
     92 
     93       void Pread(FilePosition, Byte*, FileMemSize) override
     94       { LIBSHIT_UNREACHABLE("UniquePtrProvider Pread"); }
     95 
     96       std::unique_ptr<char[]> data;
     97     };
     98 
     99   }
    100 
    101 
    102   Source Source::FromFile(const boost::filesystem::path& fname)
    103   {
    104     LIBSHIT_ADD_INFOS(return FromFile_(fname), "File name", fname.string());
    105   }
    106 
    107   Source Source::FromFile_(const boost::filesystem::path& fname)
    108   {
    109     Libshit::LowIo io{fname.c_str(), Libshit::LowIo::Permission::READ_ONLY,
    110       Libshit::LowIo::Mode::OPEN_ONLY};
    111 
    112     FilePosition size = io.GetSize();
    113 
    114     Libshit::SmartPtr<Provider> p;
    115     try { p = Libshit::MakeSmart<MmapProvider>(Libshit::Move(io), fname, size); }
    116     catch (const Libshit::SystemError& e)
    117     {
    118       WARN << "Mmap failed, falling back to normal reading: "
    119            << Libshit::PrintException(true) << std::endl;
    120       p = Libshit::MakeSmart<UnixProvider>(Libshit::Move(io), fname, size);
    121     }
    122     return Libshit::MakeNotNull(Libshit::Move(p));
    123   }
    124 
    125   Source Source::FromFd(
    126     boost::filesystem::path fname, Libshit::LowIo::FdType fd, bool owning)
    127   {
    128     Libshit::LowIo io{fd, owning};
    129     auto size = io.GetSize();
    130     return {Libshit::MakeSmart<UnixProvider>(
    131         Libshit::Move(io), Libshit::Move(fname), size)};
    132   }
    133 
    134   Source Source::FromMemory(boost::filesystem::path fname, std::string str)
    135   {
    136     return {Libshit::MakeSmart<StringProvider>(
    137         Libshit::Move(fname), Libshit::Move(str))};
    138   }
    139 
    140   Source Source::FromMemory(boost::filesystem::path fname,
    141                             std::unique_ptr<char[]> data, std::size_t len)
    142   {
    143     return {Libshit::MakeSmart<UniquePtrProvider>(
    144         Libshit::Move(fname), Libshit::Move(data), len)};
    145   }
    146 
    147 
    148   void Source::Pread_(FilePosition offs, Byte* buf, FileMemSize len) const
    149   {
    150     offs += offset;
    151     while (len)
    152     {
    153       if (p->LruGet(offs))
    154       {
    155         auto& x = p->lru[0];
    156         auto buf_offs = offs - x.offset;
    157         auto to_cpy = std::min<FilePosition>(len, x.size - buf_offs);
    158         memcpy(buf, x.ptr + buf_offs, to_cpy);
    159         offs += to_cpy;
    160         buf += to_cpy;
    161         len -= to_cpy;
    162       }
    163       else
    164         return p->Pread(offs, buf, len);
    165     }
    166   }
    167 
    168   std::string Source::PreadCString(FilePosition offs) const
    169   {
    170     std::string ret;
    171     Libshit::StringView e;
    172     size_t len;
    173     do
    174     {
    175       e = GetChunk(offs);
    176       len = strnlen(e.data(), e.size());
    177       ret.append(e.data(), len);
    178       offs += e.size();
    179     } while (len == e.size());
    180     return ret;
    181   }
    182 
    183   Source::BufEntry Source::GetTemporaryEntry(FilePosition offs) const
    184   {
    185     if (p->LruGet(offs)) return p->lru[0];
    186     p->Pread(offs, nullptr, 0);
    187     LIBSHIT_ASSERT(p->lru[0].offset <= offs &&
    188                    p->lru[0].offset + p->lru[0].size > offs);
    189     return p->lru[0];
    190   }
    191 
    192   Libshit::StringView Source::GetChunk(FilePosition offs) const
    193   {
    194     LIBSHIT_ASSERT(offs < size);
    195     auto e = GetTemporaryEntry(offs + offset);
    196     auto eoffs = offs + offset - e.offset;
    197     auto size = std::min(e.size - eoffs, GetSize() - offs);
    198     return { e.ptr + eoffs, std::size_t(size) };
    199   }
    200 
    201 
    202   void Source::Provider::LruPush(
    203     const Byte* ptr, FilePosition offset, FileMemSize size)
    204   {
    205     memmove(&lru[1], &lru[0], sizeof(BufEntry)*(lru.size()-1));
    206     lru[0].ptr = ptr;
    207     lru[0].offset = offset;
    208     lru[0].size = size;
    209   }
    210 
    211   bool Source::Provider::LruGet(FilePosition offs)
    212   {
    213     for (size_t i = 0; i < lru.size(); ++i)
    214     {
    215       auto x = lru[i];
    216       if (x.offset <= offs && x.offset + x.size > offs)
    217       {
    218         LIBSHIT_ASSERT(x.ptr);
    219         memmove(&lru[1], &lru[0], sizeof(BufEntry)*i);
    220         lru[0] = x;
    221         return true;
    222       }
    223     }
    224     return false;
    225   }
    226 
    227   template <typename T>
    228   void UnixLike<T>::Destroy() noexcept
    229   {
    230     for (size_t i = 0; i < lru.size(); ++i)
    231       if (lru[i].size)
    232         static_cast<T*>(this)->DeleteChunk(i);
    233   }
    234 
    235   template <typename T>
    236   void UnixLike<T>::Pread(FilePosition offs, Byte* buf, FileMemSize len)
    237   {
    238     if (len > static_cast<T*>(this)->CHUNK_SIZE)
    239       return io.Pread(buf, len, offs);
    240 
    241     if (len == 0) EnsureChunk(offs); // TODO: GetTemporaryEntry hack
    242     while (len)
    243     {
    244       EnsureChunk(offs);
    245       auto buf_offs = offs - lru[0].offset;
    246       auto to_cpy = std::min<FilePosition>(len, lru[0].size - buf_offs);
    247       memcpy(buf, lru[0].ptr + buf_offs, to_cpy);
    248       buf += to_cpy;
    249       offs += to_cpy;
    250       len -= to_cpy;
    251     }
    252   }
    253 
    254   template <typename T>
    255   void UnixLike<T>::EnsureChunk(FilePosition offs)
    256   {
    257     auto const CHUNK_SIZE = static_cast<T*>(this)->CHUNK_SIZE;
    258     auto ch_offs = offs/CHUNK_SIZE*CHUNK_SIZE;
    259     if (LruGet(offs)) return;
    260 
    261     auto size = std::min<FilePosition>(CHUNK_SIZE, this->size-ch_offs);
    262     auto x = static_cast<T*>(this)->ReadChunk(ch_offs, size);
    263     static_cast<T*>(this)->DeleteChunk(lru.size()-1);
    264     LruPush(static_cast<Byte*>(x), ch_offs, size);
    265   }
    266 
    267   FileMemSize MmapProvider::CHUNK_SIZE = MMAP_CHUNK;
    268   MmapProvider::MmapProvider(
    269     Libshit::LowIo&& io, boost::filesystem::path file_name, FilePosition size)
    270     : UnixLike{{}, Libshit::Move(file_name), size}
    271   {
    272     std::size_t to_map = size < MMAP_LIMIT ? size : MMAP_CHUNK;
    273 
    274     io.PrepareMmap(false);
    275     void* ptr = io.Mmap(0, to_map, false).Release();
    276 #if !LIBSHIT_OS_IS_WINDOWS
    277     if (to_map == size) io.Reset();
    278 #endif
    279     this->io = Libshit::Move(io);
    280 
    281     lru[0].ptr = static_cast<Byte*>(ptr);
    282     lru[0].offset = 0;
    283     lru[0].size = to_map;
    284   }
    285 
    286   void* MmapProvider::ReadChunk(FilePosition offs, FileMemSize size)
    287   {
    288     return io.Mmap(offs, size, false).Release();
    289   }
    290 
    291   void MmapProvider::DeleteChunk(size_t i)
    292   {
    293     if (lru[i].ptr)
    294       Libshit::LowIo::Munmap(const_cast<Byte*>(lru[i].ptr), lru[i].size);
    295   }
    296 
    297   FileMemSize UnixProvider::CHUNK_SIZE = MEM_CHUNK;
    298 
    299   void* UnixProvider::ReadChunk(FilePosition offs, FileMemSize size)
    300   {
    301     std::unique_ptr<Byte[]> x{new Byte[size]};
    302     io.Pread(x.get(), size, offs);
    303     return x.release();
    304   }
    305 
    306   void UnixProvider::DeleteChunk(size_t i)
    307   {
    308     delete[] lru[i].ptr;
    309   }
    310 
    311   void Source::Inspect(std::ostream& os) const
    312   {
    313     os << "neptools.source.from_memory("
    314        << Libshit::Quoted(GetFileName().string()) << ", "
    315        << Quoted(*this)  << ")";
    316   }
    317 
    318   std::string Source::Inspect() const
    319   {
    320     std::stringstream ss;
    321     Inspect(ss);
    322     return ss.str();
    323   }
    324 
    325   void Source::Dump(Sink& sink) const
    326   {
    327     FilePosition offset = 0;
    328     auto size = GetSize();
    329     while (offset < size)
    330     {
    331       auto chunk = GetChunk(offset);
    332       sink.Write(chunk);
    333       offset += chunk.size();
    334     }
    335   }
    336 
    337   void DumpableSource::Inspect_(std::ostream& os, unsigned) const
    338   {
    339     os << "neptools.dumpable_source(";
    340     src.Inspect(os);
    341     os << ')';
    342   }
    343 
    344   std::string to_string(const Source& src)
    345   {
    346     std::stringstream ss;
    347     ss << src.GetFileName() << ", pos: " << src.Tell();
    348     return ss.str();
    349   }
    350 
    351 #if LIBSHIT_WITH_LUA
    352 
    353   LIBSHIT_LUAGEN(name: "read")
    354   static Libshit::Lua::RetNum LuaRead(
    355     Libshit::Lua::StateRef vm, Source& src, FileMemSize len)
    356   {
    357     std::unique_ptr<char[]> ptr{new char[len]};
    358     src.Read<Libshit::Check::Throw>(ptr.get(), len);
    359     lua_pushlstring(vm, ptr.get(), len);
    360     return {1};
    361   }
    362 
    363   LIBSHIT_LUAGEN(name: "pread")
    364   static Libshit::Lua::RetNum LuaPread(
    365     Libshit::Lua::StateRef vm, Source& src, FilePosition offs, FileMemSize len)
    366   {
    367     std::unique_ptr<char[]> ptr{new char[len]};
    368     src.Pread<Libshit::Check::Throw>(offs, ptr.get(), len);
    369     lua_pushlstring(vm, ptr.get(), len);
    370     return {1};
    371   }
    372 
    373 #endif
    374 
    375   TEST_CASE("small source")
    376   {
    377     char buf[16] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
    378     std::ofstream os{"neptools_test.tmp", std::ios_base::binary};
    379     os.write(buf, 16);
    380     os.close();
    381 
    382     auto src = Source::FromFile("neptools_test.tmp");
    383     char buf2[16];
    384     src.ReadGen(buf2);
    385     REQUIRE(memcmp(buf, buf2, 16) == 0);
    386     CHECK(src.Inspect() ==
    387           R"(neptools.source.from_memory("neptools_test.tmp", "\x00\x01\x02\x03\x04\x05\x06\a\b\t\n\v\f\r\x0e\x0f"))");
    388   }
    389   TEST_SUITE_END();
    390 }
    391 
    392 #include "source.binding.hpp"