capnproto

FORK: Cap'n Proto serialization/RPC system - core tools and C++ library
git clone https://git.neptards.moe/neptards/capnproto.git
Log | Files | Refs | README | LICENSE

module-loader.c++ (10577B)

      1 // Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
      2 // Licensed under the MIT License:
      3 //
      4 // Permission is hereby granted, free of charge, to any person obtaining a copy
      5 // of this software and associated documentation files (the "Software"), to deal
      6 // in the Software without restriction, including without limitation the rights
      7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
      8 // copies of the Software, and to permit persons to whom the Software is
      9 // furnished to do so, subject to the following conditions:
     10 //
     11 // The above copyright notice and this permission notice shall be included in
     12 // all copies or substantial portions of the Software.
     13 //
     14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
     17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
     18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
     19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
     20 // THE SOFTWARE.
     21 
     22 #include "module-loader.h"
     23 #include "lexer.h"
     24 #include "parser.h"
     25 #include <kj/vector.h>
     26 #include <kj/mutex.h>
     27 #include <kj/debug.h>
     28 #include <kj/io.h>
     29 #include <capnp/message.h>
     30 #include <unordered_map>
     31 
     32 namespace capnp {
     33 namespace compiler {
     34 
     35 namespace {
     36 
     37 struct FileKey {
     38   // Key type for the modules map. We need to implement some complicated heuristics to detect when
     39   // two files are actually the same underlying file on disk, in order to handle the case where
     40   // people have mapped the same file into multiple locations in the import tree, whether by
     41   // passing overlapping import paths, weird symlinks, or whatever.
     42   //
     43   // This is probably over-engineered.
     44 
     45   const kj::ReadableDirectory& baseDir;
     46   kj::PathPtr path;
     47   kj::Maybe<const kj::ReadableFile&> file;
     48   uint64_t hashCode;
     49   uint64_t size;
     50   kj::Date lastModified;
     51 
     52   FileKey(const kj::ReadableDirectory& baseDir, kj::PathPtr path)
     53       : baseDir(baseDir), path(path), file(nullptr),
     54         hashCode(0), size(0), lastModified(kj::UNIX_EPOCH) {}
     55   FileKey(const kj::ReadableDirectory& baseDir, kj::PathPtr path, const kj::ReadableFile& file)
     56       : FileKey(baseDir, path, file, file.stat()) {}
     57 
     58   FileKey(const kj::ReadableDirectory& baseDir, kj::PathPtr path, const kj::ReadableFile& file,
     59           kj::FsNode::Metadata meta)
     60       : baseDir(baseDir), path(path), file(&file),
     61         hashCode(meta.hashCode), size(meta.size), lastModified(meta.lastModified) {}
     62 
     63   bool operator==(const FileKey& other) const {
     64     // Allow matching on baseDir and path without a file.
     65     if (&baseDir == &other.baseDir && path == other.path) return true;
     66     if (file == nullptr || other.file == nullptr) return false;
     67 
     68     // Try comparing various file metadata to rule out obvious differences.
     69     if (hashCode != other.hashCode) return false;
     70     if (size != other.size || lastModified != other.lastModified) return false;
     71     if (path.size() > 0 && other.path.size() > 0 &&
     72         path[path.size() - 1] != other.path[other.path.size() - 1]) {
     73       // Names differ, so probably not the same file.
     74       return false;
     75     }
     76 
     77     // Same file hash, but different paths, but same size and modification date. This could be a
     78     // case of two different import paths overlapping and containing the same file. We'll need to
     79     // check the content.
     80     auto mapping1 = KJ_ASSERT_NONNULL(file).mmap(0, size);
     81     auto mapping2 = KJ_ASSERT_NONNULL(other.file).mmap(0, size);
     82     if (memcmp(mapping1.begin(), mapping2.begin(), size) != 0) return false;
     83 
     84     if (path == other.path) {
     85       // Exactly the same content was mapped at exactly the same path relative to two different
     86       // import directories. This can only really happen if this was one of the files passed on
     87       // the command line, but its --src-prefix is not also an import path, but some other
     88       // directory containing the same file was given as an import path. Whatever, we'll ignore
     89       // this.
     90       return true;
     91     }
     92 
     93     // Exactly the same content!
     94     static bool warned = false;
     95     if (!warned) {
     96       KJ_LOG(WARNING,
     97           "Found exactly the same source file mapped at two different paths. This suggests "
     98           "that your -I and --src-prefix flags are overlapping or inconsistent. Remember, these "
     99           "flags should only specify directories that are logical 'roots' of the source tree. "
    100           "It should never be the case that one of the import directories contains another one of "
    101           "them.",
    102           path, other.path);
    103       warned = true;
    104     }
    105 
    106     return true;
    107   }
    108 };
    109 
    110 struct FileKeyHash {
    111   size_t operator()(const FileKey& key) const {
    112     if (sizeof(size_t) < sizeof(key.hashCode)) {
    113       // 32-bit system, do more mixing
    114       return (key.hashCode >> 32) * 31 + static_cast<size_t>(key.hashCode) +
    115           key.size * 103 + (key.lastModified - kj::UNIX_EPOCH) / kj::MILLISECONDS * 73;
    116     } else {
    117       return key.hashCode + key.size * 103 +
    118           (key.lastModified - kj::UNIX_EPOCH) / kj::NANOSECONDS * 73ull;
    119     }
    120   }
    121 };
    122 
    123 };
    124 
    125 class ModuleLoader::Impl {
    126 public:
    127   Impl(GlobalErrorReporter& errorReporter)
    128       : errorReporter(errorReporter) {}
    129 
    130   void addImportPath(const kj::ReadableDirectory& dir) {
    131     searchPath.add(&dir);
    132   }
    133 
    134   kj::Maybe<Module&> loadModule(const kj::ReadableDirectory& dir, kj::PathPtr path);
    135   kj::Maybe<Module&> loadModuleFromSearchPath(kj::PathPtr path);
    136   kj::Maybe<kj::Array<const byte>> readEmbed(const kj::ReadableDirectory& dir, kj::PathPtr path);
    137   kj::Maybe<kj::Array<const byte>> readEmbedFromSearchPath(kj::PathPtr path);
    138   GlobalErrorReporter& getErrorReporter() { return errorReporter; }
    139 
    140 private:
    141   GlobalErrorReporter& errorReporter;
    142   kj::Vector<const kj::ReadableDirectory*> searchPath;
    143   std::unordered_map<FileKey, kj::Own<Module>, FileKeyHash> modules;
    144 };
    145 
    146 class ModuleLoader::ModuleImpl final: public Module {
    147 public:
    148   ModuleImpl(ModuleLoader::Impl& loader, kj::Own<const kj::ReadableFile> file,
    149              const kj::ReadableDirectory& sourceDir, kj::Path pathParam)
    150       : loader(loader), file(kj::mv(file)), sourceDir(sourceDir), path(kj::mv(pathParam)),
    151         sourceNameStr(path.toString()) {
    152     KJ_REQUIRE(path.size() > 0);
    153   }
    154 
    155   kj::PathPtr getPath() {
    156     return path;
    157   }
    158 
    159   kj::StringPtr getSourceName() override {
    160     return sourceNameStr;
    161   }
    162 
    163   Orphan<ParsedFile> loadContent(Orphanage orphanage) override {
    164     kj::Array<const char> content = file->mmap(0, file->stat().size).releaseAsChars();
    165 
    166     lineBreaks = nullptr;  // In case loadContent() is called multiple times.
    167     lineBreaks = lineBreaksSpace.construct(content);
    168 
    169     MallocMessageBuilder lexedBuilder;
    170     auto statements = lexedBuilder.initRoot<LexedStatements>();
    171     lex(content, statements, *this);
    172 
    173     auto parsed = orphanage.newOrphan<ParsedFile>();
    174     parseFile(statements.getStatements(), parsed.get(), *this);
    175     return parsed;
    176   }
    177 
    178   kj::Maybe<Module&> importRelative(kj::StringPtr importPath) override {
    179     if (importPath.size() > 0 && importPath[0] == '/') {
    180       return loader.loadModuleFromSearchPath(kj::Path::parse(importPath.slice(1)));
    181     } else {
    182       return loader.loadModule(sourceDir, path.parent().eval(importPath));
    183     }
    184   }
    185 
    186   kj::Maybe<kj::Array<const byte>> embedRelative(kj::StringPtr embedPath) override {
    187     if (embedPath.size() > 0 && embedPath[0] == '/') {
    188       return loader.readEmbedFromSearchPath(kj::Path::parse(embedPath.slice(1)));
    189     } else {
    190       return loader.readEmbed(sourceDir, path.parent().eval(embedPath));
    191     }
    192   }
    193 
    194   void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) override {
    195     auto& lines = *KJ_REQUIRE_NONNULL(lineBreaks,
    196         "Can't report errors until loadContent() is called.");
    197 
    198     loader.getErrorReporter().addError(sourceDir, path,
    199         lines.toSourcePos(startByte), lines.toSourcePos(endByte), message);
    200   }
    201 
    202   bool hadErrors() override {
    203     return loader.getErrorReporter().hadErrors();
    204   }
    205 
    206 private:
    207   ModuleLoader::Impl& loader;
    208   kj::Own<const kj::ReadableFile> file;
    209   const kj::ReadableDirectory& sourceDir;
    210   kj::Path path;
    211   kj::String sourceNameStr;
    212 
    213   kj::SpaceFor<LineBreakTable> lineBreaksSpace;
    214   kj::Maybe<kj::Own<LineBreakTable>> lineBreaks;
    215 };
    216 
    217 // =======================================================================================
    218 
    219 kj::Maybe<Module&> ModuleLoader::Impl::loadModule(
    220     const kj::ReadableDirectory& dir, kj::PathPtr path) {
    221   auto iter = modules.find(FileKey(dir, path));
    222   if (iter != modules.end()) {
    223     // Return existing file.
    224     return *iter->second;
    225   }
    226 
    227   KJ_IF_MAYBE(file, dir.tryOpenFile(path)) {
    228     auto pathCopy = path.clone();
    229     auto key = FileKey(dir, pathCopy, **file);
    230     auto module = kj::heap<ModuleImpl>(*this, kj::mv(*file), dir, kj::mv(pathCopy));
    231     auto& result = *module;
    232     auto insertResult = modules.insert(std::make_pair(key, kj::mv(module)));
    233     if (insertResult.second) {
    234       return result;
    235     } else {
    236       // Now that we have the file open, we noticed a collision. Return the old file.
    237       return *insertResult.first->second;
    238     }
    239   } else {
    240     // No such file.
    241     return nullptr;
    242   }
    243 }
    244 
    245 kj::Maybe<Module&> ModuleLoader::Impl::loadModuleFromSearchPath(kj::PathPtr path) {
    246   for (auto candidate: searchPath) {
    247     KJ_IF_MAYBE(module, loadModule(*candidate, path)) {
    248       return *module;
    249     }
    250   }
    251   return nullptr;
    252 }
    253 
    254 kj::Maybe<kj::Array<const byte>> ModuleLoader::Impl::readEmbed(
    255     const kj::ReadableDirectory& dir, kj::PathPtr path) {
    256   KJ_IF_MAYBE(file, dir.tryOpenFile(path)) {
    257     return file->get()->mmap(0, file->get()->stat().size);
    258   }
    259   return nullptr;
    260 }
    261 
    262 kj::Maybe<kj::Array<const byte>> ModuleLoader::Impl::readEmbedFromSearchPath(kj::PathPtr path) {
    263   for (auto candidate: searchPath) {
    264     KJ_IF_MAYBE(module, readEmbed(*candidate, path)) {
    265       return kj::mv(*module);
    266     }
    267   }
    268   return nullptr;
    269 }
    270 
    271 // =======================================================================================
    272 
    273 ModuleLoader::ModuleLoader(GlobalErrorReporter& errorReporter)
    274     : impl(kj::heap<Impl>(errorReporter)) {}
    275 ModuleLoader::~ModuleLoader() noexcept(false) {}
    276 
    277 void ModuleLoader::addImportPath(const kj::ReadableDirectory& dir) {
    278   impl->addImportPath(dir);
    279 }
    280 
    281 kj::Maybe<Module&> ModuleLoader::loadModule(const kj::ReadableDirectory& dir, kj::PathPtr path) {
    282   return impl->loadModule(dir, path);
    283 }
    284 
    285 }  // namespace compiler
    286 }  // namespace capnp