node-translator.c++ (92489B)
1 // Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors 2 // Licensed under the MIT License: 3 // 4 // Permission is hereby granted, free of charge, to any person obtaining a copy 5 // of this software and associated documentation files (the "Software"), to deal 6 // in the Software without restriction, including without limitation the rights 7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 // copies of the Software, and to permit persons to whom the Software is 9 // furnished to do so, subject to the following conditions: 10 // 11 // The above copyright notice and this permission notice shall be included in 12 // all copies or substantial portions of the Software. 13 // 14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 // THE SOFTWARE. 21 22 #include "node-translator.h" 23 #include "parser.h" // only for generateGroupId() and expressionString() 24 #include <capnp/serialize.h> 25 #include <kj/debug.h> 26 #include <kj/arena.h> 27 #include <kj/encoding.h> 28 #include <set> 29 #include <map> 30 #include <stdlib.h> 31 #include <capnp/stream.capnp.h> 32 33 namespace capnp { 34 namespace compiler { 35 36 bool shouldDetectIssue344() { 37 return getenv("CAPNP_IGNORE_ISSUE_344") == nullptr; 38 } 39 40 class NodeTranslator::StructLayout { 41 // Massive, disgusting class which implements the layout algorithm, which decides the offset 42 // for each field. 43 44 public: 45 template <typename UIntType> 46 struct HoleSet { 47 inline HoleSet(): holes{0, 0, 0, 0, 0, 0} {} 48 49 // Represents a set of "holes" within a segment of allocated space, up to one hole of each 50 // power-of-two size between 1 bit and 32 bits. 51 // 52 // The amount of "used" space in a struct's data segment can always be represented as a 53 // combination of a word count and a HoleSet. The HoleSet represents the space lost to 54 // "padding". 55 // 56 // There can never be more than one hole of any particular size. Why is this? Well, consider 57 // that every data field has a power-of-two size, every field must be aligned to a multiple of 58 // its size, and the maximum size of a single field is 64 bits. If we need to add a new field 59 // of N bits, there are two possibilities: 60 // 1. A hole of size N or larger exists. In this case, we find the smallest hole that is at 61 // least N bits. Let's say that that hole has size M. We allocate the first N bits of the 62 // hole to the new field. The remaining M - N bits become a series of holes of sizes N*2, 63 // N*4, ..., M / 2. We know no holes of these sizes existed before because we chose M to be 64 // the smallest available hole larger than N. So, there is still no more than one hole of 65 // each size, and no hole larger than any hole that existed previously. 66 // 2. No hole equal or larger N exists. In that case we extend the data section's size by one 67 // word, creating a new 64-bit hole at the end. We then allocate N bits from it, creating 68 // a series of holes between N and 64 bits, as described in point (1). Thus, again, there 69 // is still at most one hole of each size, and the largest hole is 32 bits. 70 71 UIntType holes[6]; 72 // The offset of each hole as a multiple of its size. A value of zero indicates that no hole 73 // exists. Notice that it is impossible for any actual hole to have an offset of zero, because 74 // the first field allocated is always placed at the very beginning of the section. So either 75 // the section has a size of zero (in which case there are no holes), or offset zero is 76 // already allocated and therefore cannot be a hole. 77 78 kj::Maybe<UIntType> tryAllocate(UIntType lgSize) { 79 // Try to find space for a field of size 2^lgSize within the set of holes. If found, 80 // remove it from the holes, and return its offset (as a multiple of its size). If there 81 // is no such space, returns zero (no hole can be at offset zero, as explained above). 82 83 if (lgSize >= kj::size(holes)) { 84 return nullptr; 85 } else if (holes[lgSize] != 0) { 86 UIntType result = holes[lgSize]; 87 holes[lgSize] = 0; 88 return result; 89 } else { 90 KJ_IF_MAYBE(next, tryAllocate(lgSize + 1)) { 91 UIntType result = *next * 2; 92 holes[lgSize] = result + 1; 93 return result; 94 } else { 95 return nullptr; 96 } 97 } 98 } 99 100 uint assertHoleAndAllocate(UIntType lgSize) { 101 KJ_ASSERT(holes[lgSize] != 0); 102 uint result = holes[lgSize]; 103 holes[lgSize] = 0; 104 return result; 105 } 106 107 void addHolesAtEnd(UIntType lgSize, UIntType offset, 108 UIntType limitLgSize = sizeof(HoleSet::holes) / sizeof(HoleSet::holes[0])) { 109 // Add new holes of progressively larger sizes in the range [lgSize, limitLgSize) starting 110 // from the given offset. The idea is that you just allocated an lgSize-sized field from 111 // an limitLgSize-sized space, such as a newly-added word on the end of the data segment. 112 113 KJ_DREQUIRE(limitLgSize <= kj::size(holes)); 114 115 while (lgSize < limitLgSize) { 116 KJ_DREQUIRE(holes[lgSize] == 0); 117 KJ_DREQUIRE(offset % 2 == 1); 118 holes[lgSize] = offset; 119 ++lgSize; 120 offset = (offset + 1) / 2; 121 } 122 } 123 124 bool tryExpand(UIntType oldLgSize, uint oldOffset, uint expansionFactor) { 125 // Try to expand the value at the given location by combining it with subsequent holes, so 126 // as to expand the location to be 2^expansionFactor times the size that it started as. 127 // (In other words, the new lgSize is oldLgSize + expansionFactor.) 128 129 if (expansionFactor == 0) { 130 // No expansion requested. 131 return true; 132 } 133 if (oldLgSize == kj::size(holes)) { 134 // Old value is already a full word. Further expansion is impossible. 135 return false; 136 } 137 KJ_ASSERT(oldLgSize < kj::size(holes)); 138 if (holes[oldLgSize] != oldOffset + 1) { 139 // The space immediately after the location is not a hole. 140 return false; 141 } 142 143 // We can expand the location by one factor by combining it with a hole. Try to further 144 // expand from there to the number of factors requested. 145 if (tryExpand(oldLgSize + 1, oldOffset >> 1, expansionFactor - 1)) { 146 // Success. Consume the hole. 147 holes[oldLgSize] = 0; 148 return true; 149 } else { 150 return false; 151 } 152 } 153 154 kj::Maybe<uint> smallestAtLeast(uint size) { 155 // Return the size of the smallest hole that is equal to or larger than the given size. 156 157 for (uint i = size; i < kj::size(holes); i++) { 158 if (holes[i] != 0) { 159 return i; 160 } 161 } 162 return nullptr; 163 } 164 165 uint getFirstWordUsed() { 166 // Computes the lg of the amount of space used in the first word of the section. 167 168 // If there is a 32-bit hole with a 32-bit offset, no more than the first 32 bits are used. 169 // If no more than the first 32 bits are used, and there is a 16-bit hole with a 16-bit 170 // offset, then no more than the first 16 bits are used. And so on. 171 for (uint i = kj::size(holes); i > 0; i--) { 172 if (holes[i - 1] != 1) { 173 return i; 174 } 175 } 176 return 0; 177 } 178 }; 179 180 struct StructOrGroup { 181 // Abstract interface for scopes in which fields can be added. 182 183 virtual void addVoid() = 0; 184 virtual uint addData(uint lgSize) = 0; 185 virtual uint addPointer() = 0; 186 virtual bool tryExpandData(uint oldLgSize, uint oldOffset, uint expansionFactor) = 0; 187 // Try to expand the given previously-allocated space by 2^expansionFactor. Succeeds -- 188 // returning true -- if the following space happens to be empty, making this expansion possible. 189 // Otherwise, returns false. 190 }; 191 192 struct Top: public StructOrGroup { 193 uint dataWordCount = 0; 194 uint pointerCount = 0; 195 // Size of the struct so far. 196 197 HoleSet<uint> holes; 198 199 void addVoid() override {} 200 201 uint addData(uint lgSize) override { 202 KJ_IF_MAYBE(hole, holes.tryAllocate(lgSize)) { 203 return *hole; 204 } else { 205 uint offset = dataWordCount++ << (6 - lgSize); 206 holes.addHolesAtEnd(lgSize, offset + 1); 207 return offset; 208 } 209 } 210 211 uint addPointer() override { 212 return pointerCount++; 213 } 214 215 bool tryExpandData(uint oldLgSize, uint oldOffset, uint expansionFactor) override { 216 return holes.tryExpand(oldLgSize, oldOffset, expansionFactor); 217 } 218 219 Top() = default; 220 KJ_DISALLOW_COPY(Top); 221 }; 222 223 struct Union { 224 struct DataLocation { 225 uint lgSize; 226 uint offset; 227 228 bool tryExpandTo(Union& u, uint newLgSize) { 229 if (newLgSize <= lgSize) { 230 return true; 231 } else if (u.parent.tryExpandData(lgSize, offset, newLgSize - lgSize)) { 232 offset >>= (newLgSize - lgSize); 233 lgSize = newLgSize; 234 return true; 235 } else { 236 return false; 237 } 238 } 239 }; 240 241 StructOrGroup& parent; 242 uint groupCount = 0; 243 kj::Maybe<uint> discriminantOffset; 244 kj::Vector<DataLocation> dataLocations; 245 kj::Vector<uint> pointerLocations; 246 247 inline Union(StructOrGroup& parent): parent(parent) {} 248 KJ_DISALLOW_COPY(Union); 249 250 uint addNewDataLocation(uint lgSize) { 251 // Add a whole new data location to the union with the given size. 252 253 uint offset = parent.addData(lgSize); 254 dataLocations.add(DataLocation { lgSize, offset }); 255 return offset; 256 } 257 258 uint addNewPointerLocation() { 259 // Add a whole new pointer location to the union with the given size. 260 261 return pointerLocations.add(parent.addPointer()); 262 } 263 264 void newGroupAddingFirstMember() { 265 if (++groupCount == 2) { 266 addDiscriminant(); 267 } 268 } 269 270 bool addDiscriminant() { 271 if (discriminantOffset == nullptr) { 272 discriminantOffset = parent.addData(4); // 2^4 = 16 bits 273 return true; 274 } else { 275 return false; 276 } 277 } 278 }; 279 280 struct Group final: public StructOrGroup { 281 public: 282 class DataLocationUsage { 283 public: 284 DataLocationUsage(): isUsed(false) {} 285 explicit DataLocationUsage(uint lgSize): isUsed(true), lgSizeUsed(lgSize) {} 286 287 kj::Maybe<uint> smallestHoleAtLeast(Union::DataLocation& location, uint lgSize) { 288 // Find the smallest single hole that is at least the given size. This is used to find the 289 // optimal place to allocate each field -- it is placed in the smallest slot where it fits, 290 // to reduce fragmentation. Returns the size of the hole, if found. 291 292 if (!isUsed) { 293 // The location is effectively one big hole. 294 if (lgSize <= location.lgSize) { 295 return location.lgSize; 296 } else { 297 return nullptr; 298 } 299 } else if (lgSize >= lgSizeUsed) { 300 // Requested size is at least our current usage, so clearly won't fit in any current 301 // holes, but if the location's size is larger than what we're using, we'd be able to 302 // expand. 303 if (lgSize < location.lgSize) { 304 return lgSize; 305 } else { 306 return nullptr; 307 } 308 } else KJ_IF_MAYBE(result, holes.smallestAtLeast(lgSize)) { 309 // There's a hole. 310 return *result; 311 } else { 312 // The requested size is smaller than what we're already using, but there are no holes 313 // available. If we could double our size, then we could allocate in the new space. 314 315 if (lgSizeUsed < location.lgSize) { 316 // We effectively create a new hole the same size as the current usage. 317 return lgSizeUsed; 318 } else { 319 return nullptr; 320 } 321 } 322 } 323 324 uint allocateFromHole(Group& group, Union::DataLocation& location, uint lgSize) { 325 // Allocate the given space from an existing hole, given smallestHoleAtLeast() already 326 // returned non-null indicating such a hole exists. 327 328 uint result; 329 330 if (!isUsed) { 331 // The location is totally unused, so just allocate from the beginning. 332 KJ_DASSERT(lgSize <= location.lgSize, "Did smallestHoleAtLeast() really find a hole?"); 333 result = 0; 334 isUsed = true; 335 lgSizeUsed = lgSize; 336 } else if (lgSize >= lgSizeUsed) { 337 // Requested size is at least our current usage, so clearly won't fit in any holes. 338 // We must expand to double the requested size, and return the second half. 339 KJ_DASSERT(lgSize < location.lgSize, "Did smallestHoleAtLeast() really find a hole?"); 340 holes.addHolesAtEnd(lgSizeUsed, 1, lgSize); 341 lgSizeUsed = lgSize + 1; 342 result = 1; 343 } else KJ_IF_MAYBE(hole, holes.tryAllocate(lgSize)) { 344 // Found a hole. 345 result = *hole; 346 } else { 347 // The requested size is smaller than what we're using so far, but didn't fit in a 348 // hole. We should double our "used" size, then allocate from the new space. 349 KJ_DASSERT(lgSizeUsed < location.lgSize, 350 "Did smallestHoleAtLeast() really find a hole?"); 351 result = 1 << (lgSizeUsed - lgSize); 352 holes.addHolesAtEnd(lgSize, result + 1, lgSizeUsed); 353 lgSizeUsed += 1; 354 } 355 356 // Adjust the offset according to the location's offset before returning. 357 uint locationOffset = location.offset << (location.lgSize - lgSize); 358 return locationOffset + result; 359 } 360 361 kj::Maybe<uint> tryAllocateByExpanding( 362 Group& group, Union::DataLocation& location, uint lgSize) { 363 // Attempt to allocate the given size by requesting that the parent union expand this 364 // location to fit. This is used if smallestHoleAtLeast() already determined that there 365 // are no holes that would fit, so we don't bother checking that. 366 367 if (!isUsed) { 368 if (location.tryExpandTo(group.parent, lgSize)) { 369 isUsed = true; 370 lgSizeUsed = lgSize; 371 return location.offset << (location.lgSize - lgSize); 372 } else { 373 return nullptr; 374 } 375 } else { 376 uint newSize = kj::max(lgSizeUsed, lgSize) + 1; 377 if (tryExpandUsage(group, location, newSize, true)) { 378 uint result = KJ_ASSERT_NONNULL(holes.tryAllocate(lgSize)); 379 uint locationOffset = location.offset << (location.lgSize - lgSize); 380 return locationOffset + result; 381 } else { 382 return nullptr; 383 } 384 } 385 } 386 387 bool tryExpand(Group& group, Union::DataLocation& location, 388 uint oldLgSize, uint oldOffset, uint expansionFactor) { 389 if (oldOffset == 0 && lgSizeUsed == oldLgSize) { 390 // This location contains exactly the requested data, so just expand the whole thing. 391 return tryExpandUsage(group, location, oldLgSize + expansionFactor, false); 392 } else { 393 // This location contains the requested data plus other stuff. Therefore the data cannot 394 // possibly expand past the end of the space we've already marked used without either 395 // overlapping with something else or breaking alignment rules. We only have to combine 396 // it with holes. 397 return holes.tryExpand(oldLgSize, oldOffset, expansionFactor); 398 } 399 } 400 401 private: 402 bool isUsed; 403 // Whether or not this location has been used at all by the group. 404 405 uint8_t lgSizeUsed; 406 // Amount of space from the location which is "used". This is the minimum size needed to 407 // cover all allocated space. Only meaningful if `isUsed` is true. 408 409 HoleSet<uint8_t> holes; 410 // Indicates holes present in the space designated by `lgSizeUsed`. The offsets in this 411 // HoleSet are relative to the beginning of this particular data location, not the beginning 412 // of the struct. 413 414 bool tryExpandUsage(Group& group, Union::DataLocation& location, uint desiredUsage, 415 bool newHoles) { 416 if (desiredUsage > location.lgSize) { 417 // Need to expand the underlying slot. 418 if (!location.tryExpandTo(group.parent, desiredUsage)) { 419 return false; 420 } 421 } 422 423 // Underlying slot is big enough, so expand our size and update holes. 424 if (newHoles) { 425 holes.addHolesAtEnd(lgSizeUsed, 1, desiredUsage); 426 } else if (shouldDetectIssue344()) { 427 // Unfortunately, Cap'n Proto 0.5.x and below would always call addHolesAtEnd(), which 428 // was the wrong thing to do when called from tryExpand(), which itself is only called 429 // in cases involving unions nested in other unions. The bug could lead to multiple 430 // fields in a group incorrectly being assigned overlapping offsets. Although the bug 431 // is now fixed by adding the `newHoles` parameter, this silently breaks 432 // backwards-compatibility with affected schemas. Therefore, for now, we throw an 433 // exception to alert developers of the problem. 434 // 435 // TODO(cleanup): Once sufficient time has elapsed, remove this assert. 436 KJ_FAIL_ASSERT("Bad news: Cap'n Proto 0.5.x and previous contained a bug which would cause this schema to be compiled incorrectly. Please see: https://github.com/sandstorm-io/capnproto/issues/344"); 437 } 438 lgSizeUsed = desiredUsage; 439 return true; 440 } 441 }; 442 443 Union& parent; 444 445 kj::Vector<DataLocationUsage> parentDataLocationUsage; 446 // Vector corresponding to the parent union's `dataLocations`, indicating how much of each 447 // location has already been allocated. 448 449 uint parentPointerLocationUsage = 0; 450 // Number of parent's pointer locations that have been used by this group. 451 452 bool hasMembers = false; 453 454 inline Group(Union& parent): parent(parent) {} 455 KJ_DISALLOW_COPY(Group); 456 457 void addMember() { 458 if (!hasMembers) { 459 hasMembers = true; 460 parent.newGroupAddingFirstMember(); 461 } 462 } 463 464 void addVoid() override { 465 addMember(); 466 467 // Make sure that if this is a member of a union which is in turn a member of another union, 468 // that we let the outer union know that a field is being added, even though it is a 469 // zero-size field. This is important because the union needs to allocate its discriminant 470 // just before its second member is added. 471 parent.parent.addVoid(); 472 } 473 474 uint addData(uint lgSize) override { 475 addMember(); 476 477 uint bestSize = kj::maxValue; 478 kj::Maybe<uint> bestLocation = nullptr; 479 480 for (uint i = 0; i < parent.dataLocations.size(); i++) { 481 // If we haven't seen this DataLocation yet, add a corresponding DataLocationUsage. 482 if (parentDataLocationUsage.size() == i) { 483 parentDataLocationUsage.add(); 484 } 485 486 auto& usage = parentDataLocationUsage[i]; 487 KJ_IF_MAYBE(hole, usage.smallestHoleAtLeast(parent.dataLocations[i], lgSize)) { 488 if (*hole < bestSize) { 489 bestSize = *hole; 490 bestLocation = i; 491 } 492 } 493 } 494 495 KJ_IF_MAYBE(best, bestLocation) { 496 return parentDataLocationUsage[*best].allocateFromHole( 497 *this, parent.dataLocations[*best], lgSize); 498 } 499 500 // There are no holes at all in the union big enough to fit this field. Go back through all 501 // of the locations and attempt to expand them to fit. 502 for (uint i = 0; i < parent.dataLocations.size(); i++) { 503 KJ_IF_MAYBE(result, parentDataLocationUsage[i].tryAllocateByExpanding( 504 *this, parent.dataLocations[i], lgSize)) { 505 return *result; 506 } 507 } 508 509 // Couldn't find any space in the existing locations, so add a new one. 510 uint result = parent.addNewDataLocation(lgSize); 511 parentDataLocationUsage.add(lgSize); 512 return result; 513 } 514 515 uint addPointer() override { 516 addMember(); 517 518 if (parentPointerLocationUsage < parent.pointerLocations.size()) { 519 return parent.pointerLocations[parentPointerLocationUsage++]; 520 } else { 521 parentPointerLocationUsage++; 522 return parent.addNewPointerLocation(); 523 } 524 } 525 526 bool tryExpandData(uint oldLgSize, uint oldOffset, uint expansionFactor) override { 527 bool mustFail = false; 528 if (oldLgSize + expansionFactor > 6 || 529 (oldOffset & ((1 << expansionFactor) - 1)) != 0) { 530 // Expansion is not possible because the new size is too large or the offset is not 531 // properly-aligned. 532 533 // Unfortunately, Cap'n Proto 0.5.x and prior forgot to "return false" here, instead 534 // continuing to execute the rest of the method. In most cases, the method failed later 535 // on, causing no harm. But, in cases where the method later succeeded, it probably 536 // led to bogus layouts. We cannot simply add the return statement now as this would 537 // silently break backwards-compatibility with affected schemas. Instead, we detect the 538 // problem and throw an exception. 539 // 540 // TODO(cleanup): Once sufficient time has elapsed, switch to "return false;" here. 541 if (shouldDetectIssue344()) { 542 mustFail = true; 543 } else { 544 return false; 545 } 546 } 547 548 for (uint i = 0; i < parentDataLocationUsage.size(); i++) { 549 auto& location = parent.dataLocations[i]; 550 if (location.lgSize >= oldLgSize && 551 oldOffset >> (location.lgSize - oldLgSize) == location.offset) { 552 // The location we're trying to expand is a subset of this data location. 553 auto& usage = parentDataLocationUsage[i]; 554 555 // Adjust the offset to be only within this location. 556 uint localOldOffset = oldOffset - (location.offset << (location.lgSize - oldLgSize)); 557 558 // Try to expand. 559 bool result = usage.tryExpand( 560 *this, location, oldLgSize, localOldOffset, expansionFactor); 561 if (mustFail && result) { 562 KJ_FAIL_ASSERT("Bad news: Cap'n Proto 0.5.x and previous contained a bug which would cause this schema to be compiled incorrectly. Please see: https://github.com/sandstorm-io/capnproto/issues/344"); 563 } 564 return result; 565 } 566 } 567 568 KJ_FAIL_ASSERT("Tried to expand field that was never allocated."); 569 return false; 570 } 571 }; 572 573 Top& getTop() { return top; } 574 575 private: 576 Top top; 577 }; 578 579 // ======================================================================================= 580 581 NodeTranslator::NodeTranslator( 582 Resolver& resolver, ErrorReporter& errorReporter, 583 const Declaration::Reader& decl, Orphan<schema::Node> wipNodeParam, 584 bool compileAnnotations) 585 : resolver(resolver), errorReporter(errorReporter), 586 orphanage(Orphanage::getForMessageContaining(wipNodeParam.get())), 587 compileAnnotations(compileAnnotations), 588 localBrand(kj::refcounted<BrandScope>( 589 errorReporter, wipNodeParam.getReader().getId(), 590 decl.getParameters().size(), resolver)), 591 wipNode(kj::mv(wipNodeParam)), 592 sourceInfo(orphanage.newOrphan<schema::Node::SourceInfo>()) { 593 compileNode(decl, wipNode.get()); 594 } 595 596 NodeTranslator::~NodeTranslator() noexcept(false) {} 597 598 NodeTranslator::NodeSet NodeTranslator::getBootstrapNode() { 599 auto sourceInfos = kj::heapArrayBuilder<schema::Node::SourceInfo::Reader>( 600 1 + groups.size() + paramStructs.size()); 601 sourceInfos.add(sourceInfo.getReader()); 602 for (auto& group: groups) { 603 sourceInfos.add(group.sourceInfo.getReader()); 604 } 605 for (auto& paramStruct: paramStructs) { 606 sourceInfos.add(paramStruct.sourceInfo.getReader()); 607 } 608 609 auto nodeReader = wipNode.getReader(); 610 if (nodeReader.isInterface()) { 611 return NodeSet { 612 nodeReader, 613 KJ_MAP(g, paramStructs) { return g.node.getReader(); }, 614 sourceInfos.finish() 615 }; 616 } else { 617 return NodeSet { 618 nodeReader, 619 KJ_MAP(g, groups) { return g.node.getReader(); }, 620 sourceInfos.finish() 621 }; 622 } 623 } 624 625 NodeTranslator::NodeSet NodeTranslator::finish(Schema selfBootstrapSchema) { 626 // Careful about iteration here: compileFinalValue() may actually add more elements to 627 // `unfinishedValues`, invalidating iterators in the process. 628 for (size_t i = 0; i < unfinishedValues.size(); i++) { 629 auto& value = unfinishedValues[i]; 630 compileValue(value.source, value.type, value.typeScope.orDefault(selfBootstrapSchema), 631 value.target, false); 632 } 633 634 return getBootstrapNode(); 635 } 636 637 class NodeTranslator::DuplicateNameDetector { 638 public: 639 inline explicit DuplicateNameDetector(ErrorReporter& errorReporter) 640 : errorReporter(errorReporter) {} 641 void check(List<Declaration>::Reader nestedDecls, Declaration::Which parentKind); 642 643 private: 644 ErrorReporter& errorReporter; 645 std::map<kj::StringPtr, LocatedText::Reader> names; 646 }; 647 648 void NodeTranslator::compileNode(Declaration::Reader decl, schema::Node::Builder builder) { 649 DuplicateNameDetector(errorReporter) 650 .check(decl.getNestedDecls(), decl.which()); 651 652 auto genericParams = decl.getParameters(); 653 if (genericParams.size() > 0) { 654 auto paramsBuilder = builder.initParameters(genericParams.size()); 655 for (auto i: kj::indices(genericParams)) { 656 paramsBuilder[i].setName(genericParams[i].getName()); 657 } 658 } 659 660 builder.setIsGeneric(localBrand->isGeneric()); 661 662 kj::StringPtr targetsFlagName; 663 664 switch (decl.which()) { 665 case Declaration::FILE: 666 targetsFlagName = "targetsFile"; 667 break; 668 case Declaration::CONST: 669 compileConst(decl.getConst(), builder.initConst()); 670 targetsFlagName = "targetsConst"; 671 break; 672 case Declaration::ANNOTATION: 673 compileAnnotation(decl.getAnnotation(), builder.initAnnotation()); 674 targetsFlagName = "targetsAnnotation"; 675 break; 676 case Declaration::ENUM: 677 compileEnum(decl.getEnum(), decl.getNestedDecls(), builder); 678 targetsFlagName = "targetsEnum"; 679 break; 680 case Declaration::STRUCT: 681 compileStruct(decl.getStruct(), decl.getNestedDecls(), builder); 682 targetsFlagName = "targetsStruct"; 683 break; 684 case Declaration::INTERFACE: 685 compileInterface(decl.getInterface(), decl.getNestedDecls(), builder); 686 targetsFlagName = "targetsInterface"; 687 break; 688 689 default: 690 KJ_FAIL_REQUIRE("This Declaration is not a node."); 691 break; 692 } 693 694 builder.adoptAnnotations(compileAnnotationApplications(decl.getAnnotations(), targetsFlagName)); 695 696 auto di = sourceInfo.get(); 697 di.setId(wipNode.getReader().getId()); 698 if (decl.hasDocComment()) { 699 di.setDocComment(decl.getDocComment()); 700 } 701 } 702 703 static kj::StringPtr getExpressionTargetName(Expression::Reader exp) { 704 kj::StringPtr targetName; 705 switch (exp.which()) { 706 case Expression::ABSOLUTE_NAME: 707 return exp.getAbsoluteName().getValue(); 708 case Expression::RELATIVE_NAME: 709 return exp.getRelativeName().getValue(); 710 case Expression::APPLICATION: 711 return getExpressionTargetName(exp.getApplication().getFunction()); 712 case Expression::MEMBER: 713 return exp.getMember().getName().getValue(); 714 default: 715 return nullptr; 716 } 717 } 718 719 void NodeTranslator::DuplicateNameDetector::check( 720 List<Declaration>::Reader nestedDecls, Declaration::Which parentKind) { 721 for (auto decl: nestedDecls) { 722 { 723 auto name = decl.getName(); 724 auto nameText = name.getValue(); 725 auto insertResult = names.insert(std::make_pair(nameText, name)); 726 if (!insertResult.second) { 727 if (nameText.size() == 0 && decl.isUnion()) { 728 errorReporter.addErrorOn( 729 name, kj::str("An unnamed union is already defined in this scope.")); 730 errorReporter.addErrorOn( 731 insertResult.first->second, kj::str("Previously defined here.")); 732 } else { 733 errorReporter.addErrorOn( 734 name, kj::str("'", nameText, "' is already defined in this scope.")); 735 errorReporter.addErrorOn( 736 insertResult.first->second, kj::str("'", nameText, "' previously defined here.")); 737 } 738 } 739 740 switch (decl.which()) { 741 case Declaration::USING: { 742 kj::StringPtr targetName = getExpressionTargetName(decl.getUsing().getTarget()); 743 if (targetName.size() > 0 && targetName[0] >= 'a' && targetName[0] <= 'z') { 744 // Target starts with lower-case letter, so alias should too. 745 if (nameText.size() > 0 && (nameText[0] < 'a' || nameText[0] > 'z')) { 746 errorReporter.addErrorOn(name, 747 "Non-type names must begin with a lower-case letter."); 748 } 749 } else { 750 // Target starts with capital or is not named (probably, an import). Require 751 // capitalization. 752 if (nameText.size() > 0 && (nameText[0] < 'A' || nameText[0] > 'Z')) { 753 errorReporter.addErrorOn(name, 754 "Type names must begin with a capital letter."); 755 } 756 } 757 break; 758 } 759 760 case Declaration::ENUM: 761 case Declaration::STRUCT: 762 case Declaration::INTERFACE: 763 if (nameText.size() > 0 && (nameText[0] < 'A' || nameText[0] > 'Z')) { 764 errorReporter.addErrorOn(name, 765 "Type names must begin with a capital letter."); 766 } 767 break; 768 769 case Declaration::CONST: 770 case Declaration::ANNOTATION: 771 case Declaration::ENUMERANT: 772 case Declaration::METHOD: 773 case Declaration::FIELD: 774 case Declaration::UNION: 775 case Declaration::GROUP: 776 if (nameText.size() > 0 && (nameText[0] < 'a' || nameText[0] > 'z')) { 777 errorReporter.addErrorOn(name, 778 "Non-type names must begin with a lower-case letter."); 779 } 780 break; 781 782 default: 783 KJ_ASSERT(nameText.size() == 0, "Don't know what naming rules to enforce for node type.", 784 (uint)decl.which()); 785 break; 786 } 787 788 if (nameText.findFirst('_') != nullptr) { 789 errorReporter.addErrorOn(name, 790 "Cap'n Proto declaration names should use camelCase and must not contain " 791 "underscores. (Code generators may convert names to the appropriate style for the " 792 "target language.)"); 793 } 794 } 795 796 switch (decl.which()) { 797 case Declaration::USING: 798 case Declaration::CONST: 799 case Declaration::ENUM: 800 case Declaration::STRUCT: 801 case Declaration::INTERFACE: 802 case Declaration::ANNOTATION: 803 switch (parentKind) { 804 case Declaration::FILE: 805 case Declaration::STRUCT: 806 case Declaration::INTERFACE: 807 // OK. 808 break; 809 default: 810 errorReporter.addErrorOn(decl, "This kind of declaration doesn't belong here."); 811 break; 812 } 813 break; 814 815 case Declaration::ENUMERANT: 816 if (parentKind != Declaration::ENUM) { 817 errorReporter.addErrorOn(decl, "Enumerants can only appear in enums."); 818 } 819 break; 820 case Declaration::METHOD: 821 if (parentKind != Declaration::INTERFACE) { 822 errorReporter.addErrorOn(decl, "Methods can only appear in interfaces."); 823 } 824 break; 825 case Declaration::FIELD: 826 case Declaration::UNION: 827 case Declaration::GROUP: 828 switch (parentKind) { 829 case Declaration::STRUCT: 830 case Declaration::UNION: 831 case Declaration::GROUP: 832 // OK. 833 break; 834 default: 835 errorReporter.addErrorOn(decl, "This declaration can only appear in structs."); 836 break; 837 } 838 839 // Struct members may have nested decls. We need to check those here, because no one else 840 // is going to do it. 841 if (decl.getName().getValue().size() == 0) { 842 // Unnamed union. Check members as if they are in the same scope. 843 check(decl.getNestedDecls(), decl.which()); 844 } else { 845 // Children are in their own scope. 846 DuplicateNameDetector(errorReporter) 847 .check(decl.getNestedDecls(), decl.which()); 848 } 849 850 break; 851 852 default: 853 errorReporter.addErrorOn(decl, "This kind of declaration doesn't belong here."); 854 break; 855 } 856 } 857 } 858 859 void NodeTranslator::compileConst(Declaration::Const::Reader decl, 860 schema::Node::Const::Builder builder) { 861 auto typeBuilder = builder.initType(); 862 if (compileType(decl.getType(), typeBuilder, ImplicitParams::none())) { 863 compileBootstrapValue(decl.getValue(), typeBuilder.asReader(), builder.initValue()); 864 } 865 } 866 867 void NodeTranslator::compileAnnotation(Declaration::Annotation::Reader decl, 868 schema::Node::Annotation::Builder builder) { 869 compileType(decl.getType(), builder.initType(), ImplicitParams::none()); 870 871 // Dynamically copy over the values of all of the "targets" members. 872 DynamicStruct::Reader src = decl; 873 DynamicStruct::Builder dst = builder; 874 for (auto srcField: src.getSchema().getFields()) { 875 kj::StringPtr fieldName = srcField.getProto().getName(); 876 if (fieldName.startsWith("targets")) { 877 auto dstField = dst.getSchema().getFieldByName(fieldName); 878 dst.set(dstField, src.get(srcField)); 879 } 880 } 881 } 882 883 class NodeTranslator::DuplicateOrdinalDetector { 884 public: 885 DuplicateOrdinalDetector(ErrorReporter& errorReporter): errorReporter(errorReporter) {} 886 887 void check(LocatedInteger::Reader ordinal) { 888 if (ordinal.getValue() < expectedOrdinal) { 889 errorReporter.addErrorOn(ordinal, "Duplicate ordinal number."); 890 KJ_IF_MAYBE(last, lastOrdinalLocation) { 891 errorReporter.addErrorOn( 892 *last, kj::str("Ordinal @", last->getValue(), " originally used here.")); 893 // Don't report original again. 894 lastOrdinalLocation = nullptr; 895 } 896 } else if (ordinal.getValue() > expectedOrdinal) { 897 errorReporter.addErrorOn(ordinal, 898 kj::str("Skipped ordinal @", expectedOrdinal, ". Ordinals must be sequential with no " 899 "holes.")); 900 expectedOrdinal = ordinal.getValue() + 1; 901 } else { 902 ++expectedOrdinal; 903 lastOrdinalLocation = ordinal; 904 } 905 } 906 907 private: 908 ErrorReporter& errorReporter; 909 uint expectedOrdinal = 0; 910 kj::Maybe<LocatedInteger::Reader> lastOrdinalLocation; 911 }; 912 913 void NodeTranslator::compileEnum(Void decl, 914 List<Declaration>::Reader members, 915 schema::Node::Builder builder) { 916 // maps ordinal -> (code order, declaration) 917 std::multimap<uint, std::pair<uint, Declaration::Reader>> enumerants; 918 919 uint codeOrder = 0; 920 for (auto member: members) { 921 if (member.isEnumerant()) { 922 enumerants.insert( 923 std::make_pair(member.getId().getOrdinal().getValue(), 924 std::make_pair(codeOrder++, member))); 925 } 926 } 927 928 auto list = builder.initEnum().initEnumerants(enumerants.size()); 929 auto sourceInfoList = sourceInfo.get().initMembers(enumerants.size()); 930 uint i = 0; 931 DuplicateOrdinalDetector dupDetector(errorReporter); 932 933 for (auto& entry: enumerants) { 934 uint codeOrder = entry.second.first; 935 Declaration::Reader enumerantDecl = entry.second.second; 936 937 dupDetector.check(enumerantDecl.getId().getOrdinal()); 938 939 if (enumerantDecl.hasDocComment()) { 940 sourceInfoList[i].setDocComment(enumerantDecl.getDocComment()); 941 } 942 943 auto enumerantBuilder = list[i++]; 944 enumerantBuilder.setName(enumerantDecl.getName().getValue()); 945 enumerantBuilder.setCodeOrder(codeOrder); 946 enumerantBuilder.adoptAnnotations(compileAnnotationApplications( 947 enumerantDecl.getAnnotations(), "targetsEnumerant")); 948 } 949 } 950 951 // ------------------------------------------------------------------- 952 953 class NodeTranslator::StructTranslator { 954 public: 955 explicit StructTranslator(NodeTranslator& translator, ImplicitParams implicitMethodParams) 956 : translator(translator), errorReporter(translator.errorReporter), 957 implicitMethodParams(implicitMethodParams) {} 958 KJ_DISALLOW_COPY(StructTranslator); 959 960 void translate(Void decl, List<Declaration>::Reader members, schema::Node::Builder builder, 961 schema::Node::SourceInfo::Builder sourceInfo) { 962 // Build the member-info-by-ordinal map. 963 MemberInfo root(builder, sourceInfo); 964 traverseTopOrGroup(members, root, layout.getTop()); 965 translateInternal(root, builder); 966 } 967 968 void translate(List<Declaration::Param>::Reader params, schema::Node::Builder builder, 969 schema::Node::SourceInfo::Builder sourceInfo) { 970 // Build a struct from a method param / result list. 971 MemberInfo root(builder, sourceInfo); 972 traverseParams(params, root, layout.getTop()); 973 translateInternal(root, builder); 974 } 975 976 private: 977 NodeTranslator& translator; 978 ErrorReporter& errorReporter; 979 ImplicitParams implicitMethodParams; 980 StructLayout layout; 981 kj::Arena arena; 982 983 struct NodeSourceInfoBuilderPair { 984 schema::Node::Builder node; 985 schema::Node::SourceInfo::Builder sourceInfo; 986 }; 987 988 struct FieldSourceInfoBuilderPair { 989 schema::Field::Builder field; 990 schema::Node::SourceInfo::Member::Builder sourceInfo; 991 }; 992 993 struct MemberInfo { 994 MemberInfo* parent; 995 // The MemberInfo for the parent scope. 996 997 uint codeOrder; 998 // Code order within the parent. 999 1000 uint index = 0; 1001 // Index within the parent. 1002 1003 uint childCount = 0; 1004 // Number of children this member has. 1005 1006 uint childInitializedCount = 0; 1007 // Number of children whose `schema` member has been initialized. This initialization happens 1008 // while walking the fields in ordinal order. 1009 1010 uint unionDiscriminantCount = 0; 1011 // Number of children who are members of the scope's union and have had their discriminant 1012 // value decided. 1013 1014 bool isInUnion; 1015 // Whether or not this field is in the parent's union. 1016 1017 kj::StringPtr name; 1018 Declaration::Id::Reader declId; 1019 Declaration::Which declKind; 1020 bool isParam = false; 1021 bool hasDefaultValue = false; // if declKind == FIELD 1022 Expression::Reader fieldType; // if declKind == FIELD 1023 Expression::Reader fieldDefaultValue; // if declKind == FIELD && hasDefaultValue 1024 List<Declaration::AnnotationApplication>::Reader declAnnotations; 1025 uint startByte = 0; 1026 uint endByte = 0; 1027 // Information about the field declaration. We don't use Declaration::Reader because it might 1028 // have come from a Declaration::Param instead. 1029 1030 kj::Maybe<Text::Reader> docComment = nullptr; 1031 1032 kj::Maybe<schema::Field::Builder> schema; 1033 // Schema for the field. Initialized when getSchema() is first called. 1034 1035 schema::Node::Builder node; 1036 schema::Node::SourceInfo::Builder sourceInfo; 1037 // If it's a group, or the top-level struct. 1038 1039 union { 1040 StructLayout::StructOrGroup* fieldScope; 1041 // If this member is a field, the scope of that field. This will be used to assign an 1042 // offset for the field when going through in ordinal order. 1043 1044 StructLayout::Union* unionScope; 1045 // If this member is a union, or it is a group or top-level struct containing an unnamed 1046 // union, this is the union. This will be used to assign a discriminant offset when the 1047 // union's ordinal comes up (if the union has an explicit ordinal), as well as to finally 1048 // copy over the discriminant offset to the schema. 1049 }; 1050 1051 inline explicit MemberInfo(schema::Node::Builder node, 1052 schema::Node::SourceInfo::Builder sourceInfo) 1053 : parent(nullptr), codeOrder(0), isInUnion(false), node(node), sourceInfo(sourceInfo), 1054 unionScope(nullptr) {} 1055 inline MemberInfo(MemberInfo& parent, uint codeOrder, 1056 const Declaration::Reader& decl, 1057 StructLayout::StructOrGroup& fieldScope, 1058 bool isInUnion) 1059 : parent(&parent), codeOrder(codeOrder), isInUnion(isInUnion), 1060 name(decl.getName().getValue()), declId(decl.getId()), declKind(Declaration::FIELD), 1061 declAnnotations(decl.getAnnotations()), 1062 startByte(decl.getStartByte()), endByte(decl.getEndByte()), 1063 node(nullptr), sourceInfo(nullptr), fieldScope(&fieldScope) { 1064 KJ_REQUIRE(decl.which() == Declaration::FIELD); 1065 auto fieldDecl = decl.getField(); 1066 fieldType = fieldDecl.getType(); 1067 if (fieldDecl.getDefaultValue().isValue()) { 1068 hasDefaultValue = true; 1069 fieldDefaultValue = fieldDecl.getDefaultValue().getValue(); 1070 } 1071 if (decl.hasDocComment()) { 1072 docComment = decl.getDocComment(); 1073 } 1074 } 1075 inline MemberInfo(MemberInfo& parent, uint codeOrder, 1076 const Declaration::Param::Reader& decl, 1077 StructLayout::StructOrGroup& fieldScope, 1078 bool isInUnion) 1079 : parent(&parent), codeOrder(codeOrder), isInUnion(isInUnion), 1080 name(decl.getName().getValue()), declKind(Declaration::FIELD), isParam(true), 1081 declAnnotations(decl.getAnnotations()), 1082 startByte(decl.getStartByte()), endByte(decl.getEndByte()), 1083 node(nullptr), sourceInfo(nullptr), fieldScope(&fieldScope) { 1084 fieldType = decl.getType(); 1085 if (decl.getDefaultValue().isValue()) { 1086 hasDefaultValue = true; 1087 fieldDefaultValue = decl.getDefaultValue().getValue(); 1088 } 1089 } 1090 inline MemberInfo(MemberInfo& parent, uint codeOrder, 1091 const Declaration::Reader& decl, 1092 NodeSourceInfoBuilderPair builderPair, 1093 bool isInUnion) 1094 : parent(&parent), codeOrder(codeOrder), isInUnion(isInUnion), 1095 name(decl.getName().getValue()), declId(decl.getId()), declKind(decl.which()), 1096 declAnnotations(decl.getAnnotations()), 1097 startByte(decl.getStartByte()), endByte(decl.getEndByte()), 1098 node(builderPair.node), sourceInfo(builderPair.sourceInfo), unionScope(nullptr) { 1099 KJ_REQUIRE(decl.which() != Declaration::FIELD); 1100 if (decl.hasDocComment()) { 1101 docComment = decl.getDocComment(); 1102 } 1103 } 1104 1105 schema::Field::Builder getSchema() { 1106 KJ_IF_MAYBE(result, schema) { 1107 return *result; 1108 } else { 1109 index = parent->childInitializedCount; 1110 auto builderPair = parent->addMemberSchema(); 1111 auto builder = builderPair.field; 1112 if (isInUnion) { 1113 builder.setDiscriminantValue(parent->unionDiscriminantCount++); 1114 } 1115 builder.setName(name); 1116 builder.setCodeOrder(codeOrder); 1117 1118 KJ_IF_MAYBE(dc, docComment) { 1119 builderPair.sourceInfo.setDocComment(*dc); 1120 } 1121 1122 schema = builder; 1123 return builder; 1124 } 1125 } 1126 1127 FieldSourceInfoBuilderPair addMemberSchema() { 1128 // Get the schema builder for the child member at the given index. This lazily/dynamically 1129 // builds the builder tree. 1130 1131 KJ_REQUIRE(childInitializedCount < childCount); 1132 1133 auto structNode = node.getStruct(); 1134 if (!structNode.hasFields()) { 1135 if (parent != nullptr) { 1136 getSchema(); // Make sure field exists in parent once the first child is added. 1137 } 1138 FieldSourceInfoBuilderPair result { 1139 structNode.initFields(childCount)[childInitializedCount], 1140 sourceInfo.initMembers(childCount)[childInitializedCount] 1141 }; 1142 ++childInitializedCount; 1143 return result; 1144 } else { 1145 FieldSourceInfoBuilderPair result { 1146 structNode.getFields()[childInitializedCount], 1147 sourceInfo.getMembers()[childInitializedCount] 1148 }; 1149 ++childInitializedCount; 1150 return result; 1151 } 1152 } 1153 1154 void finishGroup() { 1155 if (unionScope != nullptr) { 1156 unionScope->addDiscriminant(); // if it hasn't happened already 1157 auto structNode = node.getStruct(); 1158 structNode.setDiscriminantCount(unionDiscriminantCount); 1159 structNode.setDiscriminantOffset(KJ_ASSERT_NONNULL(unionScope->discriminantOffset)); 1160 } 1161 1162 if (parent != nullptr) { 1163 uint64_t groupId = generateGroupId(parent->node.getId(), index); 1164 node.setId(groupId); 1165 node.setScopeId(parent->node.getId()); 1166 getSchema().initGroup().setTypeId(groupId); 1167 1168 sourceInfo.setId(groupId); 1169 KJ_IF_MAYBE(dc, docComment) { 1170 sourceInfo.setDocComment(*dc); 1171 } 1172 } 1173 } 1174 }; 1175 1176 std::multimap<uint, MemberInfo*> membersByOrdinal; 1177 // Every member that has an explicit ordinal goes into this map. We then iterate over the map 1178 // to assign field offsets (or discriminant offsets for unions). 1179 1180 kj::Vector<MemberInfo*> allMembers; 1181 // All members, including ones that don't have ordinals. 1182 1183 void traverseUnion(const Declaration::Reader& decl, 1184 List<Declaration>::Reader members, MemberInfo& parent, 1185 StructLayout::Union& layout, uint& codeOrder) { 1186 if (members.size() < 2) { 1187 errorReporter.addErrorOn(decl, "Union must have at least two members."); 1188 } 1189 1190 for (auto member: members) { 1191 kj::Maybe<uint> ordinal; 1192 MemberInfo* memberInfo = nullptr; 1193 1194 switch (member.which()) { 1195 case Declaration::FIELD: { 1196 parent.childCount++; 1197 // For layout purposes, pretend this field is enclosed in a one-member group. 1198 StructLayout::Group& singletonGroup = 1199 arena.allocate<StructLayout::Group>(layout); 1200 memberInfo = &arena.allocate<MemberInfo>(parent, codeOrder++, member, singletonGroup, 1201 true); 1202 allMembers.add(memberInfo); 1203 ordinal = member.getId().getOrdinal().getValue(); 1204 break; 1205 } 1206 1207 case Declaration::UNION: 1208 if (member.getName().getValue() == "") { 1209 errorReporter.addErrorOn(member, "Unions cannot contain unnamed unions."); 1210 } else { 1211 parent.childCount++; 1212 1213 // For layout purposes, pretend this union is enclosed in a one-member group. 1214 StructLayout::Group& singletonGroup = 1215 arena.allocate<StructLayout::Group>(layout); 1216 StructLayout::Union& unionLayout = arena.allocate<StructLayout::Union>(singletonGroup); 1217 1218 memberInfo = &arena.allocate<MemberInfo>( 1219 parent, codeOrder++, member, 1220 newGroupNode(parent.node, member.getName().getValue()), 1221 true); 1222 allMembers.add(memberInfo); 1223 memberInfo->unionScope = &unionLayout; 1224 uint subCodeOrder = 0; 1225 traverseUnion(member, member.getNestedDecls(), *memberInfo, unionLayout, subCodeOrder); 1226 if (member.getId().isOrdinal()) { 1227 ordinal = member.getId().getOrdinal().getValue(); 1228 } 1229 } 1230 break; 1231 1232 case Declaration::GROUP: { 1233 parent.childCount++; 1234 StructLayout::Group& group = arena.allocate<StructLayout::Group>(layout); 1235 memberInfo = &arena.allocate<MemberInfo>( 1236 parent, codeOrder++, member, 1237 newGroupNode(parent.node, member.getName().getValue()), 1238 true); 1239 allMembers.add(memberInfo); 1240 traverseGroup(member.getNestedDecls(), *memberInfo, group); 1241 break; 1242 } 1243 1244 default: 1245 // Ignore others. 1246 break; 1247 } 1248 1249 KJ_IF_MAYBE(o, ordinal) { 1250 membersByOrdinal.insert(std::make_pair(*o, memberInfo)); 1251 } 1252 } 1253 } 1254 1255 void traverseGroup(List<Declaration>::Reader members, MemberInfo& parent, 1256 StructLayout::StructOrGroup& layout) { 1257 if (members.size() < 1) { 1258 errorReporter.addError(parent.startByte, parent.endByte, 1259 "Group must have at least one member."); 1260 } 1261 1262 traverseTopOrGroup(members, parent, layout); 1263 } 1264 1265 void traverseTopOrGroup(List<Declaration>::Reader members, MemberInfo& parent, 1266 StructLayout::StructOrGroup& layout) { 1267 uint codeOrder = 0; 1268 1269 for (auto member: members) { 1270 kj::Maybe<uint> ordinal; 1271 MemberInfo* memberInfo = nullptr; 1272 1273 switch (member.which()) { 1274 case Declaration::FIELD: { 1275 parent.childCount++; 1276 memberInfo = &arena.allocate<MemberInfo>( 1277 parent, codeOrder++, member, layout, false); 1278 allMembers.add(memberInfo); 1279 ordinal = member.getId().getOrdinal().getValue(); 1280 break; 1281 } 1282 1283 case Declaration::UNION: { 1284 StructLayout::Union& unionLayout = arena.allocate<StructLayout::Union>(layout); 1285 1286 uint independentSubCodeOrder = 0; 1287 uint* subCodeOrder = &independentSubCodeOrder; 1288 if (member.getName().getValue() == "") { 1289 memberInfo = &parent; 1290 subCodeOrder = &codeOrder; 1291 } else { 1292 parent.childCount++; 1293 memberInfo = &arena.allocate<MemberInfo>( 1294 parent, codeOrder++, member, 1295 newGroupNode(parent.node, member.getName().getValue()), 1296 false); 1297 allMembers.add(memberInfo); 1298 } 1299 memberInfo->unionScope = &unionLayout; 1300 traverseUnion(member, member.getNestedDecls(), *memberInfo, unionLayout, *subCodeOrder); 1301 if (member.getId().isOrdinal()) { 1302 ordinal = member.getId().getOrdinal().getValue(); 1303 } 1304 break; 1305 } 1306 1307 case Declaration::GROUP: 1308 parent.childCount++; 1309 memberInfo = &arena.allocate<MemberInfo>( 1310 parent, codeOrder++, member, 1311 newGroupNode(parent.node, member.getName().getValue()), 1312 false); 1313 allMembers.add(memberInfo); 1314 1315 // Members of the group are laid out just like they were members of the parent, so we 1316 // just pass along the parent layout. 1317 traverseGroup(member.getNestedDecls(), *memberInfo, layout); 1318 1319 // No ordinal for groups. 1320 break; 1321 1322 default: 1323 // Ignore others. 1324 break; 1325 } 1326 1327 KJ_IF_MAYBE(o, ordinal) { 1328 membersByOrdinal.insert(std::make_pair(*o, memberInfo)); 1329 } 1330 } 1331 } 1332 1333 void traverseParams(List<Declaration::Param>::Reader params, MemberInfo& parent, 1334 StructLayout::StructOrGroup& layout) { 1335 for (uint i: kj::indices(params)) { 1336 auto param = params[i]; 1337 parent.childCount++; 1338 MemberInfo* memberInfo = &arena.allocate<MemberInfo>(parent, i, param, layout, false); 1339 allMembers.add(memberInfo); 1340 membersByOrdinal.insert(std::make_pair(i, memberInfo)); 1341 } 1342 } 1343 1344 NodeSourceInfoBuilderPair newGroupNode(schema::Node::Reader parent, kj::StringPtr name) { 1345 AuxNode aux { 1346 translator.orphanage.newOrphan<schema::Node>(), 1347 translator.orphanage.newOrphan<schema::Node::SourceInfo>() 1348 }; 1349 auto node = aux.node.get(); 1350 auto sourceInfo = aux.sourceInfo.get(); 1351 1352 // We'll set the ID and scope ID later. 1353 node.setDisplayName(kj::str(parent.getDisplayName(), '.', name)); 1354 node.setDisplayNamePrefixLength(node.getDisplayName().size() - name.size()); 1355 node.setIsGeneric(parent.getIsGeneric()); 1356 node.initStruct().setIsGroup(true); 1357 1358 // The remaining contents of node.struct will be filled in later. 1359 1360 translator.groups.add(kj::mv(aux)); 1361 return { node, sourceInfo }; 1362 } 1363 1364 void translateInternal(MemberInfo& root, schema::Node::Builder builder) { 1365 auto structBuilder = builder.initStruct(); 1366 1367 // Go through each member in ordinal order, building each member schema. 1368 DuplicateOrdinalDetector dupDetector(errorReporter); 1369 for (auto& entry: membersByOrdinal) { 1370 MemberInfo& member = *entry.second; 1371 1372 // Make sure the exceptions added relating to 1373 // https://github.com/sandstorm-io/capnproto/issues/344 identify the affected field. 1374 KJ_CONTEXT(member.name); 1375 1376 if (member.declId.isOrdinal()) { 1377 dupDetector.check(member.declId.getOrdinal()); 1378 } 1379 1380 schema::Field::Builder fieldBuilder = member.getSchema(); 1381 fieldBuilder.getOrdinal().setExplicit(entry.first); 1382 1383 switch (member.declKind) { 1384 case Declaration::FIELD: { 1385 auto slot = fieldBuilder.initSlot(); 1386 auto typeBuilder = slot.initType(); 1387 if (translator.compileType(member.fieldType, typeBuilder, implicitMethodParams)) { 1388 if (member.hasDefaultValue) { 1389 if (member.isParam && 1390 member.fieldDefaultValue.isRelativeName() && 1391 member.fieldDefaultValue.getRelativeName().getValue() == "null") { 1392 // special case: parameter set null 1393 switch (typeBuilder.which()) { 1394 case schema::Type::TEXT: 1395 case schema::Type::DATA: 1396 case schema::Type::LIST: 1397 case schema::Type::STRUCT: 1398 case schema::Type::INTERFACE: 1399 case schema::Type::ANY_POINTER: 1400 break; 1401 default: 1402 errorReporter.addErrorOn(member.fieldDefaultValue.getRelativeName(), 1403 "Only pointer parameters can declare their default as 'null'."); 1404 break; 1405 } 1406 translator.compileDefaultDefaultValue(typeBuilder, slot.initDefaultValue()); 1407 } else { 1408 translator.compileBootstrapValue(member.fieldDefaultValue, 1409 typeBuilder, slot.initDefaultValue()); 1410 } 1411 slot.setHadExplicitDefault(true); 1412 } else { 1413 translator.compileDefaultDefaultValue(typeBuilder, slot.initDefaultValue()); 1414 } 1415 } else { 1416 translator.compileDefaultDefaultValue(typeBuilder, slot.initDefaultValue()); 1417 } 1418 1419 int lgSize = -1; 1420 switch (typeBuilder.which()) { 1421 case schema::Type::VOID: lgSize = -1; break; 1422 case schema::Type::BOOL: lgSize = 0; break; 1423 case schema::Type::INT8: lgSize = 3; break; 1424 case schema::Type::INT16: lgSize = 4; break; 1425 case schema::Type::INT32: lgSize = 5; break; 1426 case schema::Type::INT64: lgSize = 6; break; 1427 case schema::Type::UINT8: lgSize = 3; break; 1428 case schema::Type::UINT16: lgSize = 4; break; 1429 case schema::Type::UINT32: lgSize = 5; break; 1430 case schema::Type::UINT64: lgSize = 6; break; 1431 case schema::Type::FLOAT32: lgSize = 5; break; 1432 case schema::Type::FLOAT64: lgSize = 6; break; 1433 1434 case schema::Type::TEXT: lgSize = -2; break; 1435 case schema::Type::DATA: lgSize = -2; break; 1436 case schema::Type::LIST: lgSize = -2; break; 1437 case schema::Type::ENUM: lgSize = 4; break; 1438 case schema::Type::STRUCT: lgSize = -2; break; 1439 case schema::Type::INTERFACE: lgSize = -2; break; 1440 case schema::Type::ANY_POINTER: lgSize = -2; break; 1441 } 1442 1443 if (lgSize == -2) { 1444 // pointer 1445 slot.setOffset(member.fieldScope->addPointer()); 1446 } else if (lgSize == -1) { 1447 // void 1448 member.fieldScope->addVoid(); 1449 slot.setOffset(0); 1450 } else { 1451 slot.setOffset(member.fieldScope->addData(lgSize)); 1452 } 1453 break; 1454 } 1455 1456 case Declaration::UNION: 1457 if (!member.unionScope->addDiscriminant()) { 1458 errorReporter.addErrorOn(member.declId.getOrdinal(), 1459 "Union ordinal, if specified, must be greater than no more than one of its " 1460 "member ordinals (i.e. there can only be one field retroactively unionized)."); 1461 } 1462 break; 1463 1464 case Declaration::GROUP: 1465 KJ_FAIL_ASSERT("Groups don't have ordinals."); 1466 break; 1467 1468 default: 1469 KJ_FAIL_ASSERT("Unexpected member type."); 1470 break; 1471 } 1472 } 1473 1474 // OK, we should have built all the members. Now go through and make sure the discriminant 1475 // offsets have been copied over to the schemas and annotations have been applied. 1476 root.finishGroup(); 1477 for (auto member: allMembers) { 1478 kj::StringPtr targetsFlagName; 1479 if (member->isParam) { 1480 targetsFlagName = "targetsParam"; 1481 } else { 1482 switch (member->declKind) { 1483 case Declaration::FIELD: 1484 targetsFlagName = "targetsField"; 1485 break; 1486 1487 case Declaration::UNION: 1488 member->finishGroup(); 1489 targetsFlagName = "targetsUnion"; 1490 break; 1491 1492 case Declaration::GROUP: 1493 member->finishGroup(); 1494 targetsFlagName = "targetsGroup"; 1495 break; 1496 1497 default: 1498 KJ_FAIL_ASSERT("Unexpected member type."); 1499 break; 1500 } 1501 } 1502 1503 member->getSchema().adoptAnnotations(translator.compileAnnotationApplications( 1504 member->declAnnotations, targetsFlagName)); 1505 } 1506 1507 // And fill in the sizes. 1508 structBuilder.setDataWordCount(layout.getTop().dataWordCount); 1509 structBuilder.setPointerCount(layout.getTop().pointerCount); 1510 structBuilder.setPreferredListEncoding(schema::ElementSize::INLINE_COMPOSITE); 1511 1512 for (auto& group: translator.groups) { 1513 auto groupBuilder = group.node.get().getStruct(); 1514 groupBuilder.setDataWordCount(structBuilder.getDataWordCount()); 1515 groupBuilder.setPointerCount(structBuilder.getPointerCount()); 1516 groupBuilder.setPreferredListEncoding(structBuilder.getPreferredListEncoding()); 1517 } 1518 } 1519 }; 1520 1521 void NodeTranslator::compileStruct(Void decl, List<Declaration>::Reader members, 1522 schema::Node::Builder builder) { 1523 StructTranslator(*this, ImplicitParams::none()) 1524 .translate(decl, members, builder, sourceInfo.get()); 1525 } 1526 1527 // ------------------------------------------------------------------- 1528 1529 void NodeTranslator::compileInterface(Declaration::Interface::Reader decl, 1530 List<Declaration>::Reader members, 1531 schema::Node::Builder builder) { 1532 auto interfaceBuilder = builder.initInterface(); 1533 1534 auto superclassesDecl = decl.getSuperclasses(); 1535 auto superclassesBuilder = interfaceBuilder.initSuperclasses(superclassesDecl.size()); 1536 for (uint i: kj::indices(superclassesDecl)) { 1537 auto superclass = superclassesDecl[i]; 1538 1539 KJ_IF_MAYBE(decl, compileDeclExpression(superclass, ImplicitParams::none())) { 1540 KJ_IF_MAYBE(kind, decl->getKind()) { 1541 if (*kind == Declaration::INTERFACE) { 1542 auto s = superclassesBuilder[i]; 1543 s.setId(decl->getIdAndFillBrand([&]() { return s.initBrand(); })); 1544 } else { 1545 decl->addError(errorReporter, kj::str( 1546 "'", decl->toString(), "' is not an interface.")); 1547 } 1548 } else { 1549 // A variable? 1550 decl->addError(errorReporter, kj::str( 1551 "'", decl->toString(), "' is an unbound generic parameter. Currently we don't support " 1552 "extending these.")); 1553 } 1554 } 1555 } 1556 1557 // maps ordinal -> (code order, declaration) 1558 std::multimap<uint, std::pair<uint, Declaration::Reader>> methods; 1559 1560 uint codeOrder = 0; 1561 for (auto member: members) { 1562 if (member.isMethod()) { 1563 methods.insert( 1564 std::make_pair(member.getId().getOrdinal().getValue(), 1565 std::make_pair(codeOrder++, member))); 1566 } 1567 } 1568 1569 auto list = interfaceBuilder.initMethods(methods.size()); 1570 auto sourceInfoList = sourceInfo.get().initMembers(methods.size()); 1571 uint i = 0; 1572 DuplicateOrdinalDetector dupDetector(errorReporter); 1573 1574 for (auto& entry: methods) { 1575 uint codeOrder = entry.second.first; 1576 Declaration::Reader methodDecl = entry.second.second; 1577 auto methodReader = methodDecl.getMethod(); 1578 1579 auto ordinalDecl = methodDecl.getId().getOrdinal(); 1580 dupDetector.check(ordinalDecl); 1581 uint16_t ordinal = ordinalDecl.getValue(); 1582 1583 if (methodDecl.hasDocComment()) { 1584 sourceInfoList[i].setDocComment(methodDecl.getDocComment()); 1585 } 1586 1587 auto methodBuilder = list[i++]; 1588 methodBuilder.setName(methodDecl.getName().getValue()); 1589 methodBuilder.setCodeOrder(codeOrder); 1590 1591 auto implicits = methodDecl.getParameters(); 1592 auto implicitsBuilder = methodBuilder.initImplicitParameters(implicits.size()); 1593 for (auto i: kj::indices(implicits)) { 1594 implicitsBuilder[i].setName(implicits[i].getName()); 1595 } 1596 1597 auto params = methodReader.getParams(); 1598 if (params.isStream()) { 1599 errorReporter.addErrorOn(params, "'stream' can only appear after '->', not before."); 1600 } 1601 methodBuilder.setParamStructType(compileParamList( 1602 methodDecl.getName().getValue(), ordinal, false, 1603 params, implicits, 1604 [&]() { return methodBuilder.initParamBrand(); })); 1605 1606 auto results = methodReader.getResults(); 1607 Declaration::ParamList::Reader resultList; 1608 if (results.isExplicit()) { 1609 resultList = results.getExplicit(); 1610 } else { 1611 // We just stick with `resultList` uninitialized, which is equivalent to the default 1612 // instance. This works because `namedList` is the default kind of ParamList, and it will 1613 // default to an empty list. 1614 } 1615 methodBuilder.setResultStructType(compileParamList( 1616 methodDecl.getName().getValue(), ordinal, true, 1617 resultList, implicits, 1618 [&]() { return methodBuilder.initResultBrand(); })); 1619 1620 methodBuilder.adoptAnnotations(compileAnnotationApplications( 1621 methodDecl.getAnnotations(), "targetsMethod")); 1622 } 1623 } 1624 1625 template <typename InitBrandFunc> 1626 uint64_t NodeTranslator::compileParamList( 1627 kj::StringPtr methodName, uint16_t ordinal, bool isResults, 1628 Declaration::ParamList::Reader paramList, 1629 typename List<Declaration::BrandParameter>::Reader implicitParams, 1630 InitBrandFunc&& initBrand) { 1631 switch (paramList.which()) { 1632 case Declaration::ParamList::NAMED_LIST: { 1633 auto newStruct = orphanage.newOrphan<schema::Node>(); 1634 auto newSourceInfo = orphanage.newOrphan<schema::Node::SourceInfo>(); 1635 auto builder = newStruct.get(); 1636 auto parent = wipNode.getReader(); 1637 1638 kj::String typeName = kj::str(methodName, isResults ? "$Results" : "$Params"); 1639 1640 builder.setId(generateMethodParamsId(parent.getId(), ordinal, isResults)); 1641 builder.setDisplayName(kj::str(parent.getDisplayName(), '.', typeName)); 1642 builder.setDisplayNamePrefixLength(builder.getDisplayName().size() - typeName.size()); 1643 builder.setIsGeneric(parent.getIsGeneric() || implicitParams.size() > 0); 1644 builder.setScopeId(0); // detached struct type 1645 1646 builder.initStruct(); 1647 1648 // Note that the struct we create here has a brand parameter list mirrioring the method's 1649 // implicit parameter list. Of course, fields inside the struct using the method's implicit 1650 // params as types actually need to refer to them as regular params, so we create an 1651 // ImplicitParams with a scopeId here. 1652 StructTranslator(*this, ImplicitParams { builder.getId(), implicitParams }) 1653 .translate(paramList.getNamedList(), builder, newSourceInfo.get()); 1654 uint64_t id = builder.getId(); 1655 paramStructs.add(AuxNode { kj::mv(newStruct), kj::mv(newSourceInfo) }); 1656 1657 auto brand = localBrand->push(builder.getId(), implicitParams.size()); 1658 1659 if (implicitParams.size() > 0) { 1660 auto implicitDecls = kj::heapArrayBuilder<BrandedDecl>(implicitParams.size()); 1661 auto implicitBuilder = builder.initParameters(implicitParams.size()); 1662 1663 for (auto i: kj::indices(implicitParams)) { 1664 auto param = implicitParams[i]; 1665 implicitDecls.add(BrandedDecl::implicitMethodParam(i)); 1666 implicitBuilder[i].setName(param.getName()); 1667 } 1668 1669 brand->setParams(implicitDecls.finish(), Declaration::STRUCT, Expression::Reader()); 1670 } 1671 1672 brand->compile(initBrand); 1673 return id; 1674 } 1675 case Declaration::ParamList::TYPE: 1676 KJ_IF_MAYBE(target, compileDeclExpression( 1677 paramList.getType(), ImplicitParams { 0, implicitParams })) { 1678 KJ_IF_MAYBE(kind, target->getKind()) { 1679 if (*kind == Declaration::STRUCT) { 1680 return target->getIdAndFillBrand(kj::fwd<InitBrandFunc>(initBrand)); 1681 } else { 1682 errorReporter.addErrorOn( 1683 paramList.getType(), 1684 kj::str("'", expressionString(paramList.getType()), "' is not a struct type.")); 1685 } 1686 } else { 1687 // A variable? 1688 target->addError(errorReporter, 1689 "Cannot use generic parameter as whole input or output of a method. Instead, " 1690 "use a parameter/result list containing a field with this type."); 1691 return 0; 1692 } 1693 } 1694 return 0; 1695 case Declaration::ParamList::STREAM: 1696 KJ_IF_MAYBE(streamCapnp, resolver.resolveImport("/capnp/stream.capnp")) { 1697 if (streamCapnp->resolver->resolveMember("StreamResult") == nullptr) { 1698 errorReporter.addErrorOn(paramList, 1699 "The version of '/capnp/stream.capnp' found in your import path does not appear " 1700 "to be the official one; it is missing the declaration of StreamResult."); 1701 } 1702 } else { 1703 errorReporter.addErrorOn(paramList, 1704 "A method declaration uses streaming, but '/capnp/stream.capnp' is not found " 1705 "in the import path. This is a standard file that should always be installed " 1706 "with the Cap'n Proto compiler."); 1707 } 1708 return typeId<StreamResult>(); 1709 } 1710 KJ_UNREACHABLE; 1711 } 1712 1713 // ------------------------------------------------------------------- 1714 1715 kj::Maybe<BrandedDecl> 1716 NodeTranslator::compileDeclExpression( 1717 Expression::Reader source, ImplicitParams implicitMethodParams) { 1718 return localBrand->compileDeclExpression(source, resolver, implicitMethodParams); 1719 } 1720 1721 /* static */ kj::Maybe<Resolver::ResolveResult> NodeTranslator::compileDecl( 1722 uint64_t scopeId, uint scopeParameterCount, Resolver& resolver, ErrorReporter& errorReporter, 1723 Expression::Reader expression, schema::Brand::Builder brandBuilder) { 1724 auto scope = kj::refcounted<BrandScope>(errorReporter, scopeId, scopeParameterCount, resolver); 1725 KJ_IF_MAYBE(decl, scope->compileDeclExpression(expression, resolver, ImplicitParams::none())) { 1726 return decl->asResolveResult(scope->getScopeId(), brandBuilder); 1727 } else { 1728 return nullptr; 1729 } 1730 } 1731 1732 bool NodeTranslator::compileType(Expression::Reader source, schema::Type::Builder target, 1733 ImplicitParams implicitMethodParams) { 1734 KJ_IF_MAYBE(decl, compileDeclExpression(source, implicitMethodParams)) { 1735 return decl->compileAsType(errorReporter, target); 1736 } else { 1737 return false; 1738 } 1739 } 1740 1741 // ------------------------------------------------------------------- 1742 1743 void NodeTranslator::compileDefaultDefaultValue( 1744 schema::Type::Reader type, schema::Value::Builder target) { 1745 switch (type.which()) { 1746 case schema::Type::VOID: target.setVoid(); break; 1747 case schema::Type::BOOL: target.setBool(false); break; 1748 case schema::Type::INT8: target.setInt8(0); break; 1749 case schema::Type::INT16: target.setInt16(0); break; 1750 case schema::Type::INT32: target.setInt32(0); break; 1751 case schema::Type::INT64: target.setInt64(0); break; 1752 case schema::Type::UINT8: target.setUint8(0); break; 1753 case schema::Type::UINT16: target.setUint16(0); break; 1754 case schema::Type::UINT32: target.setUint32(0); break; 1755 case schema::Type::UINT64: target.setUint64(0); break; 1756 case schema::Type::FLOAT32: target.setFloat32(0); break; 1757 case schema::Type::FLOAT64: target.setFloat64(0); break; 1758 case schema::Type::ENUM: target.setEnum(0); break; 1759 case schema::Type::INTERFACE: target.setInterface(); break; 1760 1761 // Bit of a hack: For Text/Data, we adopt a null orphan, which sets the field to null. 1762 // TODO(cleanup): Create a cleaner way to do this. 1763 case schema::Type::TEXT: target.adoptText(Orphan<Text>()); break; 1764 case schema::Type::DATA: target.adoptData(Orphan<Data>()); break; 1765 case schema::Type::STRUCT: target.initStruct(); break; 1766 case schema::Type::LIST: target.initList(); break; 1767 case schema::Type::ANY_POINTER: target.initAnyPointer(); break; 1768 } 1769 } 1770 1771 void NodeTranslator::compileBootstrapValue( 1772 Expression::Reader source, schema::Type::Reader type, schema::Value::Builder target, 1773 kj::Maybe<Schema> typeScope) { 1774 // Start by filling in a default default value so that if for whatever reason we don't end up 1775 // initializing the value, this won't cause schema validation to fail. 1776 compileDefaultDefaultValue(type, target); 1777 1778 switch (type.which()) { 1779 case schema::Type::LIST: 1780 case schema::Type::STRUCT: 1781 case schema::Type::INTERFACE: 1782 case schema::Type::ANY_POINTER: 1783 unfinishedValues.add(UnfinishedValue { source, type, typeScope, target }); 1784 break; 1785 1786 default: 1787 // Primitive value. (Note that the scope can't possibly matter since primitives are not 1788 // generic.) 1789 compileValue(source, type, typeScope.orDefault(Schema()), target, true); 1790 break; 1791 } 1792 } 1793 1794 void NodeTranslator::compileValue(Expression::Reader source, schema::Type::Reader type, 1795 Schema typeScope, schema::Value::Builder target, 1796 bool isBootstrap) { 1797 class ResolverGlue: public ValueTranslator::Resolver { 1798 public: 1799 inline ResolverGlue(NodeTranslator& translator, bool isBootstrap) 1800 : translator(translator), isBootstrap(isBootstrap) {} 1801 1802 kj::Maybe<DynamicValue::Reader> resolveConstant(Expression::Reader name) override { 1803 return translator.readConstant(name, isBootstrap); 1804 } 1805 1806 kj::Maybe<kj::Array<const byte>> readEmbed(LocatedText::Reader filename) override { 1807 return translator.readEmbed(filename); 1808 } 1809 1810 private: 1811 NodeTranslator& translator; 1812 bool isBootstrap; 1813 }; 1814 1815 ResolverGlue glue(*this, isBootstrap); 1816 ValueTranslator valueTranslator(glue, errorReporter, orphanage); 1817 1818 KJ_IF_MAYBE(typeSchema, resolver.resolveBootstrapType(type, typeScope)) { 1819 kj::StringPtr fieldName = Schema::from<schema::Type>() 1820 .getUnionFields()[static_cast<uint>(typeSchema->which())].getProto().getName(); 1821 1822 KJ_IF_MAYBE(value, valueTranslator.compileValue(source, *typeSchema)) { 1823 if (typeSchema->isEnum()) { 1824 target.setEnum(value->getReader().as<DynamicEnum>().getRaw()); 1825 } else { 1826 toDynamic(target).adopt(fieldName, kj::mv(*value)); 1827 } 1828 } 1829 } 1830 } 1831 1832 kj::Maybe<Orphan<DynamicValue>> ValueTranslator::compileValue(Expression::Reader src, Type type) { 1833 if (type.isAnyPointer()) { 1834 if (type.getBrandParameter() != nullptr || type.getImplicitParameter() != nullptr) { 1835 errorReporter.addErrorOn(src, 1836 "Cannot interpret value because the type is a generic type parameter which is not " 1837 "yet bound. We don't know what type to expect here."); 1838 return nullptr; 1839 } 1840 } 1841 1842 Orphan<DynamicValue> result = compileValueInner(src, type); 1843 1844 // compileValueInner() evaluated `src` and only used `type` as a hint in interpreting `src` if 1845 // `src`'s type wasn't already obvious. So, now we need to check that the resulting value 1846 // actually matches `type`. 1847 1848 switch (result.getType()) { 1849 case DynamicValue::UNKNOWN: 1850 // Error already reported. 1851 return nullptr; 1852 1853 case DynamicValue::VOID: 1854 if (type.isVoid()) { 1855 return kj::mv(result); 1856 } 1857 break; 1858 1859 case DynamicValue::BOOL: 1860 if (type.isBool()) { 1861 return kj::mv(result); 1862 } 1863 break; 1864 1865 case DynamicValue::INT: { 1866 int64_t value = result.getReader().as<int64_t>(); 1867 if (value < 0) { 1868 int64_t minValue = 1; 1869 switch (type.which()) { 1870 case schema::Type::INT8: minValue = (int8_t)kj::minValue; break; 1871 case schema::Type::INT16: minValue = (int16_t)kj::minValue; break; 1872 case schema::Type::INT32: minValue = (int32_t)kj::minValue; break; 1873 case schema::Type::INT64: minValue = (int64_t)kj::minValue; break; 1874 case schema::Type::UINT8: minValue = (uint8_t)kj::minValue; break; 1875 case schema::Type::UINT16: minValue = (uint16_t)kj::minValue; break; 1876 case schema::Type::UINT32: minValue = (uint32_t)kj::minValue; break; 1877 case schema::Type::UINT64: minValue = (uint64_t)kj::minValue; break; 1878 1879 case schema::Type::FLOAT32: 1880 case schema::Type::FLOAT64: 1881 // Any integer is acceptable. 1882 minValue = (int64_t)kj::minValue; 1883 break; 1884 1885 default: break; 1886 } 1887 if (minValue == 1) break; 1888 1889 if (value < minValue) { 1890 errorReporter.addErrorOn(src, "Integer value out of range."); 1891 result = minValue; 1892 } 1893 return kj::mv(result); 1894 } 1895 1896 } KJ_FALLTHROUGH; // value is positive, so we can just go on to the uint case below. 1897 1898 case DynamicValue::UINT: { 1899 uint64_t maxValue = 0; 1900 switch (type.which()) { 1901 case schema::Type::INT8: maxValue = (int8_t)kj::maxValue; break; 1902 case schema::Type::INT16: maxValue = (int16_t)kj::maxValue; break; 1903 case schema::Type::INT32: maxValue = (int32_t)kj::maxValue; break; 1904 case schema::Type::INT64: maxValue = (int64_t)kj::maxValue; break; 1905 case schema::Type::UINT8: maxValue = (uint8_t)kj::maxValue; break; 1906 case schema::Type::UINT16: maxValue = (uint16_t)kj::maxValue; break; 1907 case schema::Type::UINT32: maxValue = (uint32_t)kj::maxValue; break; 1908 case schema::Type::UINT64: maxValue = (uint64_t)kj::maxValue; break; 1909 1910 case schema::Type::FLOAT32: 1911 case schema::Type::FLOAT64: 1912 // Any integer is acceptable. 1913 maxValue = (uint64_t)kj::maxValue; 1914 break; 1915 1916 default: break; 1917 } 1918 if (maxValue == 0) break; 1919 1920 if (result.getReader().as<uint64_t>() > maxValue) { 1921 errorReporter.addErrorOn(src, "Integer value out of range."); 1922 result = maxValue; 1923 } 1924 return kj::mv(result); 1925 } 1926 1927 case DynamicValue::FLOAT: 1928 if (type.isFloat32() || type.isFloat64()) { 1929 return kj::mv(result); 1930 } 1931 break; 1932 1933 case DynamicValue::TEXT: 1934 if (type.isText()) { 1935 return kj::mv(result); 1936 } 1937 break; 1938 1939 case DynamicValue::DATA: 1940 if (type.isData()) { 1941 return kj::mv(result); 1942 } 1943 break; 1944 1945 case DynamicValue::LIST: 1946 if (type.isList()) { 1947 if (result.getReader().as<DynamicList>().getSchema() == type.asList()) { 1948 return kj::mv(result); 1949 } 1950 } else if (type.isAnyPointer()) { 1951 switch (type.whichAnyPointerKind()) { 1952 case schema::Type::AnyPointer::Unconstrained::ANY_KIND: 1953 case schema::Type::AnyPointer::Unconstrained::LIST: 1954 return kj::mv(result); 1955 case schema::Type::AnyPointer::Unconstrained::STRUCT: 1956 case schema::Type::AnyPointer::Unconstrained::CAPABILITY: 1957 break; 1958 } 1959 } 1960 break; 1961 1962 case DynamicValue::ENUM: 1963 if (type.isEnum()) { 1964 if (result.getReader().as<DynamicEnum>().getSchema() == type.asEnum()) { 1965 return kj::mv(result); 1966 } 1967 } 1968 break; 1969 1970 case DynamicValue::STRUCT: 1971 if (type.isStruct()) { 1972 if (result.getReader().as<DynamicStruct>().getSchema() == type.asStruct()) { 1973 return kj::mv(result); 1974 } 1975 } else if (type.isAnyPointer()) { 1976 switch (type.whichAnyPointerKind()) { 1977 case schema::Type::AnyPointer::Unconstrained::ANY_KIND: 1978 case schema::Type::AnyPointer::Unconstrained::STRUCT: 1979 return kj::mv(result); 1980 case schema::Type::AnyPointer::Unconstrained::LIST: 1981 case schema::Type::AnyPointer::Unconstrained::CAPABILITY: 1982 break; 1983 } 1984 } 1985 break; 1986 1987 case DynamicValue::CAPABILITY: 1988 KJ_FAIL_ASSERT("Interfaces can't have literal values."); 1989 1990 case DynamicValue::ANY_POINTER: 1991 KJ_FAIL_ASSERT("AnyPointers can't have literal values."); 1992 } 1993 1994 errorReporter.addErrorOn(src, kj::str("Type mismatch; expected ", makeTypeName(type), ".")); 1995 return nullptr; 1996 } 1997 1998 Orphan<DynamicValue> ValueTranslator::compileValueInner(Expression::Reader src, Type type) { 1999 switch (src.which()) { 2000 case Expression::RELATIVE_NAME: { 2001 auto name = src.getRelativeName(); 2002 2003 // The name is just a bare identifier. It may be a literal value or an enumerant. 2004 kj::StringPtr id = name.getValue(); 2005 2006 if (type.isEnum()) { 2007 KJ_IF_MAYBE(enumerant, type.asEnum().findEnumerantByName(id)) { 2008 return DynamicEnum(*enumerant); 2009 } 2010 } else { 2011 // Interpret known constant values. 2012 if (id == "void") { 2013 return VOID; 2014 } else if (id == "true") { 2015 return true; 2016 } else if (id == "false") { 2017 return false; 2018 } else if (id == "nan") { 2019 return kj::nan(); 2020 } else if (id == "inf") { 2021 return kj::inf(); 2022 } 2023 } 2024 2025 // Apparently not a literal. Try resolving it. 2026 KJ_IF_MAYBE(constValue, resolver.resolveConstant(src)) { 2027 return orphanage.newOrphanCopy(*constValue); 2028 } else { 2029 return nullptr; 2030 } 2031 } 2032 2033 case Expression::ABSOLUTE_NAME: 2034 case Expression::IMPORT: 2035 case Expression::APPLICATION: 2036 case Expression::MEMBER: 2037 KJ_IF_MAYBE(constValue, resolver.resolveConstant(src)) { 2038 return orphanage.newOrphanCopy(*constValue); 2039 } else { 2040 return nullptr; 2041 } 2042 2043 case Expression::EMBED: 2044 KJ_IF_MAYBE(data, resolver.readEmbed(src.getEmbed())) { 2045 switch (type.which()) { 2046 case schema::Type::TEXT: { 2047 // Sadly, we need to make a copy to add the NUL terminator. 2048 auto text = orphanage.newOrphan<Text>(data->size()); 2049 memcpy(text.get().begin(), data->begin(), data->size()); 2050 return kj::mv(text); 2051 } 2052 case schema::Type::DATA: 2053 // TODO(perf): It would arguably be neat to use orphanage.referenceExternalData(), 2054 // since typically the data is mmap()ed and this would avoid forcing a large file 2055 // to become memory-resident. However, we'd have to figure out who should own the 2056 // Array<byte>. Also, we'd have to deal with the possibility of misaligned data -- 2057 // though arguably in that case we know it's not mmap()ed so whatever. One more 2058 // thing: it would be neat to be able to reference text blobs this way too, if only 2059 // we could rely on the assumption that as long as the data doesn't end on a page 2060 // boundary, it will be zero-padded, thus giving us our NUL terminator (4095/4096 of 2061 // the time), but this seems to require documenting constraints on the underlying 2062 // file-reading interfaces. Hm. 2063 return orphanage.newOrphanCopy(Data::Reader(*data)); 2064 case schema::Type::STRUCT: { 2065 // We will almost certainly 2066 if (data->size() % sizeof(word) != 0) { 2067 errorReporter.addErrorOn(src, 2068 "Embedded file is not a valid Cap'n Proto message."); 2069 return nullptr; 2070 } 2071 kj::Array<word> copy; 2072 kj::ArrayPtr<const word> words; 2073 if (reinterpret_cast<uintptr_t>(data->begin()) % sizeof(void*) == 0) { 2074 // Hooray, data is aligned. 2075 words = kj::ArrayPtr<const word>( 2076 reinterpret_cast<const word*>(data->begin()), 2077 data->size() / sizeof(word)); 2078 } else { 2079 // Ugh, data not aligned. Make a copy. 2080 copy = kj::heapArray<word>(data->size() / sizeof(word)); 2081 memcpy(copy.begin(), data->begin(), data->size()); 2082 words = copy; 2083 } 2084 ReaderOptions options; 2085 options.traversalLimitInWords = kj::maxValue; 2086 options.nestingLimit = kj::maxValue; 2087 FlatArrayMessageReader reader(words, options); 2088 return orphanage.newOrphanCopy(reader.getRoot<DynamicStruct>(type.asStruct())); 2089 } 2090 default: 2091 errorReporter.addErrorOn(src, 2092 "Embeds can only be used when Text, Data, or a struct is expected."); 2093 return nullptr; 2094 } 2095 } else { 2096 return nullptr; 2097 } 2098 2099 case Expression::POSITIVE_INT: 2100 return src.getPositiveInt(); 2101 2102 case Expression::NEGATIVE_INT: { 2103 uint64_t nValue = src.getNegativeInt(); 2104 if (nValue > ((uint64_t)kj::maxValue >> 1) + 1) { 2105 errorReporter.addErrorOn(src, "Integer is too big to be negative."); 2106 return nullptr; 2107 } else { 2108 return kj::implicitCast<int64_t>(-nValue); 2109 } 2110 } 2111 2112 case Expression::FLOAT: 2113 return src.getFloat(); 2114 break; 2115 2116 case Expression::STRING: 2117 if (type.isData()) { 2118 Text::Reader text = src.getString(); 2119 return orphanage.newOrphanCopy(Data::Reader(text.asBytes())); 2120 } else { 2121 return orphanage.newOrphanCopy(src.getString()); 2122 } 2123 break; 2124 2125 case Expression::BINARY: 2126 if (!type.isData()) { 2127 errorReporter.addErrorOn(src, kj::str("Type mismatch; expected ", makeTypeName(type), ".")); 2128 return nullptr; 2129 } 2130 return orphanage.newOrphanCopy(src.getBinary()); 2131 2132 case Expression::LIST: { 2133 if (!type.isList()) { 2134 errorReporter.addErrorOn(src, kj::str("Type mismatch; expected ", makeTypeName(type), ".")); 2135 return nullptr; 2136 } 2137 auto listSchema = type.asList(); 2138 Type elementType = listSchema.getElementType(); 2139 auto srcList = src.getList(); 2140 Orphan<DynamicList> result = orphanage.newOrphan(listSchema, srcList.size()); 2141 auto dstList = result.get(); 2142 for (uint i = 0; i < srcList.size(); i++) { 2143 KJ_IF_MAYBE(value, compileValue(srcList[i], elementType)) { 2144 dstList.adopt(i, kj::mv(*value)); 2145 } 2146 } 2147 return kj::mv(result); 2148 } 2149 2150 case Expression::TUPLE: { 2151 if (!type.isStruct()) { 2152 errorReporter.addErrorOn(src, kj::str("Type mismatch; expected ", makeTypeName(type), ".")); 2153 return nullptr; 2154 } 2155 auto structSchema = type.asStruct(); 2156 Orphan<DynamicStruct> result = orphanage.newOrphan(structSchema); 2157 fillStructValue(result.get(), src.getTuple()); 2158 return kj::mv(result); 2159 } 2160 2161 case Expression::UNKNOWN: 2162 // Ignore earlier error. 2163 return nullptr; 2164 } 2165 2166 KJ_UNREACHABLE; 2167 } 2168 2169 void ValueTranslator::fillStructValue(DynamicStruct::Builder builder, 2170 List<Expression::Param>::Reader assignments) { 2171 for (auto assignment: assignments) { 2172 if (assignment.isNamed()) { 2173 auto fieldName = assignment.getNamed(); 2174 KJ_IF_MAYBE(field, builder.getSchema().findFieldByName(fieldName.getValue())) { 2175 auto fieldProto = field->getProto(); 2176 auto value = assignment.getValue(); 2177 2178 switch (fieldProto.which()) { 2179 case schema::Field::SLOT: 2180 KJ_IF_MAYBE(compiledValue, compileValue(value, field->getType())) { 2181 builder.adopt(*field, kj::mv(*compiledValue)); 2182 } 2183 break; 2184 2185 case schema::Field::GROUP: 2186 if (value.isTuple()) { 2187 fillStructValue(builder.init(*field).as<DynamicStruct>(), value.getTuple()); 2188 } else { 2189 errorReporter.addErrorOn(value, "Type mismatch; expected group."); 2190 } 2191 break; 2192 } 2193 } else { 2194 errorReporter.addErrorOn(fieldName, kj::str( 2195 "Struct has no field named '", fieldName.getValue(), "'.")); 2196 } 2197 } else { 2198 errorReporter.addErrorOn(assignment.getValue(), kj::str("Missing field name.")); 2199 } 2200 } 2201 } 2202 2203 kj::String ValueTranslator::makeNodeName(Schema schema) { 2204 schema::Node::Reader proto = schema.getProto(); 2205 return kj::str(proto.getDisplayName().slice(proto.getDisplayNamePrefixLength())); 2206 } 2207 2208 kj::String ValueTranslator::makeTypeName(Type type) { 2209 switch (type.which()) { 2210 case schema::Type::VOID: return kj::str("Void"); 2211 case schema::Type::BOOL: return kj::str("Bool"); 2212 case schema::Type::INT8: return kj::str("Int8"); 2213 case schema::Type::INT16: return kj::str("Int16"); 2214 case schema::Type::INT32: return kj::str("Int32"); 2215 case schema::Type::INT64: return kj::str("Int64"); 2216 case schema::Type::UINT8: return kj::str("UInt8"); 2217 case schema::Type::UINT16: return kj::str("UInt16"); 2218 case schema::Type::UINT32: return kj::str("UInt32"); 2219 case schema::Type::UINT64: return kj::str("UInt64"); 2220 case schema::Type::FLOAT32: return kj::str("Float32"); 2221 case schema::Type::FLOAT64: return kj::str("Float64"); 2222 case schema::Type::TEXT: return kj::str("Text"); 2223 case schema::Type::DATA: return kj::str("Data"); 2224 case schema::Type::LIST: 2225 return kj::str("List(", makeTypeName(type.asList().getElementType()), ")"); 2226 case schema::Type::ENUM: return makeNodeName(type.asEnum()); 2227 case schema::Type::STRUCT: return makeNodeName(type.asStruct()); 2228 case schema::Type::INTERFACE: return makeNodeName(type.asInterface()); 2229 case schema::Type::ANY_POINTER: return kj::str("AnyPointer"); 2230 } 2231 KJ_UNREACHABLE; 2232 } 2233 2234 kj::Maybe<DynamicValue::Reader> NodeTranslator::readConstant( 2235 Expression::Reader source, bool isBootstrap) { 2236 // Look up the constant decl. 2237 BrandedDecl constDecl = nullptr; 2238 KJ_IF_MAYBE(decl, compileDeclExpression(source, ImplicitParams::none())) { 2239 constDecl = *decl; 2240 } else { 2241 // Lookup will have reported an error. 2242 return nullptr; 2243 } 2244 2245 // Is it a constant? 2246 if(constDecl.getKind().orDefault(Declaration::FILE) != Declaration::CONST) { 2247 errorReporter.addErrorOn(source, 2248 kj::str("'", expressionString(source), "' does not refer to a constant.")); 2249 return nullptr; 2250 } 2251 2252 // Extract the ID and brand. 2253 MallocMessageBuilder builder(256); 2254 auto constBrand = builder.getRoot<schema::Brand>(); 2255 uint64_t id = constDecl.getIdAndFillBrand([&]() { return constBrand; }); 2256 2257 // Look up the schema -- we'll need this to compile the constant's type. 2258 Schema constSchema; 2259 KJ_IF_MAYBE(s, resolver.resolveBootstrapSchema(id, constBrand)) { 2260 constSchema = *s; 2261 } else { 2262 // The constant's schema is broken for reasons already reported. 2263 return nullptr; 2264 } 2265 2266 // If we're bootstrapping, then we know we're expecting a primitive value, so if the 2267 // constant turns out to be non-primitive, we'll error out anyway. If we're not 2268 // bootstrapping, we may be compiling a non-primitive value and so we need the final 2269 // version of the constant to make sure its value is filled in. 2270 schema::Node::Reader proto = constSchema.getProto(); 2271 if (!isBootstrap) { 2272 KJ_IF_MAYBE(finalProto, resolver.resolveFinalSchema(id)) { 2273 proto = *finalProto; 2274 } else { 2275 // The constant's final schema is broken for reasons already reported. 2276 return nullptr; 2277 } 2278 } 2279 2280 auto constReader = proto.getConst(); 2281 auto dynamicConst = toDynamic(constReader.getValue()); 2282 auto constValue = dynamicConst.get(KJ_ASSERT_NONNULL(dynamicConst.which())); 2283 2284 if (constValue.getType() == DynamicValue::ANY_POINTER) { 2285 // We need to assign an appropriate schema to this pointer. 2286 AnyPointer::Reader objValue = constValue.as<AnyPointer>(); 2287 2288 auto constType = constSchema.asConst().getType(); 2289 switch (constType.which()) { 2290 case schema::Type::STRUCT: 2291 constValue = objValue.getAs<DynamicStruct>(constType.asStruct()); 2292 break; 2293 case schema::Type::LIST: 2294 constValue = objValue.getAs<DynamicList>(constType.asList()); 2295 break; 2296 case schema::Type::ANY_POINTER: 2297 // Fine as-is. 2298 break; 2299 default: 2300 KJ_FAIL_ASSERT("Unrecognized AnyPointer-typed member of schema::Value."); 2301 break; 2302 } 2303 } 2304 2305 if (source.isRelativeName()) { 2306 // A fully unqualified identifier looks like it might refer to a constant visible in the 2307 // current scope, but if that's really what the user wanted, we want them to use a 2308 // qualified name to make it more obvious. Report an error. 2309 KJ_IF_MAYBE(scope, resolver.resolveBootstrapSchema(proto.getScopeId(), 2310 schema::Brand::Reader())) { 2311 auto scopeReader = scope->getProto(); 2312 kj::StringPtr parent; 2313 if (scopeReader.isFile()) { 2314 parent = ""; 2315 } else { 2316 parent = scopeReader.getDisplayName().slice(scopeReader.getDisplayNamePrefixLength()); 2317 } 2318 kj::StringPtr id = source.getRelativeName().getValue(); 2319 2320 errorReporter.addErrorOn(source, kj::str( 2321 "Constant names must be qualified to avoid confusion. Please replace '", 2322 expressionString(source), "' with '", parent, ".", id, 2323 "', if that's what you intended.")); 2324 } 2325 } 2326 2327 return constValue; 2328 } 2329 2330 kj::Maybe<kj::Array<const byte>> NodeTranslator::readEmbed(LocatedText::Reader filename) { 2331 KJ_IF_MAYBE(data, resolver.readEmbed(filename.getValue())) { 2332 return kj::mv(*data); 2333 } else { 2334 errorReporter.addErrorOn(filename, 2335 kj::str("Couldn't read file for embed: ", filename.getValue())); 2336 return nullptr; 2337 } 2338 } 2339 2340 Orphan<List<schema::Annotation>> NodeTranslator::compileAnnotationApplications( 2341 List<Declaration::AnnotationApplication>::Reader annotations, 2342 kj::StringPtr targetsFlagName) { 2343 if (annotations.size() == 0 || !compileAnnotations) { 2344 // Return null. 2345 return Orphan<List<schema::Annotation>>(); 2346 } 2347 2348 auto result = orphanage.newOrphan<List<schema::Annotation>>(annotations.size()); 2349 auto builder = result.get(); 2350 2351 for (uint i = 0; i < annotations.size(); i++) { 2352 Declaration::AnnotationApplication::Reader annotation = annotations[i]; 2353 schema::Annotation::Builder annotationBuilder = builder[i]; 2354 2355 // Set the annotation's value to void in case we fail to produce something better below. 2356 annotationBuilder.initValue().setVoid(); 2357 2358 auto name = annotation.getName(); 2359 KJ_IF_MAYBE(decl, compileDeclExpression(name, ImplicitParams::none())) { 2360 KJ_IF_MAYBE(kind, decl->getKind()) { 2361 if (*kind != Declaration::ANNOTATION) { 2362 errorReporter.addErrorOn(name, kj::str( 2363 "'", expressionString(name), "' is not an annotation.")); 2364 } else { 2365 annotationBuilder.setId(decl->getIdAndFillBrand( 2366 [&]() { return annotationBuilder.initBrand(); })); 2367 KJ_IF_MAYBE(annotationSchema, 2368 resolver.resolveBootstrapSchema(annotationBuilder.getId(), 2369 annotationBuilder.getBrand())) { 2370 auto node = annotationSchema->getProto().getAnnotation(); 2371 if (!toDynamic(node).get(targetsFlagName).as<bool>()) { 2372 errorReporter.addErrorOn(name, kj::str( 2373 "'", expressionString(name), "' cannot be applied to this kind of declaration.")); 2374 } 2375 2376 // Interpret the value. 2377 auto value = annotation.getValue(); 2378 switch (value.which()) { 2379 case Declaration::AnnotationApplication::Value::NONE: 2380 // No value, i.e. void. 2381 if (node.getType().isVoid()) { 2382 annotationBuilder.getValue().setVoid(); 2383 } else { 2384 errorReporter.addErrorOn(name, kj::str( 2385 "'", expressionString(name), "' requires a value.")); 2386 compileDefaultDefaultValue(node.getType(), annotationBuilder.getValue()); 2387 } 2388 break; 2389 2390 case Declaration::AnnotationApplication::Value::EXPRESSION: 2391 compileBootstrapValue(value.getExpression(), node.getType(), 2392 annotationBuilder.getValue(), 2393 *annotationSchema); 2394 break; 2395 } 2396 } 2397 } 2398 } else { 2399 errorReporter.addErrorOn(name, kj::str( 2400 "'", expressionString(name), "' is not an annotation.")); 2401 } 2402 } 2403 } 2404 2405 return result; 2406 } 2407 2408 } // namespace compiler 2409 } // namespace capnp