cl3.cpp (15383B)
1 #include "cl3.hpp" 2 3 #include "stcm/file.hpp" 4 #include "../open.hpp" 5 6 #include <libshit/container/ordered_map.lua.hpp> 7 #include <libshit/container/vector.lua.hpp> 8 #include <libshit/except.hpp> 9 #include <libshit/string_utils.hpp> 10 11 #include <boost/filesystem/operations.hpp> 12 13 #include <fstream> 14 15 namespace Neptools 16 { 17 18 void Cl3::Header::Validate(FilePosition file_size) const 19 { 20 #define VALIDATE(x) LIBSHIT_VALIDATE_FIELD("Cl3::Header", x) 21 VALIDATE(memcmp(magic, "CL3", 3) == 0); 22 VALIDATE(endian == 'L' || endian == 'B'); 23 VALIDATE(field_04 == 0); 24 VALIDATE(field_08 == 3); 25 VALIDATE(sections_offset + sections_count * sizeof(Section) <= file_size); 26 #undef VALIDATE 27 } 28 29 void endian_reverse_inplace(Cl3::Header& hdr) noexcept 30 { 31 boost::endian::endian_reverse_inplace(hdr.field_04); 32 boost::endian::endian_reverse_inplace(hdr.field_08); 33 boost::endian::endian_reverse_inplace(hdr.sections_count); 34 boost::endian::endian_reverse_inplace(hdr.sections_offset); 35 boost::endian::endian_reverse_inplace(hdr.field_14); 36 } 37 38 void Cl3::Section::Validate(FilePosition file_size) const 39 { 40 #define VALIDATE(x) LIBSHIT_VALIDATE_FIELD("Cl3::Section", x) 41 VALIDATE(name.is_valid()); 42 VALIDATE(data_offset <= file_size); 43 VALIDATE(data_offset + data_size <= file_size); 44 VALIDATE(field_2c == 0); 45 VALIDATE(field_30 == 0 && field_34 == 0 && field_38 == 0 && field_3c == 0); 46 VALIDATE(field_40 == 0 && field_44 == 0 && field_48 == 0 && field_4c == 0); 47 #undef VALIDATE 48 } 49 50 void endian_reverse_inplace(Cl3::Section& sec) noexcept 51 { 52 boost::endian::endian_reverse_inplace(sec.count); 53 boost::endian::endian_reverse_inplace(sec.data_size); 54 boost::endian::endian_reverse_inplace(sec.data_offset); 55 boost::endian::endian_reverse_inplace(sec.field_2c); 56 boost::endian::endian_reverse_inplace(sec.field_30); 57 boost::endian::endian_reverse_inplace(sec.field_34); 58 boost::endian::endian_reverse_inplace(sec.field_38); 59 boost::endian::endian_reverse_inplace(sec.field_3c); 60 boost::endian::endian_reverse_inplace(sec.field_40); 61 boost::endian::endian_reverse_inplace(sec.field_44); 62 boost::endian::endian_reverse_inplace(sec.field_48); 63 boost::endian::endian_reverse_inplace(sec.field_4c); 64 } 65 66 void Cl3::FileEntry::Validate(uint32_t block_size) const 67 { 68 #define VALIDATE(x) LIBSHIT_VALIDATE_FIELD("Cl3::FileEntry", x) 69 VALIDATE(name.is_valid()); 70 VALIDATE(data_offset <= block_size); 71 VALIDATE(data_offset + data_size <= block_size); 72 VALIDATE(field_214 == 0 && field_218 == 0 && field_21c == 0); 73 VALIDATE(field_220 == 0 && field_224 == 0 && field_228 == 0 && field_22c == 0); 74 #undef VALIDATE 75 } 76 77 void endian_reverse_inplace(Cl3::FileEntry& entry) noexcept 78 { 79 boost::endian::endian_reverse_inplace(entry.field_200); 80 boost::endian::endian_reverse_inplace(entry.data_offset); 81 boost::endian::endian_reverse_inplace(entry.data_size); 82 boost::endian::endian_reverse_inplace(entry.link_start); 83 boost::endian::endian_reverse_inplace(entry.link_count); 84 boost::endian::endian_reverse_inplace(entry.field_214); 85 boost::endian::endian_reverse_inplace(entry.field_218); 86 boost::endian::endian_reverse_inplace(entry.field_21c); 87 boost::endian::endian_reverse_inplace(entry.field_220); 88 boost::endian::endian_reverse_inplace(entry.field_224); 89 boost::endian::endian_reverse_inplace(entry.field_228); 90 boost::endian::endian_reverse_inplace(entry.field_22c); 91 } 92 93 void Cl3::LinkEntry::Validate(uint32_t i, uint32_t file_count) const 94 { 95 #define VALIDATE(x) LIBSHIT_VALIDATE_FIELD("Cl3::LinkEntry", x) 96 VALIDATE(field_00 == 0); 97 VALIDATE(linked_file_id < file_count); 98 VALIDATE(link_id == i); 99 VALIDATE(field_0c == 0); 100 VALIDATE(field_10 == 0 && field_14 == 0 && field_18 == 0 && field_1c == 0); 101 #undef VALIDATE 102 } 103 104 void endian_reverse_inplace(Cl3::LinkEntry& entry) noexcept 105 { 106 boost::endian::endian_reverse_inplace(entry.field_00); 107 boost::endian::endian_reverse_inplace(entry.linked_file_id); 108 boost::endian::endian_reverse_inplace(entry.link_id); 109 boost::endian::endian_reverse_inplace(entry.field_0c); 110 boost::endian::endian_reverse_inplace(entry.field_10); 111 boost::endian::endian_reverse_inplace(entry.field_14); 112 boost::endian::endian_reverse_inplace(entry.field_18); 113 boost::endian::endian_reverse_inplace(entry.field_1c); 114 } 115 116 void Cl3::Entry::Dispose() noexcept 117 { 118 links.clear(); 119 src.reset(); 120 } 121 122 Cl3::Cl3(Source src) 123 { 124 ADD_SOURCE(Parse_(src), src); 125 } 126 127 #if LIBSHIT_WITH_LUA 128 Cl3::Cl3(Libshit::Lua::StateRef vm, Endian endian, uint32_t field_14, 129 Libshit::Lua::RawTable tbl) 130 : endian{endian}, field_14{field_14} 131 { 132 auto [len, one] = vm.RawLen01(tbl); 133 entries.reserve(len); 134 bool do_links = false; 135 // we need two passes: first add entries, ignoring links 136 // then add links when the entries are in place 137 // can't do in one pass, as there may be forward links 138 vm.Fori(tbl, one, len, [&](size_t, int type) 139 { 140 if (type == LUA_TTABLE) 141 { 142 auto [len, one] = vm.RawLen01(-1); 143 if (len == 4) // todo: is there a better way?? 144 { 145 do_links = true; 146 lua_rawgeti(vm, -1, one); // +1 147 lua_rawgeti(vm, -2, one+1); // +2 148 lua_rawgeti(vm, -3, one+3); // +3 149 entries.emplace_back( 150 vm.Get<std::string>(-3), 151 vm.Get<uint32_t>(-2), 152 vm.Get<Libshit::SmartPtr<Dumpable>>(-1)); 153 lua_pop(vm, 3); 154 return; 155 } 156 } 157 // probably unlikely: try the 3-param ctor, or use existing entry 158 entries.push_back(vm.Get< 159 Libshit::Lua::AutoTable<Libshit::NotNull<Libshit::SmartPtr<Entry>>>>()); 160 }); 161 162 if (do_links) 163 { 164 vm.Fori(tbl, one, len, [&](size_t i, int type) 165 { 166 if (type != LUA_TTABLE) return; 167 auto [len, one] = vm.RawLen01(-1); 168 if (len != 4) return; 169 auto t = lua_rawgeti(vm, -1, one+2); // +1 170 if (t != LUA_TTABLE) vm.TypeError(false, "table", -1); 171 172 auto [len2, one2] = vm.RawLen01(-1); 173 auto& e = entries[i]; 174 e.links.reserve(len2); 175 vm.Fori(lua_absindex(vm, -1), one2, len2, [&](size_t, int) 176 { 177 auto it = entries.find(vm.Get<std::string>()); 178 if (it == entries.end()) 179 luaL_error(vm, "invalid cl3 link: '%s' not found", 180 vm.Get<const char*>()); 181 e.links.push_back(&*it); 182 }); 183 lua_pop(vm, 1); 184 }); 185 } 186 Fixup(); 187 } 188 #endif 189 190 void Cl3::Dispose() noexcept 191 { 192 entries.clear(); 193 } 194 195 void Cl3::Parse_(Source& src) 196 { 197 src.CheckSize(sizeof(Header)); 198 auto hdr = src.PreadGen<Header>(0); 199 endian = hdr.endian == 'L' ? Endian::LITTLE : Endian::BIG; 200 ToNative(hdr, endian); 201 hdr.Validate(src.GetSize()); 202 203 field_14 = hdr.field_14; 204 205 src.Seek(hdr.sections_offset); 206 uint32_t secs = hdr.sections_count; 207 208 uint32_t file_offset = 0, file_count = 0, file_size, 209 link_offset, link_count = 0; 210 for (size_t i = 0; i < secs; ++i) 211 { 212 auto sec = src.ReadGen<Section>(); 213 ToNative(sec, endian); 214 sec.Validate(src.GetSize()); 215 216 if (sec.name == "FILE_COLLECTION") 217 { 218 file_offset = sec.data_offset; 219 file_count = sec.count; 220 file_size = sec.data_size; 221 } 222 else if (sec.name == "FILE_LINK") 223 { 224 link_offset = sec.data_offset; 225 link_count = sec.count; 226 LIBSHIT_VALIDATE_FIELD( 227 "Cl3::Section", 228 sec.data_size == link_count * sizeof(LinkEntry)); 229 } 230 } 231 232 entries.reserve(file_count); 233 src.Seek(file_offset); 234 for (uint32_t i = 0; i < file_count; ++i) 235 { 236 auto e = src.ReadGen<FileEntry>(); 237 ToNative(e, endian); 238 e.Validate(file_size); 239 240 entries.emplace_back( 241 e.name.c_str(), e.field_200, Libshit::MakeSmart<DumpableSource>( 242 src, file_offset+e.data_offset, e.data_size)); 243 } 244 245 src.Seek(file_offset); 246 for (uint32_t i = 0; i < file_count; ++i) 247 { 248 auto e = src.ReadGen<FileEntry>(); 249 ToNative(e, endian); 250 auto& ls = entries[i].links; 251 uint32_t lbase = e.link_start; 252 uint32_t lcount = e.link_count; 253 254 for (uint32_t i = lbase; i < lbase+lcount; ++i) 255 { 256 auto le = src.PreadGen<LinkEntry>(link_offset + i*sizeof(LinkEntry)); 257 le.Validate(i - lbase, file_count); 258 ls.emplace_back(&entries[le.linked_file_id]); 259 } 260 } 261 } 262 263 static constexpr unsigned PAD_BYTES = 0x40; 264 static constexpr unsigned PAD = 0x3f; 265 void Cl3::Fixup() 266 { 267 data_size = 0; 268 link_count = 0; 269 for (auto& e : entries) 270 { 271 if (e.src) 272 { 273 e.src->Fixup(); 274 data_size += e.src->GetSize(); 275 } 276 data_size = (data_size + PAD) & ~PAD; 277 link_count += e.links.size(); 278 } 279 } 280 281 FilePosition Cl3::GetSize() const 282 { 283 FilePosition ret = (sizeof(Header)+PAD) & ~PAD; 284 ret = (ret+sizeof(Section)*2+PAD) & ~PAD; 285 ret = (ret+sizeof(FileEntry)*entries.size()+PAD) & ~PAD; 286 ret += data_size+sizeof(LinkEntry)*link_count; 287 return ret; 288 } 289 290 Cl3::Entry& Cl3::GetOrCreateFile(Libshit::StringView fname) 291 { 292 auto it = entries.find(fname, std::less<>{}); 293 if (it == entries.end()) 294 { 295 entries.emplace_back(fname.to_string()); 296 return entries.back(); 297 } 298 return *it; 299 } 300 301 void Cl3::ExtractTo(const boost::filesystem::path& dir) const 302 { 303 if (!boost::filesystem::is_directory(dir)) 304 boost::filesystem::create_directories(dir); 305 306 for (const auto& e : entries) 307 { 308 if (!e.src) continue; 309 auto sink = Sink::ToFile(dir / e.name.c_str(), e.src->GetSize()); 310 e.src->Dump(*sink); 311 } 312 } 313 314 void Cl3::UpdateFromDir(const boost::filesystem::path& dir) 315 { 316 for (auto& e : boost::filesystem::directory_iterator(dir)) 317 GetOrCreateFile(e.path().filename().string()).src = 318 Libshit::MakeSmart<DumpableSource>(Source::FromFile(e)); 319 320 for (auto it = entries.begin(); it != entries.end(); ) 321 if (!boost::filesystem::exists(dir / it->name)) 322 it = entries.erase(it); 323 else 324 ++it; 325 } 326 327 uint32_t Cl3::IndexOf(const Libshit::WeakSmartPtr<Entry>& ptr) const noexcept 328 { 329 auto sptr = ptr.lock(); 330 if (!sptr) return -1; 331 auto it = entries.checked_iterator_to(*sptr); 332 if (it == entries.end()) return -1; 333 return entries.index_of(it); 334 } 335 336 void Cl3::Inspect_(std::ostream& os, unsigned indent) const 337 { 338 os << "neptools.cl3(neptools.endian." << ToString(endian) << ", " 339 << field_14 << ", {\n"; 340 for (auto& e : entries) 341 { 342 Indent(os, indent+1) 343 << '{' << Libshit::Quoted(e.name) << ", " << e.field_200 << ", {"; 344 bool first = true; 345 for (auto& l : e.links) 346 { 347 if (!first) os << ", "; 348 first = false; 349 auto ll = l.lock(); 350 if (ll) Libshit::DumpBytes(os, ll->name); 351 else os << "nil"; 352 } 353 os << "}, "; 354 if (e.src) 355 e.src->Inspect(os, indent+1); 356 else 357 os << "nil"; 358 os << "},\n"; 359 } 360 os << "})"; 361 } 362 363 // workaround, revert to template if/when 364 // https://github.com/boostorg/endian/issues/41 is fixed 365 #define GEN_REVERSE(T) \ 366 static T endian_reverse(T t) noexcept \ 367 { \ 368 endian_reverse_inplace(t); \ 369 return t; \ 370 } 371 GEN_REVERSE(Cl3::Section); 372 GEN_REVERSE(Cl3::FileEntry); 373 GEN_REVERSE(Cl3::LinkEntry); 374 #undef GEN_REVERSE 375 376 void Cl3::Dump_(Sink& sink) const 377 { 378 auto sections_offset = (sizeof(Header)+PAD) & ~PAD; 379 auto files_offset = (sections_offset+sizeof(Section)*2+PAD) & ~PAD; 380 auto data_offset = (files_offset+sizeof(FileEntry)*entries.size()+PAD) & ~PAD; 381 auto link_offset = data_offset + data_size; 382 383 Header hdr; 384 memcpy(hdr.magic, "CL3", 3); 385 hdr.endian = endian == Endian::LITTLE ? 'L' : 'B'; 386 hdr.field_04 = 0; 387 hdr.field_08 = 3; 388 hdr.sections_count = 2; 389 hdr.sections_offset = sections_offset; 390 hdr.field_14 = field_14; 391 FromNative(hdr, endian); 392 sink.WriteGen(hdr); 393 sink.Pad(sections_offset-sizeof(Header)); 394 395 Section sec; 396 memset(&sec, 0, sizeof(Section)); 397 sec.name = "FILE_COLLECTION"; 398 sec.count = entries.size(); 399 sec.data_size = link_offset - files_offset; 400 sec.data_offset = files_offset; 401 sink.WriteGen(FromNativeCopy(sec, endian)); 402 403 sec.name = "FILE_LINK"; 404 sec.count = link_count; 405 sec.data_size = link_count * sizeof(LinkEntry); 406 sec.data_offset = link_offset; 407 FromNative(hdr, endian); 408 sink.WriteGen(sec); 409 sink.Pad((PAD_BYTES - ((2*sizeof(Section)) & PAD)) & PAD); 410 411 FileEntry fe; 412 fe.field_214 = fe.field_218 = fe.field_21c = 0; 413 fe.field_220 = fe.field_224 = fe.field_228 = fe.field_22c = 0; 414 415 // file entry header 416 uint32_t offset = data_offset-files_offset, link_i = 0; 417 for (auto& e : entries) 418 { 419 fe.name = e.name; 420 fe.field_200 = e.field_200; 421 fe.data_offset = offset; 422 auto size = e.src ? e.src->GetSize() : 0; 423 fe.data_size = size; 424 fe.link_start = link_i; 425 fe.link_count = e.links.size(); 426 sink.WriteGen(FromNativeCopy(fe, endian)); 427 428 offset = (offset+size+PAD) & ~PAD; 429 link_i += e.links.size(); 430 } 431 sink.Pad((PAD_BYTES - ((entries.size()*sizeof(FileEntry)) & PAD)) & PAD); 432 433 // file data 434 for (auto& e : entries) 435 { 436 if (!e.src) continue; 437 e.src->Dump(sink); 438 sink.Pad((PAD_BYTES - (e.src->GetSize() & PAD)) & PAD); 439 } 440 441 // links 442 LinkEntry le; 443 memset(&le, 0, sizeof(LinkEntry)); 444 for (auto& e : entries) 445 { 446 uint32_t i = 0; 447 for (const auto& l : e.links) 448 { 449 le.linked_file_id = IndexOf(l); 450 if (le.linked_file_id == uint32_t(-1)) 451 LIBSHIT_THROW(std::runtime_error, "Invalid file link"); 452 le.link_id = i++; 453 sink.WriteGen(FromNativeCopy(le, endian)); 454 } 455 } 456 } 457 458 Stcm::File& Cl3::GetStcm() 459 { 460 auto dat = entries.find("main.DAT", std::less<>{}); 461 if (dat == entries.end() || !dat->src) 462 LIBSHIT_THROW(Libshit::DecodeError, "Invalid CL3 file: no main.DAT"); 463 464 auto stcm = dynamic_cast<Stcm::File*>(dat->src.get()); 465 if (stcm) return *stcm; 466 467 auto src = Libshit::asserted_cast<DumpableSource*>(dat->src.get()); 468 auto nstcm = Libshit::MakeSmart<Stcm::File>(src->GetSource()); 469 auto ret = nstcm.get(); 470 dat->src = std::move(nstcm); 471 return *ret; 472 } 473 474 Libshit::NotNullSharedPtr<TxtSerializable> Cl3::GetDefaultTxtSerializable( 475 const Libshit::NotNullSharedPtr<Dumpable>& thiz) 476 { 477 auto& stcm = GetStcm(); 478 if (!stcm.GetGbnl()) 479 LIBSHIT_THROW(Libshit::DecodeError, "No GBNL found in STCM"); 480 return Libshit::NotNullRefCountedPtr<Stcm::File>{&stcm}; 481 } 482 483 static OpenFactory cl3_open{[](const Source& src) -> Libshit::SmartPtr<Dumpable> 484 { 485 if (src.GetSize() < sizeof(Cl3::Header)) return nullptr; 486 char buf[3]; 487 src.PreadGen(0, buf); 488 if (memcmp(buf, "CL3", 3) == 0) 489 return Libshit::MakeSmart<Cl3>(src); 490 else 491 return nullptr; 492 }}; 493 494 } 495 496 LIBSHIT_ORDERED_MAP_LUAGEN( 497 cl3_entry, Neptools::Cl3::Entry, Neptools::Cl3::EntryKeyOfValue); 498 LIBSHIT_STD_VECTOR_LUAGEN( 499 cl3_entry, Libshit::WeakRefCountedPtr<Neptools::Cl3::Entry>); 500 #include "cl3.binding.hpp"