http_downloader.cpp (9507B)
1 // SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com> 2 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) 3 4 #include "http_downloader.h" 5 6 #include "common/assert.h" 7 #include "common/log.h" 8 #include "common/progress_callback.h" 9 #include "common/string_util.h" 10 #include "common/timer.h" 11 12 Log_SetChannel(HTTPDownloader); 13 14 static constexpr float DEFAULT_TIMEOUT_IN_SECONDS = 30; 15 static constexpr u32 DEFAULT_MAX_ACTIVE_REQUESTS = 4; 16 17 const char HTTPDownloader::DEFAULT_USER_AGENT[] = 18 "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:85.0) Gecko/20100101 Firefox/85.0"; 19 20 HTTPDownloader::HTTPDownloader() 21 : m_timeout(DEFAULT_TIMEOUT_IN_SECONDS), m_max_active_requests(DEFAULT_MAX_ACTIVE_REQUESTS) 22 { 23 } 24 25 HTTPDownloader::~HTTPDownloader() = default; 26 27 void HTTPDownloader::SetTimeout(float timeout) 28 { 29 m_timeout = timeout; 30 } 31 32 void HTTPDownloader::SetMaxActiveRequests(u32 max_active_requests) 33 { 34 Assert(max_active_requests > 0); 35 m_max_active_requests = max_active_requests; 36 } 37 38 void HTTPDownloader::CreateRequest(std::string url, Request::Callback callback, ProgressCallback* progress) 39 { 40 Request* req = InternalCreateRequest(); 41 req->parent = this; 42 req->type = Request::Type::Get; 43 req->url = std::move(url); 44 req->callback = std::move(callback); 45 req->progress = progress; 46 req->start_time = Common::Timer::GetCurrentValue(); 47 48 std::unique_lock<std::mutex> lock(m_pending_http_request_lock); 49 if (LockedGetActiveRequestCount() < m_max_active_requests) 50 { 51 if (!StartRequest(req)) 52 return; 53 } 54 55 LockedAddRequest(req); 56 } 57 58 void HTTPDownloader::CreatePostRequest(std::string url, std::string post_data, Request::Callback callback, 59 ProgressCallback* progress) 60 { 61 Request* req = InternalCreateRequest(); 62 req->parent = this; 63 req->type = Request::Type::Post; 64 req->url = std::move(url); 65 req->post_data = std::move(post_data); 66 req->callback = std::move(callback); 67 req->progress = progress; 68 req->start_time = Common::Timer::GetCurrentValue(); 69 70 std::unique_lock<std::mutex> lock(m_pending_http_request_lock); 71 if (LockedGetActiveRequestCount() < m_max_active_requests) 72 { 73 if (!StartRequest(req)) 74 return; 75 } 76 77 LockedAddRequest(req); 78 } 79 80 void HTTPDownloader::LockedPollRequests(std::unique_lock<std::mutex>& lock) 81 { 82 if (m_pending_http_requests.empty()) 83 return; 84 85 InternalPollRequests(); 86 87 const Common::Timer::Value current_time = Common::Timer::GetCurrentValue(); 88 u32 active_requests = 0; 89 u32 unstarted_requests = 0; 90 91 for (size_t index = 0; index < m_pending_http_requests.size();) 92 { 93 Request* req = m_pending_http_requests[index]; 94 if (req->state == Request::State::Pending) 95 { 96 unstarted_requests++; 97 index++; 98 continue; 99 } 100 101 if ((req->state == Request::State::Started || req->state == Request::State::Receiving) && 102 current_time >= req->start_time && 103 Common::Timer::ConvertValueToSeconds(current_time - req->start_time) >= m_timeout) 104 { 105 // request timed out 106 ERROR_LOG("Request for '{}' timed out", req->url); 107 108 req->state.store(Request::State::Cancelled); 109 m_pending_http_requests.erase(m_pending_http_requests.begin() + index); 110 lock.unlock(); 111 112 req->callback(HTTP_STATUS_TIMEOUT, std::string(), Request::Data()); 113 114 CloseRequest(req); 115 116 lock.lock(); 117 continue; 118 } 119 else if ((req->state == Request::State::Started || req->state == Request::State::Receiving) && req->progress && 120 req->progress->IsCancelled()) 121 { 122 // request timed out 123 ERROR_LOG("Request for '{}' cancelled", req->url); 124 125 req->state.store(Request::State::Cancelled); 126 m_pending_http_requests.erase(m_pending_http_requests.begin() + index); 127 lock.unlock(); 128 129 req->callback(HTTP_STATUS_CANCELLED, std::string(), Request::Data()); 130 131 CloseRequest(req); 132 133 lock.lock(); 134 continue; 135 } 136 137 if (req->state != Request::State::Complete) 138 { 139 if (req->progress) 140 { 141 const u32 size = static_cast<u32>(req->data.size()); 142 if (size != req->last_progress_update) 143 { 144 req->last_progress_update = size; 145 req->progress->SetProgressRange(req->content_length); 146 req->progress->SetProgressValue(req->last_progress_update); 147 } 148 } 149 150 active_requests++; 151 index++; 152 continue; 153 } 154 155 // request complete 156 VERBOSE_LOG("Request for '{}' complete, returned status code {} and {} bytes", req->url, req->status_code, 157 req->data.size()); 158 m_pending_http_requests.erase(m_pending_http_requests.begin() + index); 159 160 // run callback with lock unheld 161 lock.unlock(); 162 req->callback(req->status_code, req->content_type, std::move(req->data)); 163 CloseRequest(req); 164 lock.lock(); 165 } 166 167 // start new requests when we finished some 168 if (unstarted_requests > 0 && active_requests < m_max_active_requests) 169 { 170 for (size_t index = 0; index < m_pending_http_requests.size();) 171 { 172 Request* req = m_pending_http_requests[index]; 173 if (req->state != Request::State::Pending) 174 { 175 index++; 176 continue; 177 } 178 179 if (!StartRequest(req)) 180 { 181 m_pending_http_requests.erase(m_pending_http_requests.begin() + index); 182 continue; 183 } 184 185 active_requests++; 186 index++; 187 188 if (active_requests >= m_max_active_requests) 189 break; 190 } 191 } 192 } 193 194 void HTTPDownloader::PollRequests() 195 { 196 std::unique_lock<std::mutex> lock(m_pending_http_request_lock); 197 LockedPollRequests(lock); 198 } 199 200 void HTTPDownloader::WaitForAllRequests() 201 { 202 std::unique_lock<std::mutex> lock(m_pending_http_request_lock); 203 while (!m_pending_http_requests.empty()) 204 { 205 // Don't burn too much CPU. 206 Common::Timer::NanoSleep(1000000); 207 LockedPollRequests(lock); 208 } 209 } 210 211 void HTTPDownloader::LockedAddRequest(Request* request) 212 { 213 m_pending_http_requests.push_back(request); 214 } 215 216 u32 HTTPDownloader::LockedGetActiveRequestCount() 217 { 218 u32 count = 0; 219 for (Request* req : m_pending_http_requests) 220 { 221 if (req->state == Request::State::Started || req->state == Request::State::Receiving) 222 count++; 223 } 224 return count; 225 } 226 227 bool HTTPDownloader::HasAnyRequests() 228 { 229 std::unique_lock<std::mutex> lock(m_pending_http_request_lock); 230 return !m_pending_http_requests.empty(); 231 } 232 233 std::string HTTPDownloader::GetExtensionForContentType(const std::string& content_type) 234 { 235 // Based on https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types 236 static constexpr const char* table[][2] = { 237 {"audio/aac", "aac"}, 238 {"application/x-abiword", "abw"}, 239 {"application/x-freearc", "arc"}, 240 {"image/avif", "avif"}, 241 {"video/x-msvideo", "avi"}, 242 {"application/vnd.amazon.ebook", "azw"}, 243 {"application/octet-stream", "bin"}, 244 {"image/bmp", "bmp"}, 245 {"application/x-bzip", "bz"}, 246 {"application/x-bzip2", "bz2"}, 247 {"application/x-cdf", "cda"}, 248 {"application/x-csh", "csh"}, 249 {"text/css", "css"}, 250 {"text/csv", "csv"}, 251 {"application/msword", "doc"}, 252 {"application/vnd.openxmlformats-officedocument.wordprocessingml.document", "docx"}, 253 {"application/vnd.ms-fontobject", "eot"}, 254 {"application/epub+zip", "epub"}, 255 {"application/gzip", "gz"}, 256 {"image/gif", "gif"}, 257 {"text/html", "htm"}, 258 {"image/vnd.microsoft.icon", "ico"}, 259 {"text/calendar", "ics"}, 260 {"application/java-archive", "jar"}, 261 {"image/jpeg", "jpg"}, 262 {"text/javascript", "js"}, 263 {"application/json", "json"}, 264 {"application/ld+json", "jsonld"}, 265 {"audio/midi audio/x-midi", "mid"}, 266 {"text/javascript", "mjs"}, 267 {"audio/mpeg", "mp3"}, 268 {"video/mp4", "mp4"}, 269 {"video/mpeg", "mpeg"}, 270 {"application/vnd.apple.installer+xml", "mpkg"}, 271 {"application/vnd.oasis.opendocument.presentation", "odp"}, 272 {"application/vnd.oasis.opendocument.spreadsheet", "ods"}, 273 {"application/vnd.oasis.opendocument.text", "odt"}, 274 {"audio/ogg", "oga"}, 275 {"video/ogg", "ogv"}, 276 {"application/ogg", "ogx"}, 277 {"audio/opus", "opus"}, 278 {"font/otf", "otf"}, 279 {"image/png", "png"}, 280 {"application/pdf", "pdf"}, 281 {"application/x-httpd-php", "php"}, 282 {"application/vnd.ms-powerpoint", "ppt"}, 283 {"application/vnd.openxmlformats-officedocument.presentationml.presentation", "pptx"}, 284 {"application/vnd.rar", "rar"}, 285 {"application/rtf", "rtf"}, 286 {"application/x-sh", "sh"}, 287 {"image/svg+xml", "svg"}, 288 {"application/x-tar", "tar"}, 289 {"image/tiff", "tif"}, 290 {"video/mp2t", "ts"}, 291 {"font/ttf", "ttf"}, 292 {"text/plain", "txt"}, 293 {"application/vnd.visio", "vsd"}, 294 {"audio/wav", "wav"}, 295 {"audio/webm", "weba"}, 296 {"video/webm", "webm"}, 297 {"image/webp", "webp"}, 298 {"font/woff", "woff"}, 299 {"font/woff2", "woff2"}, 300 {"application/xhtml+xml", "xhtml"}, 301 {"application/vnd.ms-excel", "xls"}, 302 {"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx"}, 303 {"application/xml", "xml"}, 304 {"text/xml", "xml"}, 305 {"application/vnd.mozilla.xul+xml", "xul"}, 306 {"application/zip", "zip"}, 307 {"video/3gpp", "3gp"}, 308 {"audio/3gpp", "3gp"}, 309 {"video/3gpp2", "3g2"}, 310 {"audio/3gpp2", "3g2"}, 311 {"application/x-7z-compressed", "7z"}, 312 }; 313 314 std::string ret; 315 for (size_t i = 0; i < std::size(table); i++) 316 { 317 if (StringUtil::Strncasecmp(table[i][0], content_type.data(), content_type.length()) == 0) 318 { 319 ret = table[i][1]; 320 break; 321 } 322 } 323 return ret; 324 }