Line data Source code
1 : /* Copyright (C) 2022 Wildfire Games.
2 : * This file is part of 0 A.D.
3 : *
4 : * 0 A.D. is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 2 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * 0 A.D. is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with 0 A.D. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "precompiled.h"
19 :
20 : #include "UserReport.h"
21 :
22 : #include "lib/timer.h"
23 : #include "lib/utf8.h"
24 : #include "lib/external_libraries/curl.h"
25 : #include "lib/external_libraries/zlib.h"
26 : #include "lib/file/archive/stream.h"
27 : #include "lib/os_path.h"
28 : #include "lib/sysdep/sysdep.h"
29 : #include "ps/ConfigDB.h"
30 : #include "ps/Filesystem.h"
31 : #include "ps/Profiler2.h"
32 : #include "ps/Pyrogenesis.h"
33 : #include "ps/Threading.h"
34 :
35 : #include <condition_variable>
36 : #include <deque>
37 : #include <fstream>
38 : #include <mutex>
39 : #include <string>
40 : #include <thread>
41 :
42 : #define DEBUG_UPLOADS 0
43 :
44 : /*
45 : * The basic idea is that the game submits reports to us, which we send over
46 : * HTTP to a server for storage and analysis.
47 : *
48 : * We can't use libcurl's asynchronous 'multi' API, because DNS resolution can
49 : * be synchronous and slow (which would make the game pause).
50 : * So we use the 'easy' API in a background thread.
51 : * The main thread submits reports, toggles whether uploading is enabled,
52 : * and polls for the current status (typically to display in the GUI);
53 : * the worker thread does all of the uploading.
54 : *
55 : * It'd be nice to extend this in the future to handle things like crash reports.
56 : * The game should store the crashlogs (suitably anonymised) in a directory, and
57 : * we should detect those files and upload them when we're restarted and online.
58 : */
59 :
60 :
61 : /**
62 : * Version number stored in config file when the user agrees to the reporting.
63 : * Reporting will be disabled if the config value is missing or is less than
64 : * this value. If we start reporting a lot more data, we should increase this
65 : * value and get the user to re-confirm.
66 : */
67 : static const int REPORTER_VERSION = 1;
68 :
69 : /**
70 : * Time interval (seconds) at which the worker thread will check its reconnection
71 : * timers. (This should be relatively high so the thread doesn't waste much time
72 : * continually waking up.)
73 : */
74 : static const double TIMER_CHECK_INTERVAL = 10.0;
75 :
76 : /**
77 : * Seconds we should wait before reconnecting to the server after a failure.
78 : */
79 : static const double RECONNECT_INVERVAL = 60.0;
80 :
81 1 : CUserReporter g_UserReporter;
82 :
83 0 : struct CUserReport
84 : {
85 : time_t m_Time;
86 : std::string m_Type;
87 : int m_Version;
88 : std::string m_Data;
89 : };
90 :
91 : class CUserReporterWorker
92 : {
93 : public:
94 0 : CUserReporterWorker(const std::string& userID, const std::string& url) :
95 : m_URL(url), m_UserID(userID), m_Enabled(false), m_Shutdown(false), m_Status("disabled"),
96 0 : m_PauseUntilTime(timer_Time()), m_LastUpdateTime(timer_Time())
97 : {
98 : // Set up libcurl:
99 :
100 0 : m_Curl = curl_easy_init();
101 0 : ENSURE(m_Curl);
102 :
103 : #if DEBUG_UPLOADS
104 : curl_easy_setopt(m_Curl, CURLOPT_VERBOSE, 1L);
105 : #endif
106 :
107 : // Capture error messages
108 0 : curl_easy_setopt(m_Curl, CURLOPT_ERRORBUFFER, m_ErrorBuffer);
109 :
110 : // Disable signal handlers (required for multithreaded applications)
111 0 : curl_easy_setopt(m_Curl, CURLOPT_NOSIGNAL, 1L);
112 :
113 : // To minimise security risks, don't support redirects
114 0 : curl_easy_setopt(m_Curl, CURLOPT_FOLLOWLOCATION, 0L);
115 :
116 : // Prevent this thread from blocking the engine shutdown for 5 minutes in case the server is unavailable
117 0 : curl_easy_setopt(m_Curl, CURLOPT_CONNECTTIMEOUT, 10L);
118 :
119 : // Set IO callbacks
120 0 : curl_easy_setopt(m_Curl, CURLOPT_WRITEFUNCTION, ReceiveCallback);
121 0 : curl_easy_setopt(m_Curl, CURLOPT_WRITEDATA, this);
122 0 : curl_easy_setopt(m_Curl, CURLOPT_READFUNCTION, SendCallback);
123 0 : curl_easy_setopt(m_Curl, CURLOPT_READDATA, this);
124 :
125 : // Set URL to POST to
126 0 : curl_easy_setopt(m_Curl, CURLOPT_URL, url.c_str());
127 0 : curl_easy_setopt(m_Curl, CURLOPT_POST, 1L);
128 :
129 : // Set up HTTP headers
130 0 : m_Headers = NULL;
131 : // Set the UA string
132 0 : std::string ua = "User-Agent: 0ad ";
133 0 : ua += curl_version();
134 0 : ua += " (http://play0ad.com/)";
135 0 : m_Headers = curl_slist_append(m_Headers, ua.c_str());
136 : // Override the default application/x-www-form-urlencoded type since we're not using that type
137 0 : m_Headers = curl_slist_append(m_Headers, "Content-Type: application/octet-stream");
138 : // Disable the Accept header because it's a waste of a dozen bytes
139 0 : m_Headers = curl_slist_append(m_Headers, "Accept: ");
140 0 : curl_easy_setopt(m_Curl, CURLOPT_HTTPHEADER, m_Headers);
141 :
142 0 : m_WorkerThread = std::thread(Threading::HandleExceptions<RunThread>::Wrapper, this);
143 0 : }
144 :
145 0 : ~CUserReporterWorker()
146 0 : {
147 0 : curl_slist_free_all(m_Headers);
148 0 : curl_easy_cleanup(m_Curl);
149 0 : }
150 :
151 : /**
152 : * Called by main thread, when the online reporting is enabled/disabled.
153 : */
154 0 : void SetEnabled(bool enabled)
155 : {
156 0 : std::lock_guard<std::mutex> lock(m_WorkerMutex);
157 0 : if (enabled != m_Enabled)
158 : {
159 0 : m_Enabled = enabled;
160 :
161 : // Wake up the worker thread
162 0 : m_WorkerCV.notify_all();
163 : }
164 0 : }
165 :
166 : /**
167 : * Called by main thread to request shutdown.
168 : * Returns true if we've shut down successfully.
169 : * Returns false if shutdown is taking too long (we might be blocked on a
170 : * sync network operation) - you mustn't destroy this object, just leak it
171 : * and terminate.
172 : */
173 0 : bool Shutdown()
174 : {
175 : {
176 0 : std::lock_guard<std::mutex> lock(m_WorkerMutex);
177 0 : m_Shutdown = true;
178 : }
179 :
180 : // Wake up the worker thread
181 0 : m_WorkerCV.notify_all();
182 :
183 : // Wait for it to shut down cleanly
184 : // TODO: should have a timeout in case of network hangs
185 0 : m_WorkerThread.join();
186 :
187 0 : return true;
188 : }
189 :
190 : /**
191 : * Called by main thread to determine the current status of the uploader.
192 : */
193 0 : std::string GetStatus()
194 : {
195 0 : std::lock_guard<std::mutex> lock(m_WorkerMutex);
196 0 : return m_Status;
197 : }
198 :
199 : /**
200 : * Called by main thread to add a new report to the queue.
201 : */
202 0 : void Submit(const std::shared_ptr<CUserReport>& report)
203 : {
204 : {
205 0 : std::lock_guard<std::mutex> lock(m_WorkerMutex);
206 0 : m_ReportQueue.push_back(report);
207 : }
208 :
209 : // Wake up the worker thread
210 0 : m_WorkerCV.notify_all();
211 0 : }
212 :
213 : /**
214 : * Called by the main thread every frame, so we can check
215 : * retransmission timers.
216 : */
217 0 : void Update()
218 : {
219 0 : double now = timer_Time();
220 0 : if (now > m_LastUpdateTime + TIMER_CHECK_INTERVAL)
221 : {
222 : // Wake up the worker thread
223 0 : m_WorkerCV.notify_all();
224 :
225 0 : m_LastUpdateTime = now;
226 : }
227 0 : }
228 :
229 : private:
230 0 : static void RunThread(CUserReporterWorker* data)
231 : {
232 0 : debug_SetThreadName("CUserReportWorker");
233 0 : g_Profiler2.RegisterCurrentThread("userreport");
234 :
235 0 : data->Run();
236 0 : }
237 :
238 0 : void Run()
239 : {
240 : // Set libcurl's proxy configuration
241 : // (This has to be done in the thread because it's potentially very slow)
242 0 : SetStatus("proxy");
243 0 : std::wstring proxy;
244 :
245 : {
246 0 : PROFILE2("get proxy config");
247 0 : if (sys_get_proxy_config(wstring_from_utf8(m_URL), proxy) == INFO::OK)
248 0 : curl_easy_setopt(m_Curl, CURLOPT_PROXY, utf8_from_wstring(proxy).c_str());
249 : }
250 :
251 0 : SetStatus("waiting");
252 :
253 : /*
254 : * We use a condition_variable to let the thread be woken up when it has
255 : * work to do. Various actions from the main thread can wake it:
256 : * * SetEnabled()
257 : * * Shutdown()
258 : * * Submit()
259 : * * Retransmission timeouts, once every several seconds
260 : *
261 : * If multiple actions have triggered wakeups, we might respond to
262 : * all of those actions after the first wakeup, which is okay (we'll do
263 : * nothing during the subsequent wakeups). We should never hang due to
264 : * processing fewer actions than wakeups.
265 : *
266 : * Retransmission timeouts are triggered via the main thread.
267 : */
268 :
269 : // Wait until the main thread wakes us up
270 : while (true)
271 : {
272 0 : g_Profiler2.RecordRegionEnter("condition_variable wait");
273 :
274 0 : std::unique_lock<std::mutex> lock(m_WorkerMutex);
275 0 : m_WorkerCV.wait(lock);
276 0 : lock.unlock();
277 :
278 0 : g_Profiler2.RecordRegionLeave();
279 :
280 : // Handle shutdown requests as soon as possible
281 0 : if (GetShutdown())
282 0 : return;
283 :
284 : // If we're not enabled, ignore this wakeup
285 0 : if (!GetEnabled())
286 0 : continue;
287 :
288 : // If we're still pausing due to a failed connection,
289 : // go back to sleep again
290 0 : if (timer_Time() < m_PauseUntilTime)
291 0 : continue;
292 :
293 : // We're enabled, so process as many reports as possible
294 0 : while (ProcessReport())
295 : {
296 : // Handle shutdowns while we were sending the report
297 0 : if (GetShutdown())
298 0 : return;
299 : }
300 0 : }
301 : }
302 :
303 0 : bool GetEnabled()
304 : {
305 0 : std::lock_guard<std::mutex> lock(m_WorkerMutex);
306 0 : return m_Enabled;
307 : }
308 :
309 0 : bool GetShutdown()
310 : {
311 0 : std::lock_guard<std::mutex> lock(m_WorkerMutex);
312 0 : return m_Shutdown;
313 : }
314 :
315 0 : void SetStatus(const std::string& status)
316 : {
317 0 : std::lock_guard<std::mutex> lock(m_WorkerMutex);
318 0 : m_Status = status;
319 : #if DEBUG_UPLOADS
320 : debug_printf(">>> CUserReporterWorker status: %s\n", status.c_str());
321 : #endif
322 0 : }
323 :
324 0 : bool ProcessReport()
325 : {
326 0 : PROFILE2("process report");
327 :
328 0 : std::shared_ptr<CUserReport> report;
329 :
330 : {
331 0 : std::lock_guard<std::mutex> lock(m_WorkerMutex);
332 0 : if (m_ReportQueue.empty())
333 0 : return false;
334 0 : report = m_ReportQueue.front();
335 0 : m_ReportQueue.pop_front();
336 : }
337 :
338 0 : ConstructRequestData(*report);
339 0 : m_RequestDataOffset = 0;
340 0 : m_ResponseData.clear();
341 0 : m_ErrorBuffer[0] = '\0';
342 :
343 0 : curl_easy_setopt(m_Curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)m_RequestData.size());
344 :
345 0 : SetStatus("connecting");
346 :
347 : #if DEBUG_UPLOADS
348 : TIMER(L"CUserReporterWorker request");
349 : #endif
350 :
351 0 : CURLcode err = curl_easy_perform(m_Curl);
352 :
353 : #if DEBUG_UPLOADS
354 : printf(">>>\n%s\n<<<\n", m_ResponseData.c_str());
355 : #endif
356 :
357 0 : if (err == CURLE_OK)
358 : {
359 0 : long code = -1;
360 0 : curl_easy_getinfo(m_Curl, CURLINFO_RESPONSE_CODE, &code);
361 0 : SetStatus("completed:" + CStr::FromInt(code));
362 :
363 : // Check for success code
364 0 : if (code == 200)
365 0 : return true;
366 :
367 : // If the server returns the 410 Gone status, interpret that as meaning
368 : // it no longer supports uploads (at least from this version of the game),
369 : // so shut down and stop talking to it (to avoid wasting bandwidth)
370 0 : if (code == 410)
371 : {
372 0 : std::lock_guard<std::mutex> lock(m_WorkerMutex);
373 0 : m_Shutdown = true;
374 0 : return false;
375 : }
376 : }
377 : else
378 : {
379 0 : std::string errorString(m_ErrorBuffer);
380 :
381 0 : if (errorString.empty())
382 0 : errorString = curl_easy_strerror(err);
383 :
384 0 : SetStatus("failed:" + CStr::FromInt(err) + ":" + errorString);
385 : }
386 :
387 : // We got an unhandled return code or a connection failure;
388 : // push this report back onto the queue and try again after
389 : // a long interval
390 :
391 : {
392 0 : std::lock_guard<std::mutex> lock(m_WorkerMutex);
393 0 : m_ReportQueue.push_front(report);
394 : }
395 :
396 0 : m_PauseUntilTime = timer_Time() + RECONNECT_INVERVAL;
397 0 : return false;
398 : }
399 :
400 0 : void ConstructRequestData(const CUserReport& report)
401 : {
402 : // Construct the POST request data in the application/x-www-form-urlencoded format
403 :
404 0 : std::string r;
405 :
406 0 : r += "user_id=";
407 0 : AppendEscaped(r, m_UserID);
408 :
409 0 : r += "&time=" + CStr::FromInt64(report.m_Time);
410 :
411 0 : r += "&type=";
412 0 : AppendEscaped(r, report.m_Type);
413 :
414 0 : r += "&version=" + CStr::FromInt(report.m_Version);
415 :
416 0 : r += "&data=";
417 0 : AppendEscaped(r, report.m_Data);
418 :
419 : // Compress the content with zlib to save bandwidth.
420 : // (Note that we send a request with unlabelled compressed data instead
421 : // of using Content-Encoding, because Content-Encoding is a mess and causes
422 : // problems with servers and breaks Content-Length and this is much easier.)
423 0 : std::string compressed;
424 0 : compressed.resize(compressBound(r.size()));
425 0 : uLongf destLen = compressed.size();
426 0 : int ok = compress((Bytef*)compressed.c_str(), &destLen, (const Bytef*)r.c_str(), r.size());
427 0 : ENSURE(ok == Z_OK);
428 0 : compressed.resize(destLen);
429 :
430 0 : m_RequestData.swap(compressed);
431 0 : }
432 :
433 0 : void AppendEscaped(std::string& buffer, const std::string& str)
434 : {
435 0 : char* escaped = curl_easy_escape(m_Curl, str.c_str(), str.size());
436 0 : buffer += escaped;
437 0 : curl_free(escaped);
438 0 : }
439 :
440 0 : static size_t ReceiveCallback(void* buffer, size_t size, size_t nmemb, void* userp)
441 : {
442 0 : CUserReporterWorker* self = static_cast<CUserReporterWorker*>(userp);
443 :
444 0 : if (self->GetShutdown())
445 0 : return 0; // signals an error
446 :
447 0 : self->m_ResponseData += std::string((char*)buffer, (char*)buffer+size*nmemb);
448 :
449 0 : return size*nmemb;
450 : }
451 :
452 0 : static size_t SendCallback(char* bufptr, size_t size, size_t nmemb, void* userp)
453 : {
454 0 : CUserReporterWorker* self = static_cast<CUserReporterWorker*>(userp);
455 :
456 0 : if (self->GetShutdown())
457 0 : return CURL_READFUNC_ABORT; // signals an error
458 :
459 : // We can return as much data as available, up to the buffer size
460 0 : size_t amount = std::min(self->m_RequestData.size() - self->m_RequestDataOffset, size*nmemb);
461 :
462 : // ...But restrict to sending a small amount at once, so that we remain
463 : // responsive to shutdown requests even if the network is pretty slow
464 0 : amount = std::min((size_t)1024, amount);
465 :
466 0 : if(amount != 0) // (avoids invalid operator[] call where index=size)
467 : {
468 0 : memcpy(bufptr, &self->m_RequestData[self->m_RequestDataOffset], amount);
469 0 : self->m_RequestDataOffset += amount;
470 : }
471 :
472 0 : self->SetStatus("sending:" + CStr::FromDouble((double)self->m_RequestDataOffset / self->m_RequestData.size()));
473 :
474 0 : return amount;
475 : }
476 :
477 : private:
478 : // Thread-related members:
479 : std::thread m_WorkerThread;
480 : std::mutex m_WorkerMutex;
481 : std::condition_variable m_WorkerCV;
482 :
483 : // Shared by main thread and worker thread:
484 : // These variables are all protected by m_WorkerMutex
485 : std::deque<std::shared_ptr<CUserReport>> m_ReportQueue;
486 : bool m_Enabled;
487 : bool m_Shutdown;
488 : std::string m_Status;
489 :
490 : // Initialised in constructor by main thread; otherwise used only by worker thread:
491 : std::string m_URL;
492 : std::string m_UserID;
493 : CURL* m_Curl;
494 : curl_slist* m_Headers;
495 : double m_PauseUntilTime;
496 :
497 : // Only used by worker thread:
498 : std::string m_ResponseData;
499 : std::string m_RequestData;
500 : size_t m_RequestDataOffset;
501 : char m_ErrorBuffer[CURL_ERROR_SIZE];
502 :
503 : // Only used by main thread:
504 : double m_LastUpdateTime;
505 : };
506 :
507 :
508 :
509 1 : CUserReporter::CUserReporter() :
510 1 : m_Worker(NULL)
511 : {
512 1 : }
513 :
514 2 : CUserReporter::~CUserReporter()
515 : {
516 1 : ENSURE(!m_Worker); // Deinitialize should have been called before shutdown
517 1 : }
518 :
519 0 : std::string CUserReporter::LoadUserID()
520 : {
521 0 : std::string userID;
522 :
523 : // Read the user ID from user.cfg (if there is one)
524 0 : CFG_GET_VAL("userreport.id", userID);
525 :
526 : // If we don't have a validly-formatted user ID, generate a new one
527 0 : if (userID.length() != 16)
528 : {
529 0 : u8 bytes[8] = {0};
530 0 : sys_generate_random_bytes(bytes, ARRAY_SIZE(bytes));
531 : // ignore failures - there's not much we can do about it
532 :
533 0 : userID = "";
534 0 : for (size_t i = 0; i < ARRAY_SIZE(bytes); ++i)
535 : {
536 : char hex[3];
537 0 : sprintf_s(hex, ARRAY_SIZE(hex), "%02x", (unsigned int)bytes[i]);
538 0 : userID += hex;
539 : }
540 :
541 0 : g_ConfigDB.SetValueString(CFG_USER, "userreport.id", userID);
542 0 : g_ConfigDB.WriteValueToFile(CFG_USER, "userreport.id", userID);
543 : }
544 :
545 0 : return userID;
546 : }
547 :
548 0 : bool CUserReporter::IsReportingEnabled()
549 : {
550 0 : int version = -1;
551 0 : CFG_GET_VAL("userreport.enabledversion", version);
552 0 : return (version >= REPORTER_VERSION);
553 : }
554 :
555 0 : void CUserReporter::SetReportingEnabled(bool enabled)
556 : {
557 0 : CStr val = CStr::FromInt(enabled ? REPORTER_VERSION : 0);
558 0 : g_ConfigDB.SetValueString(CFG_USER, "userreport.enabledversion", val);
559 0 : g_ConfigDB.WriteValueToFile(CFG_USER, "userreport.enabledversion", val);
560 :
561 0 : if (m_Worker)
562 0 : m_Worker->SetEnabled(enabled);
563 0 : }
564 :
565 0 : std::string CUserReporter::GetStatus()
566 : {
567 0 : if (!m_Worker)
568 0 : return "disabled";
569 :
570 0 : return m_Worker->GetStatus();
571 : }
572 :
573 0 : void CUserReporter::Initialize()
574 : {
575 0 : ENSURE(!m_Worker); // must only be called once
576 :
577 0 : std::string userID = LoadUserID();
578 0 : std::string url;
579 0 : CFG_GET_VAL("userreport.url_upload", url);
580 :
581 0 : m_Worker = new CUserReporterWorker(userID, url);
582 :
583 0 : m_Worker->SetEnabled(IsReportingEnabled());
584 0 : }
585 :
586 0 : void CUserReporter::Deinitialize()
587 : {
588 0 : if (!m_Worker)
589 0 : return;
590 :
591 0 : if (m_Worker->Shutdown())
592 : {
593 : // Worker was shut down cleanly
594 0 : SAFE_DELETE(m_Worker);
595 : }
596 : else
597 : {
598 : // Worker failed to shut down in a reasonable time
599 : // Leak the resources (since that's better than hanging or crashing)
600 0 : m_Worker = NULL;
601 : }
602 : }
603 :
604 0 : void CUserReporter::Update()
605 : {
606 0 : if (m_Worker)
607 0 : m_Worker->Update();
608 0 : }
609 :
610 0 : void CUserReporter::SubmitReport(const std::string& type, int version, const std::string& data, const std::string& dataHumanReadable)
611 : {
612 : // Write to logfile, enabling users to assess privacy concerns before the data is submitted
613 0 : if (!dataHumanReadable.empty())
614 : {
615 0 : OsPath path = psLogDir() / OsPath("userreport_" + type + ".txt");
616 0 : std::ofstream stream(OsString(path), std::ofstream::trunc);
617 0 : if (stream)
618 : {
619 0 : debug_printf("FILES| UserReport written to '%s'\n", path.string8().c_str());
620 0 : stream << dataHumanReadable << std::endl;
621 0 : stream.close();
622 : }
623 : else
624 0 : debug_printf("FILES| Failed to write UserReport to '%s'\n", path.string8().c_str());
625 : }
626 :
627 : // If not initialised, discard the report
628 0 : if (!m_Worker)
629 0 : return;
630 :
631 : // Actual submit
632 0 : std::shared_ptr<CUserReport> report = std::make_shared<CUserReport>();
633 0 : report->m_Time = time(NULL);
634 0 : report->m_Type = type;
635 0 : report->m_Version = version;
636 0 : report->m_Data = data;
637 :
638 0 : m_Worker->Submit(report);
639 3 : }
|
