MythTV  master
mythdownloadmanager.cpp
Go to the documentation of this file.
1 // qt
2 #include <QCoreApplication>
3 #include <QRunnable>
4 #include <QString>
5 #include <QByteArray>
6 #include <QFile>
7 #include <QDir>
8 #include <QNetworkCookieJar>
9 #include <QNetworkCookie>
10 #include <QAuthenticator>
11 #include <QTextStream>
12 #include <QNetworkProxy>
13 #include <QMutexLocker>
14 #include <QUrl>
15 #include <QTcpSocket>
16 
17 #include <cstdlib>
18 #include <unistd.h> // for usleep()
19 
20 // libmythbase
21 #include "compat.h"
22 #include "mythcorecontext.h"
23 #include "mythcoreutil.h"
24 #include "mthreadpool.h"
25 #include "mythdirs.h"
26 #include "mythevent.h"
27 #include "mythversion.h"
28 #include "remotefile.h"
29 #include "mythdate.h"
30 
31 #include "mythdownloadmanager.h"
32 #include "mythlogging.h"
33 #include "portchecker.h"
34 
35 using namespace std;
36 
37 #define LOC QString("DownloadManager: ")
38 #define CACHE_REDIRECTION_LIMIT 10
39 
41 QMutex dmCreateLock;
42 
47 {
48  public:
50  m_lastStat(MythDate::current())
51  {
52  qRegisterMetaType<QNetworkReply::NetworkError>("QNetworkReply::NetworkError");
53  }
54 
56  {
57  if (m_request)
58  delete m_request;
59  if (m_reply && m_processReply)
60  m_reply->deleteLater();
61  }
62 
63  bool IsDone(void)
64  {
65  QMutexLocker lock(&m_lock);
66  return m_done;
67  }
68 
69  void SetDone(bool done)
70  {
71  QMutexLocker lock(&m_lock);
72  m_done = done;
73  }
74 
75  QString m_url;
77  QNetworkRequest *m_request {nullptr};
78  QNetworkReply *m_reply {nullptr};
79  QString m_outFile;
80  QByteArray *m_data {nullptr};
81  QByteArray m_privData;
82  QObject *m_caller {nullptr};
83  MRequestType m_requestType {kRequestGet};
84  bool m_reload {false};
85  bool m_preferCache {false};
86  bool m_syncMode {false};
87  bool m_processReply {true};
88  bool m_done {false};
89  qint64 m_bytesReceived {0};
90  qint64 m_bytesTotal {0};
91  QDateTime m_lastStat;
92  AuthCallback m_authCallback {nullptr};
93  void *m_authArg {nullptr};
94  const QHash<QByteArray, QByteArray> *m_headers {nullptr};
95 
96  QNetworkReply::NetworkError m_errorCode {QNetworkReply::NoError};
97  QMutex m_lock;
98 };
99 
100 
105 class MythCookieJar : public QNetworkCookieJar
106 {
107  public:
108  MythCookieJar() = default;
109  void copyAllCookies(MythCookieJar &old);
110  void load(const QString &filename);
111  void save(const QString &filename);
112 };
113 
117 class RemoteFileDownloadThread : public QRunnable
118 {
119  public:
121  MythDownloadInfo *dlInfo) :
122  m_parent(parent),
123  m_dlInfo(dlInfo) {}
124 
125  void run() override // QRunnable
126  {
127  bool ok = false;
128 
129  RemoteFile *rf = new RemoteFile(m_dlInfo->m_url, false, false, 0);
130  ok = rf->SaveAs(m_dlInfo->m_privData);
131  delete rf;
132 
133  if (!ok)
134  m_dlInfo->m_errorCode = QNetworkReply::UnknownNetworkError;
135 
136  m_dlInfo->m_bytesReceived = m_dlInfo->m_privData.size();
137  m_dlInfo->m_bytesTotal = m_dlInfo->m_bytesReceived;
138 
139  m_parent->downloadFinished(m_dlInfo);
140  }
141 
142  private:
145 };
146 
150 {
151  if (downloadManager)
152  {
153  delete downloadManager;
154  downloadManager = nullptr;
155  }
156 }
157 
162 {
163  if (downloadManager)
164  return downloadManager;
165 
166  QMutexLocker locker(&dmCreateLock);
167 
168  // Check once more in case the download manager was created
169  // while we were securing the lock.
170  if (downloadManager)
171  return downloadManager;
172 
174  tmpDLM->start();
175  while (!tmpDLM->getQueueThread())
176  usleep(10000);
177 
178  tmpDLM->moveToThread(tmpDLM->getQueueThread());
179  tmpDLM->setRunThread();
180 
181  while (!tmpDLM->isRunning())
182  usleep(10000);
183 
184  downloadManager = tmpDLM;
185 
187 
188  return downloadManager;
189 }
190 
195  MThread("DownloadManager"),
196  m_manager(nullptr),
197  m_diskCache(nullptr),
198  m_proxy(nullptr),
199  m_infoLock(new QMutex(QMutex::Recursive)),
200  m_queueThread(nullptr),
201  m_runThread(false),
202  m_isRunning(false),
203  m_inCookieJar(nullptr)
204 {
205 }
206 
210 {
211  m_runThread = false;
212  m_queueWaitCond.wakeAll();
213 
214  wait();
215 
216  delete m_infoLock;
217 
218  if (m_inCookieJar)
219  delete m_inCookieJar;
220 }
221 
226 {
227  RunProlog();
228 
229  bool downloading = false;
230  bool itemsInQueue = false;
231  bool itemsInCancellationQueue = false;
232  bool waitAnyway = false;
233 
234  m_queueThread = QThread::currentThread();
235 
236  while (!m_runThread)
237  usleep(50000);
238 
239  m_manager = new QNetworkAccessManager(this);
240  m_diskCache = new QNetworkDiskCache(this);
241  m_proxy = new QNetworkProxy();
242  m_diskCache->setCacheDirectory(GetConfDir() + "/cache/" +
243  QCoreApplication::applicationName() + "-" +
245  m_manager->setCache(m_diskCache);
246 
247  // Set the proxy for the manager to be the application default proxy,
248  // which has already been setup
249  m_manager->setProxy(*m_proxy);
250 
251  // make sure the cookieJar is created in the same thread as the manager
252  // and set its parent to nullptr so it can be shared between managers
253  m_manager->cookieJar()->setParent(nullptr);
254 
255  QObject::connect(m_manager, SIGNAL(finished(QNetworkReply*)), this,
256  SLOT(downloadFinished(QNetworkReply*)));
257 
258  m_isRunning = true;
259  while (m_runThread)
260  {
261  if (m_inCookieJar)
262  {
263  LOG(VB_GENERAL, LOG_DEBUG, "Updating DLManager's Cookie Jar");
264  updateCookieJar();
265  }
266  m_infoLock->lock();
267  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items downloading %1").arg(m_downloadInfos.count()));
268  LOG(VB_FILE, LOG_DEBUG, LOC + QString("items queued %1").arg(m_downloadQueue.count()));
269  downloading = !m_downloadInfos.isEmpty();
270  itemsInCancellationQueue = !m_cancellationQueue.isEmpty();
271  m_infoLock->unlock();
272 
273  if (itemsInCancellationQueue)
274  {
276  }
277  if (downloading)
278  QCoreApplication::processEvents();
279 
280  m_infoLock->lock();
281  itemsInQueue = !m_downloadQueue.isEmpty();
282  m_infoLock->unlock();
283 
284  if (!itemsInQueue || waitAnyway)
285  {
286  waitAnyway = false;
287  m_queueWaitLock.lock();
288 
289  if (downloading)
290  {
291  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting 200ms"));
292  m_queueWaitCond.wait(&m_queueWaitLock, 200);
293  }
294  else
295  {
296  LOG(VB_FILE, LOG_DEBUG, LOC + QString("waiting for more items to download"));
298  }
299 
300  m_queueWaitLock.unlock();
301  }
302 
303  m_infoLock->lock();
304  if (!m_downloadQueue.isEmpty())
305  {
306  MythDownloadInfo *dlInfo = m_downloadQueue.front();
307 
308  m_downloadQueue.pop_front();
309 
310  if (!dlInfo)
311  {
312  m_infoLock->unlock();
313  continue;
314  }
315 
316  if (m_downloadInfos.contains(dlInfo->m_url))
317  {
318  // Push request to the end of the queue to let others process.
319  // If this is the only item in the queue, force the loop to
320  // wait a little.
321  if (m_downloadQueue.isEmpty())
322  waitAnyway = true;
323  m_downloadQueue.push_back(dlInfo);
324  m_infoLock->unlock();
325  continue;
326  }
327 
328  if (dlInfo->m_url.startsWith("myth://"))
329  downloadRemoteFile(dlInfo);
330  else
331  {
332  QMutexLocker cLock(&m_cookieLock);
333  downloadQNetworkRequest(dlInfo);
334  }
335 
336  m_downloadInfos[dlInfo->m_url] = dlInfo;
337  }
338  m_infoLock->unlock();
339  }
340  m_isRunning = false;
341 
342  RunEpilog();
343 }
344 
355 void MythDownloadManager::queueItem(const QString &url, QNetworkRequest *req,
356  const QString &dest, QByteArray *data,
357  QObject *caller, const MRequestType reqType,
358  const bool reload)
359 {
360  MythDownloadInfo *dlInfo = new MythDownloadInfo;
361 
362  dlInfo->m_url = url;
363  dlInfo->m_request = req;
364  dlInfo->m_outFile = dest;
365  dlInfo->m_data = data;
366  dlInfo->m_caller = caller;
367  dlInfo->m_requestType = reqType;
368  dlInfo->m_reload = reload;
369 
370  QMutexLocker locker(m_infoLock);
371  m_downloadQueue.push_back(dlInfo);
372  m_queueWaitCond.wakeAll();
373 }
374 
387 bool MythDownloadManager::processItem(const QString &url, QNetworkRequest *req,
388  const QString &dest, QByteArray *data,
389  const MRequestType reqType,
390  const bool reload,
391  AuthCallback authCallbackFn, void *authArg,
392  const QHash<QByteArray, QByteArray> *headers)
393 {
394  MythDownloadInfo *dlInfo = new MythDownloadInfo;
395 
396  dlInfo->m_url = url;
397  dlInfo->m_request = req;
398  dlInfo->m_outFile = dest;
399  dlInfo->m_data = data;
400  dlInfo->m_requestType = reqType;
401  dlInfo->m_reload = reload;
402  dlInfo->m_syncMode = true;
403  dlInfo->m_authCallback = authCallbackFn;
404  dlInfo->m_authArg = authArg;
405  dlInfo->m_headers = headers;
406 
407  return downloadNow(dlInfo, true);
408 }
409 
413 void MythDownloadManager::preCache(const QString &url)
414 {
415  LOG(VB_FILE, LOG_DEBUG, LOC + QString("preCache('%1')").arg(url));
416  queueItem(url, nullptr, QString(), nullptr, nullptr);
417 }
418 
425 void MythDownloadManager::queueDownload(const QString &url,
426  const QString &dest,
427  QObject *caller,
428  const bool reload)
429 {
430  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
431  .arg(url).arg(dest).arg((long long)caller));
432 
433  queueItem(url, nullptr, dest, nullptr, caller, kRequestGet, reload);
434 }
435 
441 void MythDownloadManager::queueDownload(QNetworkRequest *req,
442  QByteArray *data,
443  QObject *caller)
444 {
445  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queueDownload('%1', '%2', %3)")
446  .arg(req->url().toString()).arg((long long)data)
447  .arg((long long)caller));
448 
449  queueItem(req->url().toString(), req, QString(), data, caller,
450  kRequestGet,
451  (QNetworkRequest::AlwaysNetwork == req->attribute(
452  QNetworkRequest::CacheLoadControlAttribute,
453  QNetworkRequest::PreferNetwork).toInt()));
454 }
455 
462 bool MythDownloadManager::download(const QString &url, const QString &dest,
463  const bool reload)
464 {
465  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload);
466 }
467 
474 bool MythDownloadManager::download(const QString &url, QByteArray *data,
475  const bool reload)
476 {
477  return processItem(url, nullptr, QString(), data, kRequestGet, reload);
478 }
479 
486 QNetworkReply *MythDownloadManager::download(const QString &url,
487  const bool reload)
488 {
489  MythDownloadInfo *dlInfo = new MythDownloadInfo;
490  QNetworkReply *reply = nullptr;
491 
492  dlInfo->m_url = url;
493  dlInfo->m_reload = reload;
494  dlInfo->m_syncMode = true;
495  dlInfo->m_processReply = false;
496 
497  if (downloadNow(dlInfo, false))
498  {
499  if (dlInfo->m_reply)
500  {
501  reply = dlInfo->m_reply;
502  // prevent dlInfo dtor from deleting the reply
503  dlInfo->m_reply = nullptr;
504 
505  delete dlInfo;
506 
507  return reply;
508  }
509 
510  delete dlInfo;
511  }
512 
513  return nullptr;
514 }
515 
521 bool MythDownloadManager::download(QNetworkRequest *req, QByteArray *data)
522 {
523  LOG(VB_FILE, LOG_DEBUG, LOC + QString("download('%1', '%2')")
524  .arg(req->url().toString()).arg((long long)data));
525  return processItem(req->url().toString(), req, QString(), data,
526  kRequestGet,
527  (QNetworkRequest::AlwaysNetwork == req->attribute(
528  QNetworkRequest::CacheLoadControlAttribute,
529  QNetworkRequest::PreferNetwork).toInt()));
530 }
531 
541 bool MythDownloadManager::downloadAuth(const QString &url, const QString &dest,
542  const bool reload, AuthCallback authCallbackFn, void *authArg,
543  const QHash<QByteArray, QByteArray> *headers)
544 {
545  return processItem(url, nullptr, dest, nullptr, kRequestGet, reload, authCallbackFn,
546  authArg, headers);
547 }
548 
549 
555 void MythDownloadManager::queuePost(const QString &url,
556  QByteArray *data,
557  QObject *caller)
558 {
559  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
560  .arg(url).arg((long long)data));
561 
562  if (!data)
563  {
564  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
565  return;
566  }
567 
568  queueItem(url, nullptr, QString(), data, caller, kRequestPost);
569 }
570 
576 void MythDownloadManager::queuePost(QNetworkRequest *req,
577  QByteArray *data,
578  QObject *caller)
579 {
580  LOG(VB_FILE, LOG_DEBUG, LOC + QString("queuePost('%1', '%2')")
581  .arg(req->url().toString()).arg((long long)data));
582 
583  if (!data)
584  {
585  LOG(VB_GENERAL, LOG_ERR, LOC + "queuePost(), data is NULL!");
586  return;
587  }
588 
589  queueItem(req->url().toString(), req, QString(), data, caller,
590  kRequestPost,
591  (QNetworkRequest::AlwaysNetwork == req->attribute(
592  QNetworkRequest::CacheLoadControlAttribute,
593  QNetworkRequest::PreferNetwork).toInt()));
594 
595 }
596 
602 bool MythDownloadManager::post(const QString &url, QByteArray *data)
603 {
604  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
605  .arg(url).arg((long long)data));
606 
607  if (!data)
608  {
609  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
610  return false;
611  }
612 
613  return processItem(url, nullptr, QString(), data, kRequestPost);
614 }
615 
621 bool MythDownloadManager::post(QNetworkRequest *req, QByteArray *data)
622 {
623  LOG(VB_FILE, LOG_DEBUG, LOC + QString("post('%1', '%2')")
624  .arg(req->url().toString()).arg((long long)data));
625 
626  if (!data)
627  {
628  LOG(VB_GENERAL, LOG_ERR, LOC + "post(), data is NULL!");
629  return false;
630  }
631 
632  return processItem(req->url().toString(), req, QString(), data,
633  kRequestPost,
634  (QNetworkRequest::AlwaysNetwork == req->attribute(
635  QNetworkRequest::CacheLoadControlAttribute,
636  QNetworkRequest::PreferNetwork).toInt()));
637 
638 }
639 
648 bool MythDownloadManager::postAuth(const QString &url, QByteArray *data,
649  AuthCallback authCallbackFn, void *authArg,
650  const QHash<QByteArray, QByteArray> *headers)
651 {
652  LOG(VB_FILE, LOG_DEBUG, LOC + QString("postAuth('%1', '%2')")
653  .arg(url).arg((long long)data));
654 
655  if (!data)
656  {
657  LOG(VB_GENERAL, LOG_ERR, LOC + "postAuth(), data is NULL!");
658  return false;
659  }
660 
661  return processItem(url, nullptr, nullptr, data, kRequestPost, false, authCallbackFn,
662  authArg, headers);
663 }
664 
669 {
670  RemoteFileDownloadThread *dlThread =
671  new RemoteFileDownloadThread(this, dlInfo);
672  MThreadPool::globalInstance()->start(dlThread, "RemoteFileDownload");
673 }
674 
679 {
680  if (!dlInfo)
681  return;
682 
683  static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
684  QUrl qurl(dlInfo->m_url);
685  QNetworkRequest request;
686 
687  if (dlInfo->m_request)
688  {
689  request = *dlInfo->m_request;
690  delete dlInfo->m_request;
691  dlInfo->m_request = nullptr;
692  }
693  else
694  request.setUrl(qurl);
695 
696  if (dlInfo->m_reload)
697  {
698  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
699  QNetworkRequest::AlwaysNetwork);
700  }
701  else
702  {
703  // Prefer the in-cache item if one exists and it is less than 5 minutes
704  // old and it will not expire in the next 10 seconds
705  QDateTime now = MythDate::current();
706 
707  // Handle redirects, we want the metadata of the file headers
708  QString redirectLoc;
709  int limit = 0;
710  while (!(redirectLoc = getHeader(qurl, "Location")).isNull())
711  {
712  if (limit == CACHE_REDIRECTION_LIMIT)
713  {
714  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
715  "reached for %1")
716  .arg(qurl.toString()));
717  return;
718  }
719  qurl.setUrl(redirectLoc);
720  limit++;
721  }
722 
723  LOG(VB_NETWORK, LOG_DEBUG, QString("Checking cache for %1")
724  .arg(qurl.toString()));
725 
726  m_infoLock->lock();
727  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(qurl);
728  m_infoLock->unlock();
729  if ((urlData.isValid()) &&
730  ((!urlData.expirationDate().isValid()) ||
731  (urlData.expirationDate().toUTC().secsTo(now) < 10)))
732  {
733  QString dateString = getHeader(urlData, "Date");
734 
735  if (!dateString.isNull())
736  {
737  QDateTime loadDate =
738  MythDate::fromString(dateString, dateFormat);
739  loadDate.setTimeSpec(Qt::UTC);
740  if (loadDate.secsTo(now) <= 720)
741  {
742  dlInfo->m_preferCache = true;
743  LOG(VB_NETWORK, LOG_DEBUG, QString("Preferring cache for %1")
744  .arg(qurl.toString()));
745  }
746  }
747  }
748  }
749 
750  if (dlInfo->m_preferCache)
751  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
752  QNetworkRequest::PreferCache);
753 
754  if (!request.hasRawHeader("User-Agent"))
755  {
756  request.setRawHeader("User-Agent",
757  "MythTV v" MYTH_BINARY_VERSION
758  " MythDownloadManager");
759  }
760 
761  if (dlInfo->m_headers)
762  {
763  QHash<QByteArray, QByteArray>::const_iterator it =
764  dlInfo->m_headers->constBegin();
765  for ( ; it != dlInfo->m_headers->constEnd(); ++it )
766  {
767  if (!it.key().isEmpty() && !it.value().isEmpty())
768  {
769  request.setRawHeader(it.key(), it.value());
770  }
771  }
772  }
773 
774  switch (dlInfo->m_requestType)
775  {
776  case kRequestPost :
777  dlInfo->m_reply = m_manager->post(request, *dlInfo->m_data);
778  break;
779  case kRequestHead :
780  dlInfo->m_reply = m_manager->head(request);
781  break;
782  case kRequestGet :
783  default:
784  dlInfo->m_reply = m_manager->get(request);
785  break;
786  }
787 
788  m_downloadReplies[dlInfo->m_reply] = dlInfo;
789 
790  if (dlInfo->m_authCallback)
791  {
792  connect(m_manager, SIGNAL(authenticationRequired(QNetworkReply *,
793  QAuthenticator *)),
794  this, SLOT(authCallback(QNetworkReply *, QAuthenticator *)));
795  }
796 
797  connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)), this,
798  SLOT(downloadError(QNetworkReply::NetworkError)));
799  connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)),
800  this, SLOT(downloadProgress(qint64, qint64)));
801 }
802 
807 void MythDownloadManager::authCallback(QNetworkReply *reply,
808  QAuthenticator *authenticator)
809 {
810  if (!reply)
811  return;
812 
813  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
814 
815  if (!dlInfo)
816  return;
817 
818  if (dlInfo->m_authCallback)
819  {
820  LOG(VB_FILE, LOG_DEBUG, "Calling auth callback");
821  dlInfo->m_authCallback(reply, authenticator, dlInfo->m_authArg);
822  }
823 }
824 
832 {
833  if (!dlInfo)
834  return false;
835 
836  dlInfo->m_syncMode = true;
837 
838  // Special handling for link-local
839  // Not needed for Windows because windows does not need
840  // the scope id.
841 #ifndef _WIN32
842  if (dlInfo->m_url.startsWith("http://[fe80::",Qt::CaseInsensitive))
843  return downloadNowLinkLocal(dlInfo,deleteInfo);
844 #endif
845  m_infoLock->lock();
846  m_downloadQueue.push_back(dlInfo);
847  m_infoLock->unlock();
848  m_queueWaitCond.wakeAll();
849 
850  // timeout myth:// RemoteFile transfers 20 seconds from now
851  // timeout non-myth:// QNetworkAccessManager transfers 60 seconds after
852  // their last progress update
853  QDateTime startedAt = MythDate::current();
854  m_infoLock->lock();
855  while ((!dlInfo->IsDone()) &&
856  (dlInfo->m_errorCode == QNetworkReply::NoError) &&
857  (((!dlInfo->m_url.startsWith("myth://")) &&
858  (dlInfo->m_lastStat.secsTo(MythDate::current()) < 60)) ||
859  ((dlInfo->m_url.startsWith("myth://")) &&
860  (startedAt.secsTo(MythDate::current()) < 20))))
861  {
862  m_infoLock->unlock();
863  m_queueWaitLock.lock();
864  m_queueWaitCond.wait(&m_queueWaitLock, 200);
865  m_queueWaitLock.unlock();
866  m_infoLock->lock();
867  }
868  bool done = dlInfo->IsDone();
869  bool success =
870  done && (dlInfo->m_errorCode == QNetworkReply::NoError);
871 
872  if (!done)
873  {
874  dlInfo->m_data = nullptr; // Prevent downloadFinished() from updating
875  dlInfo->m_syncMode = false; // Let downloadFinished() cleanup for us
876  if ((dlInfo->m_reply) &&
877  (dlInfo->m_errorCode == QNetworkReply::NoError))
878  {
879  LOG(VB_FILE, LOG_DEBUG,
880  LOC + QString("Aborting download - lack of data transfer"));
881  dlInfo->m_reply->abort();
882  }
883  }
884  else if (deleteInfo)
885  delete dlInfo;
886 
887  m_infoLock->unlock();
888 
889  return success;
890 }
891 
892 #ifndef _WIN32
893 
913 bool MythDownloadManager::downloadNowLinkLocal(MythDownloadInfo *dlInfo, bool deleteInfo)
914 {
915  bool isOK = true;
916 
917  // Only certain features are supported here
918  if (dlInfo->m_authCallback || dlInfo->m_authArg)
919  {
920  LOG(VB_GENERAL, LOG_ERR, LOC +
921  QString("Unsupported authentication for %1").arg(dlInfo->m_url));
922  isOK = false;
923  }
924  if (!dlInfo->m_outFile.isEmpty())
925  {
926  LOG(VB_GENERAL, LOG_ERR, LOC +
927  QString("Unsupported File output %1 for %2")
928  .arg(dlInfo->m_outFile).arg(dlInfo->m_url));
929  isOK = false;
930  }
931 
932  if (!deleteInfo || dlInfo->m_requestType == kRequestHead)
933  {
934  // We do not have the ability to return a network reply in dlInfo
935  // so if we are asked to do that, return an error.
936  LOG(VB_GENERAL, LOG_ERR, LOC +
937  QString("Unsupported link-local operation %1")
938  .arg(dlInfo->m_url));
939  isOK = false;
940  }
941 
942  QUrl url(dlInfo->m_url);
943  QString host(url.host());
944  int port(url.port(80));
945  if (isOK && PortChecker::resolveLinkLocal(host, port))
946  {
947  QString reqType;
948  switch (dlInfo->m_requestType)
949  {
950  case kRequestPost :
951  reqType = "POST";
952  break;
953  case kRequestGet :
954  default:
955  reqType = "GET";
956  break;
957  }
958  QByteArray *aBuffer = dlInfo->m_data;
959  QHash<QByteArray, QByteArray> headers;
960  if (dlInfo->m_headers)
961  headers = *dlInfo->m_headers;
962  if (!headers.contains("User-Agent"))
963  headers.insert("User-Agent",
964  "MythDownloadManager v" MYTH_BINARY_VERSION);
965  headers.insert("Connection", "close");
966  headers.insert("Accept-Encoding", "identity");
967  if (aBuffer && !aBuffer->isEmpty())
968  headers.insert("Content-Length",
969  (QString::number(aBuffer->size())).toUtf8());
970  headers.insert("Host",
971  (url.host()+":"+QString::number(port)).toUtf8());
972 
973  QByteArray requestMessage;
974  QString path (url.path());
975  requestMessage.append("POST ");
976  requestMessage.append(path);
977  requestMessage.append(" HTTP/1.1\r\n");
978  QHashIterator<QByteArray, QByteArray> it(headers);
979  while (it.hasNext())
980  {
981  it.next();
982  requestMessage.append(it.key());
983  requestMessage.append(": ");
984  requestMessage.append(it.value());
985  requestMessage.append("\r\n");
986  }
987  requestMessage.append("\r\n");
988  if (aBuffer && !aBuffer->isEmpty())
989  {
990  requestMessage.append(*aBuffer);
991  }
992  QTcpSocket socket;
993  socket.connectToHost(host, port);
994  // QT Warning - this may not work on Windows
995  if (!socket.waitForConnected(5000))
996  isOK = false;
997  if (isOK)
998  isOK = (socket.write(requestMessage) > 0);
999  if (isOK)
1000  // QT Warning - this may not work on Windows
1001  isOK = socket.waitForDisconnected(5000);
1002  if (isOK)
1003  {
1004  *aBuffer = socket.readAll();
1005  // Find the start of the content
1006  QByteArray delim("\r\n\r\n");
1007  int delimLoc=aBuffer->indexOf(delim);
1008  if (delimLoc > -1)
1009  *aBuffer = aBuffer->right
1010  (aBuffer->size()-delimLoc-4);
1011  else
1012  isOK=false;
1013  }
1014  socket.close();
1015  }
1016  else
1017  isOK = false;
1018 
1019  if (deleteInfo)
1020  delete dlInfo;
1021 
1022  if (isOK)
1023  return true;
1024  else
1025  {
1026  LOG(VB_GENERAL, LOG_ERR, LOC + QString("Link Local request failed: %1")
1027  .arg(url.toString()));
1028  return false;
1029  }
1030 
1031 }
1032 #endif
1033 
1038 void MythDownloadManager::cancelDownload(const QString &url, bool block)
1039 {
1040  cancelDownload(QStringList(url), block);
1041 }
1042 
1047 void MythDownloadManager::cancelDownload(const QStringList &urls, bool block)
1048 {
1049  MythDownloadInfo *dlInfo;
1050 
1051  m_infoLock->lock();
1052  foreach (QString url, urls)
1053  {
1054  QMutableListIterator<MythDownloadInfo*> lit(m_downloadQueue);
1055  while (lit.hasNext())
1056  {
1057  lit.next();
1058  dlInfo = lit.value();
1059  if (dlInfo->m_url == url)
1060  {
1061  if (!m_cancellationQueue.contains(dlInfo))
1062  m_cancellationQueue.append(dlInfo);
1063  lit.remove();
1064  }
1065  }
1066 
1067  if (m_downloadInfos.contains(url))
1068  {
1069  dlInfo = m_downloadInfos[url];
1070 
1071  if (!m_cancellationQueue.contains(dlInfo))
1072  m_cancellationQueue.append(dlInfo);
1073 
1074  if (dlInfo->m_reply)
1075  m_downloadReplies.remove(dlInfo->m_reply);
1076 
1077  m_downloadInfos.remove(url);
1078  }
1079  }
1080  m_infoLock->unlock();
1081 
1082  if (QThread::currentThread() == this->thread())
1083  {
1084  downloadCanceled();
1085  return;
1086  }
1087 
1088  // wake-up running thread
1089  m_queueWaitCond.wakeAll();
1090 
1091  if (!block)
1092  return;
1093 
1094  while (!m_cancellationQueue.isEmpty())
1095  {
1096  usleep(50000); // re-test in another 50ms
1097  }
1098 }
1099 
1101 {
1102  QMutexLocker locker(m_infoLock);
1103  MythDownloadInfo *dlInfo;
1104 
1105  QMutableListIterator<MythDownloadInfo*> lit(m_cancellationQueue);
1106  while (lit.hasNext())
1107  {
1108  lit.next();
1109  dlInfo = lit.value();
1110  dlInfo->m_lock.lock();
1111 
1112  if (dlInfo->m_reply)
1113  {
1114  LOG(VB_FILE, LOG_DEBUG,
1115  LOC + QString("Aborting download - user request"));
1116  dlInfo->m_reply->abort();
1117  }
1118  lit.remove();
1119  if (dlInfo->m_done)
1120  {
1121  dlInfo->m_lock.unlock();
1122  continue;
1123  }
1124  dlInfo->m_errorCode = QNetworkReply::OperationCanceledError;
1125  dlInfo->m_done = true;
1126  dlInfo->m_lock.unlock();
1127  }
1128 }
1129 
1135 {
1136  QMutexLocker locker(m_infoLock);
1137  MythDownloadInfo *dlInfo;
1138 
1139  QList <MythDownloadInfo*>::iterator lit = m_downloadQueue.begin();
1140  for (; lit != m_downloadQueue.end(); ++lit)
1141  {
1142  dlInfo = *lit;
1143  if (dlInfo->m_caller == caller)
1144  {
1145  dlInfo->m_caller = nullptr;
1146  dlInfo->m_outFile = QString();
1147  dlInfo->m_data = nullptr;
1148  }
1149  }
1150 
1151  QMap <QString, MythDownloadInfo*>::iterator mit = m_downloadInfos.begin();
1152  for (; mit != m_downloadInfos.end(); ++mit)
1153  {
1154  dlInfo = mit.value();
1155  if (dlInfo->m_caller == caller)
1156  {
1157  dlInfo->m_caller = nullptr;
1158  dlInfo->m_outFile = QString();
1159  dlInfo->m_data = nullptr;
1160  }
1161  }
1162 }
1163 
1167 void MythDownloadManager::downloadError(QNetworkReply::NetworkError errorCode)
1168 {
1169  QNetworkReply *reply = (QNetworkReply*)sender();
1170 
1171  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadError %1 ")
1172  .arg(errorCode) + reply->errorString() );
1173 
1174  QMutexLocker locker(m_infoLock);
1175  if (!m_downloadReplies.contains(reply))
1176  {
1177  reply->deleteLater();
1178  return;
1179  }
1180 
1181  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1182 
1183  if (!dlInfo)
1184  return;
1185 
1186  dlInfo->m_errorCode = errorCode;
1187 }
1188 
1194 QUrl MythDownloadManager::redirectUrl(const QUrl& possibleRedirectUrl,
1195  const QUrl& oldRedirectUrl) const
1196 {
1197  LOG(VB_FILE, LOG_DEBUG, LOC + QString("redirectUrl()"));
1198  QUrl redirectUrl;
1199 
1200  if(!possibleRedirectUrl.isEmpty() && possibleRedirectUrl != oldRedirectUrl)
1201  redirectUrl = possibleRedirectUrl;
1202 
1203  return redirectUrl;
1204 }
1205 
1209 void MythDownloadManager::downloadFinished(QNetworkReply* reply)
1210 {
1211  LOG(VB_FILE, LOG_DEBUG, LOC + QString("downloadFinished(%1)")
1212  .arg((long long)reply));
1213 
1214  QMutexLocker locker(m_infoLock);
1215  if (!m_downloadReplies.contains(reply))
1216  {
1217  reply->deleteLater();
1218  return;
1219  }
1220 
1221  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1222 
1223  if (!dlInfo || !dlInfo->m_reply)
1224  return;
1225 
1226  downloadFinished(dlInfo);
1227 }
1228 
1233 {
1234  if (!dlInfo)
1235  return;
1236 
1237  int statusCode = -1;
1238  static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1239  QNetworkReply *reply = dlInfo->m_reply;
1240 
1241  if (reply)
1242  {
1243  QUrl possibleRedirectUrl =
1244  reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
1245 
1246  if (!possibleRedirectUrl.isEmpty() &&
1247  possibleRedirectUrl.isValid() &&
1248  possibleRedirectUrl.isRelative()) // Turn relative Url to absolute
1249  possibleRedirectUrl = reply->url().resolved(possibleRedirectUrl);
1250 
1251  dlInfo->m_redirectedTo =
1252  redirectUrl(possibleRedirectUrl, dlInfo->m_redirectedTo);
1253 
1254  QVariant status =
1255  reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
1256  if (status.isValid())
1257  statusCode = status.toInt();
1258  }
1259 
1260  if(reply && !dlInfo->m_redirectedTo.isEmpty() &&
1261  ((dlInfo->m_requestType != kRequestPost) ||
1262  (statusCode == 301 || statusCode == 302 ||
1263  statusCode == 303)))
1264  {
1265  LOG(VB_FILE, LOG_DEBUG, LOC +
1266  QString("downloadFinished(%1): Redirect: %2 -> %3")
1267  .arg((long long)dlInfo)
1268  .arg(reply->url().toString())
1269  .arg(dlInfo->m_redirectedTo.toString()));
1270 
1271  if (dlInfo->m_data)
1272  dlInfo->m_data->clear();
1273 
1274  dlInfo->m_bytesReceived = 0;
1275  dlInfo->m_bytesTotal = 0;
1276 
1277  QNetworkRequest request(dlInfo->m_redirectedTo);
1278 
1279  if (dlInfo->m_reload)
1280  {
1281  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1282  QNetworkRequest::AlwaysNetwork);
1283  }
1284  else if (dlInfo->m_preferCache)
1285  {
1286  request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
1287  QNetworkRequest::PreferCache);
1288  }
1289 
1290  request.setRawHeader("User-Agent",
1291  "MythDownloadManager v" MYTH_BINARY_VERSION);
1292 
1293  switch (dlInfo->m_requestType)
1294  {
1295  case kRequestHead :
1296  dlInfo->m_reply = m_manager->head(request);
1297  break;
1298  case kRequestGet :
1299  default:
1300  dlInfo->m_reply = m_manager->get(request);
1301  break;
1302  }
1303 
1304  m_downloadReplies[dlInfo->m_reply] = dlInfo;
1305 
1306  connect(dlInfo->m_reply, SIGNAL(error(QNetworkReply::NetworkError)),
1307  this, SLOT(downloadError(QNetworkReply::NetworkError)));
1308  connect(dlInfo->m_reply, SIGNAL(downloadProgress(qint64, qint64)),
1309  this, SLOT(downloadProgress(qint64, qint64)));
1310 
1311  m_downloadReplies.remove(reply);
1312  reply->deleteLater();
1313  }
1314  else
1315  {
1316  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): COMPLETE: %2")
1317  .arg((long long)dlInfo).arg(dlInfo->m_url));
1318 
1319  // HACK Insert a Date header into the cached metadata if one doesn't
1320  // already exist
1321  QUrl fileUrl = dlInfo->m_url;
1322  QString redirectLoc;
1323  int limit = 0;
1324  while (!(redirectLoc = getHeader(fileUrl, "Location")).isNull())
1325  {
1326  if (limit == CACHE_REDIRECTION_LIMIT)
1327  {
1328  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1329  "reached for %1")
1330  .arg(fileUrl.toString()));
1331  return;
1332  }
1333  fileUrl.setUrl(redirectLoc);
1334  limit++;
1335  }
1336 
1337  m_infoLock->lock();
1338  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(fileUrl);
1339  m_infoLock->unlock();
1340  if (getHeader(urlData, "Date").isNull())
1341  {
1342  QNetworkCacheMetaData::RawHeaderList headers = urlData.rawHeaders();
1343  QNetworkCacheMetaData::RawHeader newheader;
1344  QDateTime now = MythDate::current();
1345  newheader = QNetworkCacheMetaData::RawHeader("Date",
1346  now.toString(dateFormat).toLatin1());
1347  headers.append(newheader);
1348  urlData.setRawHeaders(headers);
1349  m_infoLock->lock();
1350  m_manager->cache()->updateMetaData(urlData);
1351  m_infoLock->unlock();
1352  }
1353  // End HACK
1354 
1355  dlInfo->m_redirectedTo.clear();
1356 
1357  int dataSize = -1;
1358 
1359  // If we downloaded via the QNetworkAccessManager
1360  // AND the caller isn't handling the reply directly
1361  if (reply && dlInfo->m_processReply)
1362  {
1363  bool append = (!dlInfo->m_syncMode && dlInfo->m_caller);
1364  QByteArray data = reply->readAll();
1365  dataSize = data.size();
1366 
1367  if (append)
1368  dlInfo->m_bytesReceived += dataSize;
1369  else
1370  dlInfo->m_bytesReceived = dataSize;
1371 
1372  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1373 
1374  if (dlInfo->m_data)
1375  {
1376  if (append)
1377  dlInfo->m_data->append(data);
1378  else
1379  *dlInfo->m_data = data;
1380  }
1381  else if (!dlInfo->m_outFile.isEmpty())
1382  {
1383  saveFile(dlInfo->m_outFile, data, append);
1384  }
1385  }
1386  else if (!reply) // If we downloaded via RemoteFile
1387  {
1388  if (dlInfo->m_data)
1389  {
1390  (*dlInfo->m_data) = dlInfo->m_privData;
1391  }
1392  else if (!dlInfo->m_outFile.isEmpty())
1393  {
1394  saveFile(dlInfo->m_outFile, dlInfo->m_privData);
1395  }
1396  dlInfo->m_bytesReceived += dataSize;
1397  dlInfo->m_bytesTotal = dlInfo->m_bytesReceived;
1398  }
1399  // else we downloaded via QNetworkAccessManager
1400  // AND the caller is handling the reply
1401 
1402  m_infoLock->lock();
1403  if (!m_downloadInfos.remove(dlInfo->m_url))
1404  LOG(VB_GENERAL, LOG_ERR, LOC +
1405  QString("ERROR download finished but failed to remove url: %1")
1406  .arg(dlInfo->m_url));
1407 
1408  if (reply)
1409  m_downloadReplies.remove(reply);
1410  m_infoLock->unlock();
1411 
1412  dlInfo->SetDone(true);
1413 
1414  if (!dlInfo->m_syncMode)
1415  {
1416  if (dlInfo->m_caller)
1417  {
1418  LOG(VB_FILE, LOG_DEBUG, QString("downloadFinished(%1): "
1419  "COMPLETE: %2, sending event to caller")
1420  .arg((long long)dlInfo).arg(dlInfo->m_url));
1421 
1422  QStringList args;
1423  args << dlInfo->m_url;
1424  args << dlInfo->m_outFile;
1425  args << QString::number(dlInfo->m_bytesTotal);
1426  // placeholder for error string
1427  args << (reply ? reply->errorString() : QString());
1428  args << QString::number((int)(reply ? reply->error() :
1429  dlInfo->m_errorCode));
1430 
1431  QCoreApplication::postEvent(dlInfo->m_caller,
1432  new MythEvent("DOWNLOAD_FILE FINISHED", args));
1433  }
1434 
1435  delete dlInfo;
1436  }
1437 
1438  m_queueWaitCond.wakeAll();
1439  }
1440 }
1441 
1447 void MythDownloadManager::downloadProgress(qint64 bytesReceived,
1448  qint64 bytesTotal)
1449 {
1450  QNetworkReply *reply = (QNetworkReply*)sender();
1451 
1452  LOG(VB_FILE, LOG_DEBUG, LOC +
1453  QString("downloadProgress(%1, %2) (for reply %3)")
1454  .arg(bytesReceived).arg(bytesTotal).arg((long long)reply));
1455 
1456  QMutexLocker locker(m_infoLock);
1457  if (!m_downloadReplies.contains(reply))
1458  return;
1459 
1460  MythDownloadInfo *dlInfo = m_downloadReplies[reply];
1461 
1462  if (!dlInfo)
1463  return;
1464 
1465  dlInfo->m_lastStat = MythDate::current();
1466 
1467  LOG(VB_FILE, LOG_DEBUG, LOC +
1468  QString("downloadProgress: %1 to %2 is at %3 of %4 bytes downloaded")
1469  .arg(dlInfo->m_url).arg(dlInfo->m_outFile)
1470  .arg(bytesReceived).arg(bytesTotal));
1471 
1472  if (!dlInfo->m_syncMode && dlInfo->m_caller)
1473  {
1474  LOG(VB_FILE, LOG_DEBUG, QString("downloadProgress(%1): "
1475  "sending event to caller")
1476  .arg(reply->url().toString()));
1477 
1478  bool appendToFile = (dlInfo->m_bytesReceived != 0);
1479  QByteArray data = reply->readAll();
1480  if (!dlInfo->m_outFile.isEmpty())
1481  saveFile(dlInfo->m_outFile, data, appendToFile);
1482 
1483  if (dlInfo->m_data)
1484  dlInfo->m_data->append(data);
1485 
1486  dlInfo->m_bytesReceived = bytesReceived;
1487  dlInfo->m_bytesTotal = bytesTotal;
1488 
1489  QStringList args;
1490  args << dlInfo->m_url;
1491  args << dlInfo->m_outFile;
1492  args << QString::number(bytesReceived);
1493  args << QString::number(bytesTotal);
1494 
1495  QCoreApplication::postEvent(dlInfo->m_caller,
1496  new MythEvent("DOWNLOAD_FILE UPDATE", args));
1497  }
1498 }
1499 
1507 bool MythDownloadManager::saveFile(const QString &outFile,
1508  const QByteArray &data,
1509  const bool append)
1510 {
1511  if (outFile.isEmpty() || !data.size())
1512  return false;
1513 
1514  QFile file(outFile);
1515  QFileInfo fileInfo(outFile);
1516  QDir qdir(fileInfo.absolutePath());
1517 
1518  if (!qdir.exists() && !qdir.mkpath(fileInfo.absolutePath()))
1519  {
1520  LOG(VB_GENERAL, LOG_ERR, QString("Failed to create: '%1'")
1521  .arg(fileInfo.absolutePath()));
1522  return false;
1523  }
1524 
1525  QIODevice::OpenMode mode = QIODevice::Unbuffered|QIODevice::WriteOnly;
1526  if (append)
1527  mode |= QIODevice::Append;
1528 
1529  if (!file.open(mode))
1530  {
1531  LOG(VB_GENERAL, LOG_ERR, QString("Failed to open: '%1'") .arg(outFile));
1532  return false;
1533  }
1534 
1535  off_t offset = 0;
1536  size_t remaining = data.size();
1537  uint failure_cnt = 0;
1538  while ((remaining > 0) && (failure_cnt < 5))
1539  {
1540  ssize_t written = file.write(data.data() + offset, remaining);
1541  if (written < 0)
1542  {
1543  failure_cnt++;
1544  usleep(50000);
1545  continue;
1546  }
1547 
1548  failure_cnt = 0;
1549  offset += written;
1550  remaining -= written;
1551  }
1552 
1553  if (remaining > 0)
1554  return false;
1555 
1556  return true;
1557 }
1558 
1563 QDateTime MythDownloadManager::GetLastModified(const QString &url)
1564 {
1565  // If the header has not expired and
1566  // the last modification date is less than 1 hours old or if
1567  // the cache object is less than 20 minutes old,
1568  // then use the cached header otherwise redownload the header
1569 
1570  static const char dateFormat[] = "ddd, dd MMM yyyy hh:mm:ss 'GMT'";
1571  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1')").arg(url));
1572  QDateTime result;
1573 
1574  QDateTime now = MythDate::current();
1575 
1576  QUrl cacheUrl = QUrl(url);
1577 
1578  // Deal with redirects, we want the cached data for the final url
1579  QString redirectLoc;
1580  int limit = 0;
1581  while (!(redirectLoc = getHeader(cacheUrl, "Location")).isNull())
1582  {
1583  if (limit == CACHE_REDIRECTION_LIMIT)
1584  {
1585  LOG(VB_GENERAL, LOG_WARNING, QString("Cache Redirection limit "
1586  "reached for %1")
1587  .arg(cacheUrl.toString()));
1588  return result;
1589  }
1590  cacheUrl.setUrl(redirectLoc);
1591  limit++;
1592  }
1593 
1594  m_infoLock->lock();
1595  QNetworkCacheMetaData urlData = m_manager->cache()->metaData(cacheUrl);
1596  m_infoLock->unlock();
1597 
1598  if (urlData.isValid() &&
1599  ((!urlData.expirationDate().isValid()) ||
1600  (urlData.expirationDate().secsTo(now) < 0)))
1601  {
1602  if (urlData.lastModified().toUTC().secsTo(now) <= 3600) // 1 Hour
1603  {
1604  result = urlData.lastModified().toUTC();
1605  }
1606  else
1607  {
1608  QString date = getHeader(urlData, "Date");
1609  if (!date.isNull())
1610  {
1611  QDateTime loadDate =
1612  MythDate::fromString(date, dateFormat);
1613  loadDate.setTimeSpec(Qt::UTC);
1614  if (loadDate.secsTo(now) <= 1200) // 20 Minutes
1615  {
1616  result = urlData.lastModified().toUTC();
1617  }
1618  }
1619  }
1620  }
1621 
1622  if (!result.isValid())
1623  {
1624  MythDownloadInfo *dlInfo = new MythDownloadInfo;
1625  dlInfo->m_url = url;
1626  dlInfo->m_syncMode = true;
1627  // Head request, we only want to inspect the headers
1628  dlInfo->m_requestType = kRequestHead;
1629 
1630  if (downloadNow(dlInfo, false))
1631  {
1632  if (dlInfo->m_reply)
1633  {
1634  QVariant lastMod =
1635  dlInfo->m_reply->header(
1636  QNetworkRequest::LastModifiedHeader);
1637  if (lastMod.isValid())
1638  result = lastMod.toDateTime().toUTC();
1639  }
1640 
1641  // downloadNow() will set a flag to trigger downloadFinished()
1642  // to delete the dlInfo if the download times out
1643  delete dlInfo;
1644  }
1645  }
1646 
1647  LOG(VB_FILE, LOG_DEBUG, LOC + QString("GetLastModified('%1'): Result %2")
1648  .arg(url).arg(result.toString()));
1649 
1650  return result;
1651 }
1652 
1653 
1657 void MythDownloadManager::loadCookieJar(const QString &filename)
1658 {
1659  QMutexLocker locker(&m_cookieLock);
1660 
1661  MythCookieJar *jar = new MythCookieJar;
1662  jar->load(filename);
1663  m_manager->setCookieJar(jar);
1664 }
1665 
1669 void MythDownloadManager::saveCookieJar(const QString &filename)
1670 {
1671  QMutexLocker locker(&m_cookieLock);
1672 
1673  if (!m_manager->cookieJar())
1674  return;
1675 
1676  MythCookieJar *jar = static_cast<MythCookieJar *>(m_manager->cookieJar());
1677  jar->save(filename);
1678 }
1679 
1680 void MythDownloadManager::setCookieJar(QNetworkCookieJar *cookieJar)
1681 {
1682  QMutexLocker locker(&m_cookieLock);
1683  m_manager->setCookieJar(cookieJar);
1684 }
1685 
1689 QNetworkCookieJar *MythDownloadManager::copyCookieJar(void)
1690 {
1691  QMutexLocker locker(&m_cookieLock);
1692 
1693  if (!m_manager->cookieJar())
1694  return nullptr;
1695 
1696  MythCookieJar *inJar = static_cast<MythCookieJar *>(m_manager->cookieJar());
1697  MythCookieJar *outJar = new MythCookieJar;
1698  outJar->copyAllCookies(*inJar);
1699 
1700  return static_cast<QNetworkCookieJar *>(outJar);
1701 }
1702 
1706 void MythDownloadManager::refreshCookieJar(QNetworkCookieJar *jar)
1707 {
1708  QMutexLocker locker(&m_cookieLock);
1709  if (m_inCookieJar)
1710  delete m_inCookieJar;
1711 
1712  MythCookieJar *inJar = static_cast<MythCookieJar *>(jar);
1713  MythCookieJar *outJar = new MythCookieJar;
1714  outJar->copyAllCookies(*inJar);
1715  m_inCookieJar = static_cast<QNetworkCookieJar *>(outJar);
1716 
1717  QMutexLocker locker2(&m_queueWaitLock);
1718  m_queueWaitCond.wakeAll();
1719 }
1720 
1724 {
1725  QMutexLocker locker(&m_cookieLock);
1726 
1727  MythCookieJar *inJar = static_cast<MythCookieJar *>(m_inCookieJar);
1728  MythCookieJar *outJar = new MythCookieJar;
1729  outJar->copyAllCookies(*inJar);
1730  m_manager->setCookieJar(static_cast<QNetworkCookieJar *>(outJar));
1731 
1732  delete m_inCookieJar;
1733  m_inCookieJar = nullptr;
1734 }
1735 
1736 QString MythDownloadManager::getHeader(const QUrl& url, const QString& header)
1737 {
1738  if (!m_manager || !m_manager->cache())
1739  return QString();
1740 
1741  m_infoLock->lock();
1742  QNetworkCacheMetaData metadata = m_manager->cache()->metaData(url);
1743  m_infoLock->unlock();
1744 
1745  return getHeader(metadata, header);
1746 }
1747 
1753 QString MythDownloadManager::getHeader(const QNetworkCacheMetaData &cacheData,
1754  const QString& header)
1755 {
1756  QNetworkCacheMetaData::RawHeaderList headers = cacheData.rawHeaders();
1757  bool found = false;
1758  QNetworkCacheMetaData::RawHeaderList::iterator it = headers.begin();
1759  for (; !found && it != headers.end(); ++it)
1760  {
1761  if (QString((*it).first) == header)
1762  {
1763  found = true;
1764  return QString((*it).second);
1765  }
1766  }
1767 
1768  return QString();
1769 }
1770 
1771 
1776 {
1777  const QList<QNetworkCookie> cookieList = old.allCookies();
1778  setAllCookies(cookieList);
1779 }
1780 
1784 void MythCookieJar::load(const QString &filename)
1785 {
1786  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: loading cookies from: %1").arg(filename));
1787 
1788  QFile f(filename);
1789  if (!f.open(QIODevice::ReadOnly))
1790  {
1791  LOG(VB_GENERAL, LOG_WARNING, QString("MythCookieJar::load() failed to open file for reading: %1").arg(filename));
1792  return;
1793  }
1794 
1795  QList<QNetworkCookie> cookieList;
1796  QTextStream stream(&f);
1797  while (!stream.atEnd())
1798  {
1799  QString cookie = stream.readLine();
1800  cookieList << QNetworkCookie::parseCookies(cookie.toLocal8Bit());
1801  }
1802 
1803  setAllCookies(cookieList);
1804 }
1805 
1809 void MythCookieJar::save(const QString &filename)
1810 {
1811  LOG(VB_GENERAL, LOG_DEBUG, QString("MythCookieJar: saving cookies to: %1").arg(filename));
1812 
1813  QFile f(filename);
1814  if (!f.open(QIODevice::WriteOnly))
1815  {
1816  LOG(VB_GENERAL, LOG_ERR, QString("MythCookieJar::save() failed to open file for writing: %1").arg(filename));
1817  return;
1818  }
1819 
1820  QList<QNetworkCookie> cookieList = allCookies();
1821  QTextStream stream(&f);
1822 
1823  for (QList<QNetworkCookie>::iterator it = cookieList.begin();
1824  it != cookieList.end(); ++it)
1825  {
1826  stream << (*it).toRawForm() << endl;
1827  }
1828 }
1829 
1830 
1831 /* vim: set expandtab tabstop=4 shiftwidth=4: */
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:216
QNetworkCookieJar * copyCookieJar(void)
Copy from one cookie jar to another.
QNetworkReply::NetworkError m_errorCode
void start(QThread::Priority=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:295
void save(const QString &filename)
Saves the cookie jar to a cookie file.
QList< MythDownloadInfo * > m_cancellationQueue
MythDownloadManager * m_parent
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:46
MRequestType
void queueItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, QObject *caller, const MRequestType reqType=kRequestGet, const bool reload=false)
Adds a request to the download queue.
QMap< QString, MythDownloadInfo * > m_downloadInfos
MythDownloadManager * downloadManager
void start(QRunnable *runnable, QString debugName, int priority=0)
void removeListener(QObject *caller)
Disconnects the specified caller from any existing MythDownloadInfo instances.
QNetworkDiskCache * m_diskCache
void queueDownload(const QString &url, const QString &dest, QObject *caller, const bool reload=false)
Adds a url to the download queue.
~MythDownloadManager()
Destructor for MythDownloadManager.
static void error(const char *str,...)
Definition: vbi.c:41
void ShutdownMythDownloadManager(void)
Deletes the running MythDownloadManager at program exit.
#define CACHE_REDIRECTION_LIMIT
A subclassed QNetworkCookieJar that allows for reading and writing cookie files that contain raw form...
void downloadQNetworkRequest(MythDownloadInfo *dlInfo)
Downloads a QNetworkRequest via the QNetworkAccessManager.
AuthCallback m_authCallback
bool wait(unsigned long time=ULONG_MAX)
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:312
bool saveFile(const QString &outFile, const QByteArray &data, const bool append=false)
Saves a QByteArray of data to a given filename.
QNetworkReply * m_reply
QList< MythDownloadInfo * > m_downloadQueue
unsigned int uint
Definition: compat.h:140
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
MythDownloadManager()
Constructor for MythDownloadManager.
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
Slot to process download update events.
const QHash< QByteArray, QByteArray > * m_headers
friend class RemoteFileDownloadThread
bool postAuth(const QString &url, QByteArray *data, AuthCallback authCallback, void *authArg, const QHash< QByteArray, QByteArray > *headers=nullptr)
Posts data to a url via the QNetworkAccessManager.
MRequestType m_requestType
void run(void) override
Runs a loop to process incoming download requests and triggers download events to be processed.
void downloadFinished(QNetworkReply *reply)
Slot to process download finished events.
#define off_t
static bool resolveLinkLocal(QString &host, int port, int timeLimit=30000)
Convenience method to resolve link-local address.
QString GetConfDir(void)
Definition: mythdirs.cpp:224
void authCallback(QNetworkReply *reply, QAuthenticator *authenticator)
Signal handler for authentication requests.
QNetworkAccessManager * m_manager
This class is used as a container for messages.
Definition: mythevent.h:15
VERBOSE_PREAMBLE false
Definition: verbosedefs.h:85
QUrl redirectUrl(const QUrl &possibleRedirectUrl, const QUrl &oldRedirectUrl) const
Checks whether we were redirected to the given URL.
void setCookieJar(QNetworkCookieJar *cookieJar)
void copyAllCookies(MythCookieJar &old)
Copies all cookies from one MythCookieJar to another.
RemoteFileDownloadThread(MythDownloadManager *parent, MythDownloadInfo *dlInfo)
QWaitCondition m_queueWaitCond
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
void queuePost(const QString &url, QByteArray *data, QObject *caller)
Queues a post to a URL via the QNetworkAccessManager.
void refreshCookieJar(QNetworkCookieJar *jar)
Refresh the temporary cookie jar from another cookie jar.
MythDownloadManager * GetMythDownloadManager(void)
Gets the pointer to the MythDownloadManager singleton.
bool download(const QString &url, const QString &dest, const bool reload=false)
Downloads a URL to a file in blocking mode.
QDateTime GetLastModified(const QString &url)
Gets the Last Modified timestamp for a URI.
QNetworkCookieJar * m_inCookieJar
void downloadError(QNetworkReply::NetworkError errorCode)
Slot to process download error events.
#define LOC
void(* AuthCallback)(QNetworkReply *, QAuthenticator *, void *)
QString getHeader(const QUrl &url, const QString &header)
static MThreadPool * globalInstance(void)
bool SaveAs(QByteArray &data)
void downloadRemoteFile(MythDownloadInfo *dlInfo)
Triggers a myth:// URI download in the background via RemoteFile.
void SetDone(bool done)
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
QMap< QNetworkReply *, MythDownloadInfo * > m_downloadReplies
QNetworkRequest * m_request
bool downloadAuth(const QString &url, const QString &dest, const bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr)
Downloads a URL to a file in blocking mode.
void updateCookieJar(void)
Update the cookie jar from the temporary cookie jar.
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:203
bool downloadNow(MythDownloadInfo *dlInfo, bool deleteInfo=true)
Download helper for download() blocking methods.
#define MYTH_BINARY_VERSION
Update this whenever the plug-in ABI changes.
Definition: mythversion.h:16
void cancelDownload(const QString &url, bool block=true)
Cancel a queued or current download.
bool post(const QString &url, QByteArray *data)
Posts data to a url via the QNetworkAccessManager.
bool processItem(const QString &url, QNetworkRequest *req, const QString &dest, QByteArray *data, const MRequestType reqType=kRequestGet, const bool reload=false, AuthCallback authCallback=nullptr, void *authArg=nullptr, const QHash< QByteArray, QByteArray > *headers=nullptr)
Processes a network request immediately and waits for a response.
static void usleep(unsigned long time)
Definition: mthread.cpp:349
QThread * getQueueThread(void)
QMutex dmCreateLock
QString GetHostName(void)
void saveCookieJar(const QString &filename)
Saves the cookie jar to a cookie file.
QDateTime fromString(const QString &dtstr)
Converts kFilename && kISODate formats to QDateTime.
Definition: mythdate.cpp:30
void loadCookieJar(const QString &filename)
Loads the cookie jar from a cookie file.
void load(const QString &filename)
Loads the cookie jar from a cookie file.
void preCache(const QString &url)
Downloads a URL but doesn't store the resulting data anywhere.