MythTV  master
netgrabbermanager.cpp
Go to the documentation of this file.
1 // qt
2 #include <QString>
3 #include <QCoreApplication>
4 #include <QFile>
5 #include <QDir>
6 
7 #include "mythdirs.h"
8 #include "mythcontext.h"
9 #include "mythsystemlegacy.h"
10 #include "exitcodes.h"
11 #include "mythdate.h"
12 #include "mythlogging.h"
13 #include "mythsystemlegacy.h"
14 
15 #include "netgrabbermanager.h"
16 #include "netutils.h"
17 
18 #define LOC QString("NetContent: ")
19 
20 using namespace std;
21 
22 // ---------------------------------------------------
23 
24 GrabberScript::GrabberScript(const QString& title, const QString& image,
25  const ArticleType &type, const QString& author,
26  const bool& search, const bool& tree,
27  const QString& description, const QString& commandline,
28  const double& version) :
29  MThread("GrabberScript"), m_lock(QMutex::Recursive)
30 {
31  m_title = title;
32  m_image = image;
33  m_type = type;
34  m_author = author;
35  m_search = search;
36  m_tree = tree;
37  m_description = description;
38  m_commandline = commandline;
40 }
41 
43 {
44  wait();
45 }
46 
48 {
49  RunProlog();
50  QMutexLocker locker(&m_lock);
51 
52  QString commandline = m_commandline;
53  MythSystemLegacy getTree(commandline, QStringList("-T"),
55  getTree.Run(900);
56  uint status = getTree.Wait();
57 
58  if( status == GENERIC_EXIT_CMD_NOT_FOUND )
59  LOG(VB_GENERAL, LOG_ERR, LOC +
60  QString("Internet Content Source %1 cannot run, file missing.")
61  .arg(m_title));
62  else if( status == GENERIC_EXIT_OK )
63  {
64  LOG(VB_GENERAL, LOG_INFO, LOC +
65  QString("Internet Content Source %1 completed download, "
66  "beginning processing...").arg(m_title));
67 
68  QByteArray result = getTree.ReadAll();
69 
70  QDomDocument domDoc;
71  domDoc.setContent(result, true);
72  QDomElement root = domDoc.documentElement();
73  QDomElement channel = root.firstChildElement("channel");
74 
76 
77  while (!channel.isNull())
78  {
79  parseDBTree(m_title, QString(), QString(), channel, GetType());
80  channel = channel.nextSiblingElement("channel");
81  }
83  LOG(VB_GENERAL, LOG_INFO, LOC +
84  QString("Internet Content Source %1 completed processing, "
85  "marking as updated.").arg(m_title));
86  }
87  else
88  LOG(VB_GENERAL, LOG_ERR, LOC +
89  QString("Internet Content Source %1 crashed while grabbing tree.")
90  .arg(m_title));
91 
92  emit finished();
93  RunEpilog();
94 }
95 
96 void GrabberScript::parseDBTree(const QString &feedtitle, const QString &path,
97  const QString &pathThumb, QDomElement& domElem,
98  const ArticleType &type)
99 {
100  QMutexLocker locker(&m_lock);
101 
102  Parse parse;
103  ResultItem::resultList articles;
104 
105  // File Handling
106  QDomElement fileitem = domElem.firstChildElement("item");
107  while (!fileitem.isNull())
108  { // Fill the article list...
109  articles.append(parse.ParseItem(fileitem));
110  fileitem = fileitem.nextSiblingElement("item");
111  }
112 
113  while (!articles.isEmpty())
114  { // Insert the articles in the DB...
115  insertTreeArticleInDB(feedtitle, path,
116  pathThumb, articles.takeFirst(), type);
117  }
118 
119  // Directory Handling
120  QDomElement diritem = domElem.firstChildElement("directory");
121  while (!diritem.isNull())
122  {
123  QDomElement subfolder = diritem;
124  QString dirname = diritem.attribute("name");
125  QString dirthumb = diritem.attribute("thumbnail");
126  dirname.replace("/", "|");
127  QString pathToUse;
128 
129  if (path.isEmpty())
130  pathToUse = dirname;
131  else
132  pathToUse = QString("%1/%2").arg(path).arg(dirname);
133 
134  parseDBTree(feedtitle,
135  pathToUse,
136  dirthumb,
137  subfolder,
138  type);
139  diritem = diritem.nextSiblingElement("directory");
140  }
141 }
142 
143 GrabberManager::GrabberManager() : m_lock(QMutex::Recursive)
144 {
146  "netsite.updateFreq", 24) * 3600 * 1000);
147  m_timer = new QTimer();
148  m_runningCount = 0;
149  m_refreshAll = false;
150  connect( m_timer, SIGNAL(timeout()),
151  this, SLOT(timeout()));
152 }
153 
155 {
156  delete m_timer;
157 }
158 
160 {
161  m_timer->start(m_updateFreq);
162 }
163 
165 {
166  m_timer->stop();
167 }
168 
170 {
172  if (m_refreshAll)
173  gdt->refreshAll();
174  gdt->start(QThread::LowPriority);
175 
176  m_timer->start(m_updateFreq);
177 }
178 
180 {
181  QMutexLocker locker(&m_lock);
182  doUpdate();
183 }
184 
186 {
187  m_refreshAll = true;
188 }
189 
191  MThread("GrabberDownload")
192 {
193  m_parent = parent;
194  m_refreshAll = false;
195 }
196 
198 {
199  cancel();
200  wait();
201 }
202 
204 {
205  m_mutex.lock();
206  qDeleteAll(m_scripts);
207  m_scripts.clear();
208  m_mutex.unlock();
209 }
210 
212 {
213  m_mutex.lock();
214  m_refreshAll = true;
215  if (!isRunning())
216  start();
217  m_mutex.unlock();
218 }
219 
221 {
222  RunProlog();
223 
225  uint updateFreq = gCoreContext->GetNumSetting(
226  "netsite.updateFreq", 24);
227 
228  while (m_scripts.count())
229  {
230  GrabberScript *script = m_scripts.takeFirst();
231  if (script && (needsUpdate(script, updateFreq) || m_refreshAll))
232  {
233  LOG(VB_GENERAL, LOG_INFO, LOC +
234  QString("Internet Content Source %1 Updating...")
235  .arg(script->GetTitle()));
236  script->run();
237  }
238  delete script;
239  }
240  emit finished();
241  if (m_parent)
242  QCoreApplication::postEvent(m_parent, new GrabberUpdateEvent());
243 
244  RunEpilog();
245 }
246 
248  : m_searchProcess(nullptr), m_numResults(0),
249  m_numReturned(0), m_numIndex(0)
250 {
251  m_videoList.clear();
252 }
253 
255 {
256  resetSearch();
257 
258  delete m_searchProcess;
259  m_searchProcess = nullptr;
260 }
261 
262 
263 void Search::executeSearch(const QString &script, const QString &query,
264  const QString &pagenum)
265 {
266  resetSearch();
267 
268  LOG(VB_GENERAL, LOG_DEBUG, "Search::executeSearch");
270 
271  connect(m_searchProcess, SIGNAL(finished()),
272  this, SLOT(slotProcessSearchExit()));
273  connect(m_searchProcess, SIGNAL(error(uint)),
274  this, SLOT(slotProcessSearchExit(uint)));
275 
276  QString cmd = script;
277 
278  QStringList args;
279 
280  if (!pagenum.isEmpty())
281  {
282  args.append(QString("-p"));
283  args.append(pagenum);
284  }
285 
286  args.append("-S");
287  QString term = query;
288  args.append(MythSystemLegacy::ShellEscape(term));
289 
290  LOG(VB_GENERAL, LOG_INFO, LOC +
291  QString("Internet Search Query: %1 %2").arg(cmd).arg(args.join(" ")));
292 
294  m_searchProcess->SetCommand(cmd, args, flags);
295  m_searchProcess->Run(40);
296 }
297 
299 {
300  qDeleteAll(m_videoList);
301  m_videoList.clear();
302 }
303 
305 {
306  Parse parse;
308 
309  QDomNodeList entries = m_document.elementsByTagName("channel");
310 
311  if (entries.count() == 0)
312  {
313  m_numResults = 0;
314  m_numReturned = 0;
315  m_numIndex = 0;
316  return;
317  }
318 
319  QDomNode itemNode = entries.item(0);
320 
321  QDomNode Node = itemNode.namedItem(QString("numresults"));
322  if (!Node.isNull())
323  {
324  m_numResults = Node.toElement().text().toUInt();
325  }
326  else
327  {
328  QDomNodeList count = m_document.elementsByTagName("item");
329 
330  if (count.count() == 0)
331  m_numResults = 0;
332  else
333  m_numResults = count.count();
334  }
335 
336  Node = itemNode.namedItem(QString("returned"));
337  if (!Node.isNull())
338  {
339  m_numReturned = Node.toElement().text().toUInt();
340  }
341  else
342  {
343  QDomNodeList items = m_document.elementsByTagName("item");
344 
345  if (items.count() == 0)
346  m_numReturned = 0;
347  else
348  m_numReturned = items.count();
349  }
350 
351  Node = itemNode.namedItem(QString("startindex"));
352  if (!Node.isNull())
353  {
354  m_numIndex = Node.toElement().text().toUInt();
355  }
356  else
357  m_numIndex = 0;
358 
359  Node = itemNode.namedItem(QString("nextpagetoken"));
360  if (!Node.isNull())
361  {
362  m_nextPageToken = Node.toElement().text();
363  }
364  else
365  m_nextPageToken = "";
366 
367  Node = itemNode.namedItem(QString("prevpagetoken"));
368  if (!Node.isNull())
369  {
370  m_prevPageToken = Node.toElement().text();
371  }
372  else
373  m_prevPageToken = "";
374 }
375 
377 {
378  if (exitcode == GENERIC_EXIT_TIMEOUT)
379  {
380  LOG(VB_GENERAL, LOG_WARNING, LOC + "Internet Search Timeout");
381 
382  if (m_searchProcess)
383  {
384  m_searchProcess->Term(true);
385  m_searchProcess->deleteLater();
386  m_searchProcess = nullptr;
387  }
388  emit searchTimedOut(this);
389  return;
390  }
391 
392  if (exitcode != GENERIC_EXIT_OK)
393  {
394  m_document.setContent(QString());
395  }
396  else
397  {
398  LOG(VB_GENERAL, LOG_INFO, LOC +
399  "Internet Search Successfully Completed");
400 
402  m_document.setContent(m_data, true);
403  }
404 
405  m_searchProcess->deleteLater();
406  m_searchProcess = nullptr;
407  emit finishedSearch(this);
408 }
409 
410 void Search::SetData(QByteArray data)
411 {
412  m_data = data;
413  m_document.setContent(m_data, true);
414 
415 }
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:216
void start(QThread::Priority=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:295
void Run(time_t timeout=0)
Runs a command inside the /bin/sh shell. Returns immediately.
uint m_numReturned
QString m_nextPageToken
#define LOC
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:46
const ArticleType & GetType() const
static QString ShellEscape(const QString &str)
allow access to stdout
Definition: mythsystem.h:39
bool insertTreeArticleInDB(const QString &feedtitle, const QString &path, const QString &paththumb, ResultItem *item, ArticleType type)
Definition: netutils.cpp:395
void run() override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
void SetData(QByteArray data)
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
#define GENERIC_EXIT_TIMEOUT
Process timed out.
Definition: exitcodes.h:24
static void error(const char *str,...)
Definition: vbi.c:41
void finished(void)
bool wait(unsigned long time=ULONG_MAX)
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:312
const QString & GetTitle() const
run child in the background
Definition: mythsystem.h:36
ResultItem::resultList parseRSS(QDomDocument domDoc)
Definition: rssparse.cpp:718
MythSystemLegacy * m_searchProcess
unsigned int uint
Definition: compat.h:140
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
ResultItem * ParseItem(const QDomElement &item) const
Definition: rssparse.cpp:741
GrabberScript::scriptList findAllDBTreeGrabbers()
Definition: netutils.cpp:115
QString m_prevPageToken
uint m_numResults
void SetCommand(const QString &, uint)
Resets an existing MythSystemLegacy object to a new command.
ResultItem::resultList m_videoList
bool markTreeUpdated(GrabberScript *script, QDateTime curTime)
Definition: netutils.cpp:310
void Term(bool force=false)
GrabberScript(const QString &title, const QString &image, const ArticleType &type, const QString &author, const bool &search, const bool &tree, const QString &description, const QString &commandline, const double &version)
run process through shell
Definition: mythsystem.h:41
#define GENERIC_EXIT_CMD_NOT_FOUND
Command not found.
Definition: exitcodes.h:12
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
QByteArray & ReadAll()
bool isRunning(void) const
Definition: mthread.cpp:275
QDomDocument m_document
void process(void)
GrabberDownloadThread * gdt
void run(void) override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
QList< GrabberScript * > m_scripts
uint Wait(time_t timeout=0)
bool needsUpdate(GrabberScript *script, uint updateFreq)
Definition: netutils.cpp:326
int GetNumSetting(const QString &key, int defaultval=0)
GrabberDownloadThread(QObject *parent)
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
void executeSearch(const QString &script, const QString &query, const QString &pagenum="")
bool clearTreeItems(const QString &feedcommand)
Definition: netutils.cpp:356
void slotProcessSearchExit(uint exitcode=0)
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:203
void parseDBTree(const QString &feedtitle, const QString &path, const QString &pathThumb, QDomElement &domElem, const ArticleType &type)
void searchTimedOut(Search *item)
void finishedSearch(Search *item)
QList< ResultItem * > resultList
Definition: rssparse.h:114
void resetSearch(void)
QByteArray m_data
enum ArticleTypes ArticleType
ArticleType m_type