MythTV  master
filldata.cpp
Go to the documentation of this file.
1 // POSIX headers
2 #include <unistd.h>
3 
4 // Std C headers
5 #include <cstdlib>
6 #include <ctime>
7 
8 // C++ headers
9 #include <fstream>
10 using namespace std;
11 
12 // Qt headers
13 #include <QTextStream>
14 #include <QDateTime>
15 #include <QFile>
16 #include <QList>
17 #include <QMap>
18 #include <QDir>
19 
20 // MythTV headers
21 #include "mythmiscutil.h"
22 #include "exitcodes.h"
23 #include "mythlogging.h"
24 #include "mythdbcon.h"
25 #include "compat.h"
26 #include "mythdate.h"
27 #include "mythdirs.h"
28 #include "mythdb.h"
29 #include "mythsystemlegacy.h"
30 #include "videosource.h" // for is_grabber..
31 #include "mythcorecontext.h"
32 
33 // filldata headers
34 #include "filldata.h"
35 
36 #define LOC QString("FillData: ")
37 #define LOC_WARN QString("FillData, Warning: ")
38 #define LOC_ERR QString("FillData, Error: ")
39 
40 bool updateLastRunEnd(void)
41 {
42  QDateTime qdtNow = MythDate::current();
43  return gCoreContext->SaveSettingOnHost("mythfilldatabaseLastRunEnd",
44  qdtNow.toString(Qt::ISODate),
45  nullptr);
46 }
47 
49 {
51  QDateTime qdtNow = MythDate::current();
52  return gCoreContext->SaveSettingOnHost("mythfilldatabaseLastRunStart",
53  qdtNow.toString(Qt::ISODate),
54  nullptr);
55 }
56 
57 bool updateLastRunStatus(QString &status)
58 {
59  return gCoreContext->SaveSettingOnHost("mythfilldatabaseLastRunStatus",
60  status,
61  nullptr);
62 }
63 
65 {
66  QDateTime nextSuggestedTime = MythDate::current().addDays(1);
67  return gCoreContext->SaveSettingOnHost("MythFillSuggestedRunTime",
68  nextSuggestedTime.toString(Qt::ISODate),
69  nullptr);
70 }
71 
72 void FillData::SetRefresh(int day, bool set)
73 {
74  if (kRefreshClear == day)
75  {
76  refresh_all = set;
77  refresh_day.clear();
78  }
79  else if (kRefreshAll == day)
80  {
81  refresh_all = set;
82  }
83  else
84  {
85  refresh_day[(uint)day] = set;
86  }
87 }
88 
89 // DataDirect stuff
91 {
93 
94  bool insert_channels = chan_data.insert_chan(source.id);
95  int new_channels = DataDirectProcessor::UpdateChannelsSafe(
96  source.id, insert_channels, chan_data.m_filterNewChannels);
97 
98  // User must pass "--do-channel-updates" for these updates
99  if (chan_data.m_channelUpdates)
100  {
102  source.id, chan_data.m_filterNewChannels);
103  }
104  // TODO delete any channels which no longer exist in listings source
105 
106  // Unselect channels not in users lineup for DVB, HDTV
107  if (!insert_channels && (new_channels > 0) &&
109  {
110  bool ok0 = (logged_in == source.userid);
111  bool ok1 = (raw_lineup == source.id);
112  if (!ok0)
113  {
114  LOG(VB_GENERAL, LOG_INFO,
115  "Grabbing login cookies for listing update");
116  ok0 = ddprocessor.GrabLoginCookiesAndLineups();
117  }
118  if (ok0 && !ok1)
119  {
120  LOG(VB_GENERAL, LOG_INFO, "Grabbing listing for listing update");
121  ok1 = ddprocessor.GrabLineupForModify(source.lineupid);
122  }
123  if (ok1)
124  {
125  ddprocessor.UpdateListings(source.id);
126  LOG(VB_GENERAL, LOG_INFO,
127  QString("Removed %1 channel(s) from lineup.")
128  .arg(new_channels));
129  }
130  }
131 }
132 
134 {
135  if (get_datadirect_provider(source.xmltvgrabber) >= 0)
136  {
137  ddprocessor.SetListingsProvider(
139  }
140  else
141  {
142  LOG(VB_GENERAL, LOG_ERR, LOC +
143  "We only support DataDirectUpdateChannels with "
144  "TMS Labs and Schedules Direct.");
145  return false;
146  }
147 
148  ddprocessor.SetUserID(source.userid);
149  ddprocessor.SetPassword(source.password);
150 
151  bool ok = true;
152  if (!is_grabber_labs(source.xmltvgrabber))
153  {
154  ok = ddprocessor.GrabLineupsOnly();
155  }
156  else
157  {
158  ok = ddprocessor.GrabFullLineup(
159  source.lineupid, true, chan_data.insert_chan(source.id)/*only sel*/);
160  logged_in = source.userid;
161  raw_lineup = source.id;
162  }
163 
164  if (ok)
165  DataDirectStationUpdate(source);
166 
167  return ok;
168 }
169 
170 bool FillData::GrabDDData(Source source, int poffset,
171  QDate pdate, int ddSource)
172 {
173  if (source.dd_dups.empty())
174  ddprocessor.SetCacheData(false);
175  else
176  {
177  LOG(VB_GENERAL, LOG_INFO,
178  QString("This DataDirect listings source is "
179  "shared by %1 MythTV lineups")
180  .arg(source.dd_dups.size()+1));
181  if (source.id > source.dd_dups[0])
182  {
183  LOG(VB_GENERAL, LOG_NOTICE,
184  "We should use cached data for this one");
185  }
186  else if (source.id < source.dd_dups[0])
187  {
188  LOG(VB_GENERAL, LOG_NOTICE,
189  "We should keep data around after this one");
190  }
191  ddprocessor.SetCacheData(true);
192  }
193 
194  ddprocessor.SetListingsProvider(ddSource);
195  ddprocessor.SetUserID(source.userid);
196  ddprocessor.SetPassword(source.password);
197 
198  bool needtoretrieve = true;
199 
200  if (source.userid != lastdduserid)
201  dddataretrieved = false;
202 
203  if (dd_grab_all && dddataretrieved)
204  needtoretrieve = false;
205 
206  QString status = QObject::tr("currently running.");
207 
209 
210  if (needtoretrieve)
211  {
212  LOG(VB_GENERAL, LOG_INFO, "Retrieving datadirect data.");
213  if (dd_grab_all)
214  {
215  LOG(VB_GENERAL, LOG_INFO, "Grabbing ALL available data.");
216  if (!ddprocessor.GrabAllData())
217  {
218  LOG(VB_GENERAL, LOG_ERR, "Encountered error in grabbing data.");
219  return false;
220  }
221  }
222  else
223  {
224  QDateTime fromdatetime =
225  QDateTime(pdate, QTime(0,0), Qt::UTC).addDays(poffset);
226  QDateTime todatetime = fromdatetime.addDays(1);
227 
228  LOG(VB_GENERAL, LOG_INFO, QString("Grabbing data for %1 offset %2")
229  .arg(pdate.toString())
230  .arg(poffset));
231  LOG(VB_GENERAL, LOG_INFO, QString("From %1 to %2 (UTC)")
232  .arg(fromdatetime.toString(Qt::ISODate))
233  .arg(todatetime.toString(Qt::ISODate)));
234 
235  if (!ddprocessor.GrabData(fromdatetime, todatetime))
236  {
237  LOG(VB_GENERAL, LOG_ERR, "Encountered error in grabbing data.");
238  return false;
239  }
240  }
241 
242  dddataretrieved = true;
243  lastdduserid = source.userid;
244  }
245  else
246  {
247  LOG(VB_GENERAL, LOG_INFO,
248  "Using existing grabbed data in temp tables.");
249  }
250 
251  LOG(VB_GENERAL, LOG_INFO,
252  QString("Grab complete. Actual data from %1 to %2 (UTC)")
253  .arg(ddprocessor.GetDDProgramsStartAt().toString(Qt::ISODate))
254  .arg(ddprocessor.GetDDProgramsEndAt().toString(Qt::ISODate)));
255 
257 
258  LOG(VB_GENERAL, LOG_INFO, "Main temp tables populated.");
259  if (!channel_update_run)
260  {
261  LOG(VB_GENERAL, LOG_INFO, "Updating MythTV channels.");
262  DataDirectStationUpdate(source);
263  LOG(VB_GENERAL, LOG_INFO, "Channels updated.");
264  channel_update_run = true;
265  }
266 
267 #if 0
268  LOG(VB_GENERAL, LOG_INFO, "Creating program view table...");
269 #endif
271 #if 0
272  LOG(VB_GENERAL, LOG_INFO, "Finished creating program view table...");
273 #endif
274 
275  MSqlQuery query(MSqlQuery::DDCon());
276  query.prepare("SELECT count(*) from dd_v_program;");
277  if (query.exec() && query.next())
278  {
279  if (query.value(0).toInt() < 1)
280  {
281  LOG(VB_GENERAL, LOG_INFO, "Did not find any new program data.");
282  return false;
283  }
284  }
285  else
286  {
287  LOG(VB_GENERAL, LOG_ERR, "Failed testing program view table.");
288  return false;
289  }
290 
291  LOG(VB_GENERAL, LOG_INFO, "Clearing data for source.");
292  QDateTime from = ddprocessor.GetDDProgramsStartAt();
293  QDateTime to = ddprocessor.GetDDProgramsEndAt();
294 
295  LOG(VB_GENERAL, LOG_INFO, QString("Clearing from %1 to %2 (localtime)")
296  .arg(from.toLocalTime().toString(Qt::ISODate))
297  .arg(to.toLocalTime().toString(Qt::ISODate)));
298  ProgramData::ClearDataBySource(source.id, from, to, true);
299  LOG(VB_GENERAL, LOG_INFO, "Data for source cleared.");
300 
301  LOG(VB_GENERAL, LOG_INFO, "Updating programs.");
303  LOG(VB_GENERAL, LOG_INFO, "Program table update complete.");
304 
305  return true;
306 }
307 
308 // XMLTV stuff
309 bool FillData::GrabDataFromFile(int id, QString &filename)
310 {
311  ChannelInfoList chanlist;
312  QMap<QString, QList<ProgInfo> > proglist;
313 
314  xmltv_parser.lateInit();
315  if (!xmltv_parser.parseFile(filename, &chanlist, &proglist))
316  return false;
317 
318  chan_data.handleChannels(id, &chanlist);
319  if (proglist.count() == 0)
320  {
321  LOG(VB_GENERAL, LOG_INFO, "No programs found in data.");
322  endofdata = true;
323  }
324  else
325  {
326  prog_data.HandlePrograms(id, proglist);
327  }
328  return true;
329 }
330 
331 bool FillData::GrabData(Source source, int offset, QDate *qCurrentDate)
332 {
333  QString xmltv_grabber = source.xmltvgrabber;
334 
335  int dd_provider = get_datadirect_provider(xmltv_grabber);
336  if (dd_provider >= 0)
337  {
338  if (!GrabDDData(source, offset, *qCurrentDate, dd_provider))
339  {
340  QStringList errors = ddprocessor.GetFatalErrors();
341  for (int i = 0; i < errors.size(); i++)
342  fatalErrors.push_back(errors[i]);
343  return false;
344  }
345  return true;
346  }
347 
348  const QString templatename = "/tmp/mythXXXXXX";
349  const QString tempfilename = createTempFile(templatename);
350  if (templatename == tempfilename)
351  {
352  fatalErrors.push_back("Failed to create temporary file.");
353  return false;
354  }
355 
356  QString filename = QString(tempfilename);
357 
358  QString home = QDir::homePath();
359 
360  QString configfile;
361 
362  MSqlQuery query1(MSqlQuery::InitCon());
363  query1.prepare("SELECT configpath FROM videosource"
364  " WHERE sourceid = :ID AND configpath IS NOT NULL");
365  query1.bindValue(":ID", source.id);
366  if (!query1.exec())
367  {
368  MythDB::DBError("FillData::grabData", query1);
369  return false;
370  }
371 
372  if (query1.next())
373  configfile = query1.value(0).toString();
374  else
375  configfile = QString("%1/%2.xmltv").arg(GetConfDir())
376  .arg(source.name);
377 
378  LOG(VB_GENERAL, LOG_INFO,
379  QString("XMLTV config file is: %1").arg(configfile));
380 
381  QString command = QString("nice %1 --config-file '%2' --output %3")
382  .arg(xmltv_grabber).arg(configfile).arg(filename);
383 
384 
385  if (source.xmltvgrabber_prefmethod != "allatonce" || no_allatonce)
386  {
387  // XMLTV Docs don't recommend grabbing one day at a
388  // time but the current MythTV code is heavily geared
389  // that way so until it is re-written behave as
390  // we always have done.
391  command += QString(" --days 1 --offset %1").arg(offset);
392  }
393 
394  if (!VERBOSE_LEVEL_CHECK(VB_XMLTV, LOG_ANY))
395  command += " --quiet";
396 
397  // Append additional arguments passed to mythfilldatabase
398  // using --graboptions
399  if (!graboptions.isEmpty())
400  {
401  command += graboptions;
402  LOG(VB_XMLTV, LOG_INFO,
403  QString("Using graboptions: %1").arg(graboptions));
404  }
405 
406  QString status = QObject::tr("currently running.");
407 
409  updateLastRunStatus(status);
410 
411  LOG(VB_XMLTV, LOG_INFO, QString("Grabber Command: %1").arg(command));
412 
413  LOG(VB_XMLTV, LOG_INFO,
414  "----------------- Start of XMLTV output -----------------");
415 
416  unsigned int systemcall_status;
417 
418  systemcall_status = myth_system(command, kMSRunShell);
419  bool succeeded = (systemcall_status == GENERIC_EXIT_OK);
420 
421  LOG(VB_XMLTV, LOG_INFO,
422  "------------------ End of XMLTV output ------------------");
423 
425 
426  status = QObject::tr("Successful.");
427 
428  if (!succeeded)
429  {
430  if (systemcall_status == GENERIC_EXIT_KILLED)
431  {
432  interrupted = true;
433  status = QObject::tr("FAILED: XMLTV grabber ran but was interrupted.");
434  }
435  else
436  {
437  status = QObject::tr("FAILED: XMLTV grabber returned error code %1.")
438  .arg(systemcall_status);
439  LOG(VB_GENERAL, LOG_ERR, LOC +
440  QString("XMLTV grabber returned error code %1")
441  .arg(systemcall_status));
442  }
443  }
444 
445  updateLastRunStatus(status);
446 
447  succeeded &= GrabDataFromFile(source.id, filename);
448 
449  QFile thefile(filename);
450  thefile.remove();
451 
452  return succeeded;
453 }
454 
456  int id, int offset, const QString &filename,
457  const QString &lineupid, QDate *qCurrentDate)
458 {
459  QDate *currentd = qCurrentDate;
460  QDate qcd = MythDate::current().date();
461  if (!currentd)
462  currentd = &qcd;
463 
464  ddprocessor.SetInputFile(filename);
465  Source s;
466  s.id = id;
467  s.xmltvgrabber = "datadirect";
468  s.userid = "fromfile";
469  s.password = "fromfile";
470  s.lineupid = lineupid;
471 
472  return GrabData(s, offset, currentd);
473 }
474 
475 
481 bool FillData::Run(SourceList &sourcelist)
482 {
483  SourceList::iterator it;
484  SourceList::iterator it2;
485 
486  QString status, querystr;
487  MSqlQuery query(MSqlQuery::InitCon());
488  QDateTime GuideDataBefore, GuideDataAfter;
489  int failures = 0;
490  int externally_handled = 0;
491  int total_sources = sourcelist.size();
492  int source_channels = 0;
493 
494  QString sidStr = QString("Updating source #%1 (%2) with grabber %3");
495 
496  need_post_grab_proc = false;
497  int nonewdata = 0;
498  bool has_dd_source = false;
499 
500  // find all DataDirect duplicates, so we only data download once.
501  for (it = sourcelist.begin(); it != sourcelist.end(); ++it)
502  {
503  if (!is_grabber_datadirect((*it).xmltvgrabber))
504  continue;
505 
506  has_dd_source = true;
507  for (it2 = sourcelist.begin(); it2 != sourcelist.end(); ++it2)
508  {
509  if (((*it).id != (*it2).id) &&
510  ((*it).xmltvgrabber == (*it2).xmltvgrabber) &&
511  ((*it).userid == (*it2).userid) &&
512  ((*it).password == (*it2).password))
513  {
514  (*it).dd_dups.push_back((*it2).id);
515  }
516  }
517  }
518  if (has_dd_source)
519  ddprocessor.CreateTempDirectory();
520 
521  for (it = sourcelist.begin(); it != sourcelist.end(); ++it)
522  {
523  if (!fatalErrors.empty())
524  break;
525 
526  query.prepare("SELECT MAX(endtime) FROM program p LEFT JOIN channel c "
527  "ON p.chanid=c.chanid WHERE c.sourceid= :SRCID "
528  "AND manualid = 0 AND c.xmltvid != '';");
529  query.bindValue(":SRCID", (*it).id);
530 
531  if (query.exec() && query.next())
532  {
533  if (!query.isNull(0))
534  GuideDataBefore =
535  MythDate::fromString(query.value(0).toString());
536  }
537 
538  channel_update_run = false;
539  endofdata = false;
540 
541  QString xmltv_grabber = (*it).xmltvgrabber;
542 
543  if (xmltv_grabber == "eitonly")
544  {
545  LOG(VB_GENERAL, LOG_INFO,
546  QString("Source %1 configured to use only the "
547  "broadcasted guide data. Skipping.") .arg((*it).id));
548 
549  externally_handled++;
552  continue;
553  }
554  else if (xmltv_grabber.trimmed().isEmpty() ||
555  xmltv_grabber == "/bin/true" ||
556  xmltv_grabber == "none")
557  {
558  LOG(VB_GENERAL, LOG_INFO,
559  QString("Source %1 configured with no grabber. Nothing to do.")
560  .arg((*it).id));
561 
562  externally_handled++;
565  continue;
566  }
567 
568  LOG(VB_GENERAL, LOG_INFO, sidStr.arg((*it).id)
569  .arg((*it).name)
570  .arg(xmltv_grabber));
571 
572  query.prepare(
573  "SELECT COUNT(chanid) FROM channel WHERE sourceid = "
574  ":SRCID AND xmltvid != ''");
575  query.bindValue(":SRCID", (*it).id);
576 
577  if (query.exec() && query.next())
578  {
579  source_channels = query.value(0).toInt();
580  if (source_channels > 0)
581  {
582  LOG(VB_GENERAL, LOG_INFO,
583  QString("Found %1 channels for source %2 which use grabber")
584  .arg(source_channels).arg((*it).id));
585  }
586  else
587  {
588  LOG(VB_GENERAL, LOG_INFO,
589  QString("No channels are configured to use grabber."));
590  }
591  }
592  else
593  {
594  source_channels = 0;
595  LOG(VB_GENERAL, LOG_INFO,
596  QString("Can't get a channel count for source id %1")
597  .arg((*it).id));
598  }
599 
600  bool hasprefmethod = false;
601 
602  if (is_grabber_external(xmltv_grabber))
603  {
604  uint flags = kMSRunShell | kMSStdOut;
605  MythSystemLegacy grabber_capabilities_proc(xmltv_grabber,
606  QStringList("--capabilities"),
607  flags);
608  grabber_capabilities_proc.Run(25);
609  if (grabber_capabilities_proc.Wait() != GENERIC_EXIT_OK)
610  LOG(VB_GENERAL, LOG_ERR,
611  QString("%1 --capabilities failed or we timed out waiting."
612  " You may need to upgrade your xmltv grabber")
613  .arg(xmltv_grabber));
614  else
615  {
616  QByteArray result = grabber_capabilities_proc.ReadAll();
617  QTextStream ostream(result);
618  QString capabilities;
619  while (!ostream.atEnd())
620  {
621  QString capability
622  = ostream.readLine().simplified();
623 
624  if (capability.isEmpty())
625  continue;
626 
627  capabilities += capability + ' ';
628 
629  if (capability == "baseline")
630  (*it).xmltvgrabber_baseline = true;
631 
632  if (capability == "manualconfig")
633  (*it).xmltvgrabber_manualconfig = true;
634 
635  if (capability == "cache")
636  (*it).xmltvgrabber_cache = true;
637 
638  if (capability == "preferredmethod")
639  hasprefmethod = true;
640  }
641  LOG(VB_GENERAL, LOG_INFO,
642  QString("Grabber has capabilities: %1") .arg(capabilities));
643  }
644  }
645 
646  if (hasprefmethod)
647  {
648  uint flags = kMSRunShell | kMSStdOut;
649  MythSystemLegacy grabber_method_proc(xmltv_grabber,
650  QStringList("--preferredmethod"),
651  flags);
652  grabber_method_proc.Run(15);
653  if (grabber_method_proc.Wait() != GENERIC_EXIT_OK)
654  LOG(VB_GENERAL, LOG_ERR,
655  QString("%1 --preferredmethod failed or we timed out "
656  "waiting. You may need to upgrade your xmltv "
657  "grabber").arg(xmltv_grabber));
658  else
659  {
660  QTextStream ostream(grabber_method_proc.ReadAll());
661  (*it).xmltvgrabber_prefmethod =
662  ostream.readLine().simplified();
663 
664  LOG(VB_GENERAL, LOG_INFO, QString("Grabber prefers method: %1")
665  .arg((*it).xmltvgrabber_prefmethod));
666  }
667  }
668 
669  need_post_grab_proc |= !is_grabber_datadirect(xmltv_grabber);
670 
671  if (is_grabber_datadirect(xmltv_grabber) && dd_grab_all)
672  {
673  if (only_update_channels)
674  DataDirectUpdateChannels(*it);
675  else
676  {
677  QDate qCurrentDate = MythDate::current().date();
678  if (!GrabData(*it, 0, &qCurrentDate))
679  ++failures;
680  }
681  }
682  else if ((*it).xmltvgrabber_prefmethod == "allatonce" && !no_allatonce)
683  {
684  if (!GrabData(*it, 0))
685  ++failures;
686  }
687  else if ((*it).xmltvgrabber_baseline ||
688  is_grabber_datadirect(xmltv_grabber))
689  {
690 
691  QDate qCurrentDate = MythDate::current().date();
692 
693  // We'll keep grabbing until it returns nothing
694  // Max days currently supported is 21
695  int grabdays = (is_grabber_datadirect(xmltv_grabber)) ?
696  14 : REFRESH_MAX;
697 
698  grabdays = (maxDays > 0) ? maxDays : grabdays;
699  grabdays = (only_update_channels) ? 1 : grabdays;
700 
701  vector<bool> refresh_request;
702  refresh_request.resize(grabdays, refresh_all);
703  if (!refresh_all)
704  // Set up days to grab if all is not specified
705  // If all was specified the vector was initialized
706  // with true in all occurrences.
707  for (int i = 0; i < grabdays; i++)
708  refresh_request[i] = refresh_day[i];
709 
710  if (is_grabber_datadirect(xmltv_grabber) && only_update_channels)
711  {
712  DataDirectUpdateChannels(*it);
713  grabdays = 0;
714  }
715 
716  for (int i = 0; i < grabdays; i++)
717  {
718  if (!fatalErrors.empty())
719  break;
720 
721  // We need to check and see if the current date has changed
722  // since we started in this loop. If it has, we need to adjust
723  // the value of 'i' to compensate for this.
724  if (MythDate::current().date() != qCurrentDate)
725  {
726  QDate newDate = MythDate::current().date();
727  i += (newDate.daysTo(qCurrentDate));
728  if (i < 0)
729  i = 0;
730  qCurrentDate = newDate;
731  }
732 
733  QString prevDate(qCurrentDate.addDays(i-1).toString());
734  QString currDate(qCurrentDate.addDays(i).toString());
735 
736  LOG(VB_GENERAL, LOG_INFO, ""); // add a space between days
737  LOG(VB_GENERAL, LOG_INFO, "Checking day @ " +
738  QString("offset %1, date: %2").arg(i).arg(currDate));
739 
740  bool download_needed = false;
741 
742  if (refresh_request[i])
743  {
744  if ( i == 1 )
745  {
746  LOG(VB_GENERAL, LOG_INFO,
747  "Data Refresh always needed for tomorrow");
748  }
749  else
750  {
751  LOG(VB_GENERAL, LOG_INFO,
752  "Data Refresh needed because of user request");
753  }
754  download_needed = true;
755  }
756  else
757  {
758  // Check to see if we already downloaded data for this date.
759 
760  querystr = "SELECT c.chanid, COUNT(p.starttime) "
761  "FROM channel c "
762  "LEFT JOIN program p ON c.chanid = p.chanid "
763  " AND starttime >= "
764  "DATE_ADD(DATE_ADD(CURRENT_DATE(), "
765  "INTERVAL '%1' DAY), INTERVAL '20' HOUR) "
766  " AND starttime < DATE_ADD(CURRENT_DATE(), "
767  "INTERVAL '%2' DAY) "
768  "WHERE c.sourceid = %3 AND c.xmltvid != '' "
769  "GROUP BY c.chanid;";
770 
771  if (query.exec(querystr.arg(i-1).arg(i).arg((*it).id)) &&
772  query.isActive())
773  {
774  int prevChanCount = 0;
775  int currentChanCount = 0;
776  int previousDayCount = 0;
777  int currentDayCount = 0;
778 
779  LOG(VB_CHANNEL, LOG_INFO,
780  QString("Checking program counts for day %1")
781  .arg(i-1));
782 
783  while (query.next())
784  {
785  if (query.value(1).toInt() > 0)
786  prevChanCount++;
787  previousDayCount += query.value(1).toInt();
788 
789  LOG(VB_CHANNEL, LOG_INFO,
790  QString(" chanid %1 -> %2 programs")
791  .arg(query.value(0).toString())
792  .arg(query.value(1).toInt()));
793  }
794 
795  if (query.exec(querystr.arg(i).arg(i+1).arg((*it).id))
796  && query.isActive())
797  {
798  LOG(VB_CHANNEL, LOG_INFO,
799  QString("Checking program counts for day %1")
800  .arg(i));
801  while (query.next())
802  {
803  if (query.value(1).toInt() > 0)
804  currentChanCount++;
805  currentDayCount += query.value(1).toInt();
806 
807  LOG(VB_CHANNEL, LOG_INFO,
808  QString(" chanid %1 -> %2 programs")
809  .arg(query.value(0).toString())
810  .arg(query.value(1).toInt()));
811  }
812  }
813  else
814  {
815  LOG(VB_GENERAL, LOG_INFO,
816  QString("Data Refresh because we are unable to "
817  "query the data for day %1 to "
818  "determine if we have enough").arg(i));
819  download_needed = true;
820  }
821 
822  if (currentChanCount < (prevChanCount * 0.90))
823  {
824  LOG(VB_GENERAL, LOG_INFO,
825  QString("Data refresh needed because only %1 "
826  "out of %2 channels have at least one "
827  "program listed for day @ offset %3 "
828  "from 8PM - midnight. Previous day "
829  "had %4 channels with data in that "
830  "time period.")
831  .arg(currentChanCount).arg(source_channels)
832  .arg(i).arg(prevChanCount));
833  download_needed = true;
834  }
835  else if (currentDayCount == 0)
836  {
837  LOG(VB_GENERAL, LOG_INFO,
838  QString("Data refresh needed because no data "
839  "exists for day @ offset %1 from 8PM - "
840  "midnight.").arg(i));
841  download_needed = true;
842  }
843  else if (previousDayCount == 0)
844  {
845  LOG(VB_GENERAL, LOG_INFO,
846  QString("Data refresh needed because no data "
847  "exists for day @ offset %1 from 8PM - "
848  "midnight. Unable to calculate how "
849  "much we should have for the current "
850  "day so a refresh is being forced.")
851  .arg(i-1));
852  download_needed = true;
853  }
854  else if (currentDayCount < (currentChanCount * 3))
855  {
856  LOG(VB_GENERAL, LOG_INFO,
857  QString("Data Refresh needed because offset "
858  "day %1 has less than 3 programs "
859  "per channel for the 8PM - midnight "
860  "time window for channels that "
861  "normally have data. "
862  "We want at least %2 programs, but "
863  "only found %3")
864  .arg(i).arg(currentChanCount * 3)
865  .arg(currentDayCount));
866  download_needed = true;
867  }
868  else if (currentDayCount < (previousDayCount / 2))
869  {
870  LOG(VB_GENERAL, LOG_INFO,
871  QString("Data Refresh needed because offset "
872  "day %1 has less than half the number "
873  "of programs as the previous day for "
874  "the 8PM - midnight time window. "
875  "We want at least %2 programs, but "
876  "only found %3").arg(i)
877  .arg(previousDayCount / 2)
878  .arg(currentDayCount));
879  download_needed = true;
880  }
881  }
882  else
883  {
884  LOG(VB_GENERAL, LOG_INFO,
885  QString("Data Refresh needed because we are unable "
886  "to query the data for day @ offset %1 to "
887  "determine how much we should have for "
888  "offset day %2.").arg(i-1).arg(i));
889  download_needed = true;
890  }
891  }
892 
893  if (download_needed)
894  {
895  LOG(VB_GENERAL, LOG_NOTICE,
896  QString("Refreshing data for ") + currDate);
897  if (!GrabData(*it, i, &qCurrentDate))
898  {
899  ++failures;
900  if (!fatalErrors.empty() || interrupted)
901  {
902  break;
903  }
904  }
905 
906  if (endofdata)
907  {
908  LOG(VB_GENERAL, LOG_INFO,
909  "Grabber is no longer returning program data, "
910  "finishing");
911  break;
912  }
913  }
914  else
915  {
916  LOG(VB_GENERAL, LOG_NOTICE,
917  QString("Data is already present for ") + currDate +
918  ", skipping");
919  }
920  }
921  if (!fatalErrors.empty())
922  break;
923  }
924  else
925  {
926  LOG(VB_GENERAL, LOG_ERR,
927  QString("Grabbing XMLTV data using ") + xmltv_grabber +
928  " is not supported. You may need to upgrade to"
929  " the latest version of XMLTV.");
930  }
931 
932  if (interrupted)
933  {
934  break;
935  }
936 
937  query.prepare("SELECT MAX(endtime) FROM program p LEFT JOIN channel c "
938  "ON p.chanid=c.chanid WHERE c.sourceid= :SRCID "
939  "AND manualid = 0 AND c.xmltvid != '';");
940  query.bindValue(":SRCID", (*it).id);
941 
942  if (query.exec() && query.next())
943  {
944  if (!query.isNull(0))
945  GuideDataAfter = MythDate::fromString(query.value(0).toString());
946  }
947 
948  if (GuideDataAfter == GuideDataBefore)
949  {
950  nonewdata++;
951  }
952  }
953 
954  if (!fatalErrors.empty())
955  {
956  for (int i = 0; i < fatalErrors.size(); i++)
957  {
958  LOG(VB_GENERAL, LOG_CRIT, LOC + "Encountered Fatal Error: " +
959  fatalErrors[i]);
960  }
961  return false;
962  }
963 
964  if (only_update_channels && !need_post_grab_proc)
965  return true;
966 
967  if (failures == 0)
968  {
969  if (nonewdata > 0 &&
970  (total_sources != externally_handled))
971  status = QObject::tr(
972  "mythfilldatabase ran, but did not insert "
973  "any new data into the Guide for %1 of %2 sources. "
974  "This can indicate a potential grabber failure.")
975  .arg(nonewdata)
976  .arg(total_sources);
977  else
978  status = QObject::tr("Successful.");
979 
980  updateLastRunStatus(status);
981  }
982 
983  return (failures == 0);
984 }
985 
986 /* vim: set expandtab tabstop=4 shiftwidth=4: */
static int UpdateChannelsSafe(uint sourceid, bool insert_channels, bool filter_new_channels)
Definition: datadirect.cpp:674
bool next(void)
Wrap QSqlQuery::next() so we can display the query results.
Definition: mythdbcon.cpp:794
static bool ClearDataBySource(uint sourceid, const QDateTime &from, const QDateTime &to, bool use_channel_time_offset)
void Run(time_t timeout=0)
Runs a command inside the /bin/sh shell. Returns immediately.
void bindValue(const QString &placeholder, const QVariant &val)
Definition: mythdbcon.cpp:875
QString password
Definition: filldata.h:36
vector< int > dd_dups
Definition: filldata.h:42
bool GrabDataFromFile(int id, QString &filename)
Definition: filldata.cpp:309
allow access to stdout
Definition: mythsystem.h:39
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
QString xmltvgrabber_prefmethod
Definition: filldata.h:41
bool Run(SourceList &sourcelist)
Goes through the sourcelist and updates its channels with program info grabbed with the associated gr...
Definition: filldata.cpp:481
QSqlQuery wrapper that fetches a DB connection from the connection pool.
Definition: mythdbcon.h:125
bool updateLastRunEnd(void)
Definition: filldata.cpp:40
static bool UpdateChannelsUnsafe(uint sourceid, bool filter_new_channels)
Definition: datadirect.cpp:759
#define GENERIC_EXIT_KILLED
Process killed or stopped.
Definition: exitcodes.h:23
unsigned int uint
Definition: compat.h:140
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
#define LOC
Definition: filldata.cpp:36
int id
Definition: filldata.h:32
bool GrabDataFromDDFile(int id, int offset, const QString &filename, const QString &lineupid, QDate *qCurrentDate=nullptr)
Definition: filldata.cpp:455
static void UpdateStationViewTable(QString lineupid)
Definition: datadirect.cpp:603
QString GetConfDir(void)
Definition: mythdirs.cpp:224
QVariant value(int i) const
Definition: mythdbcon.h:182
bool GrabData(Source source, int offset, QDate *qCurrentDate=nullptr)
Definition: filldata.cpp:331
#define REFRESH_MAX
Definition: filldata.h:19
#define VERBOSE_LEVEL_CHECK(_MASK_, _LEVEL_)
Definition: mythlogging.h:24
run process through shell
Definition: mythsystem.h:41
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
QByteArray & ReadAll()
static bool is_grabber_datadirect(const QString &grabber)
Definition: videosource.h:35
bool isActive(void) const
Definition: mythdbcon.h:188
static MSqlQueryInfo DDCon()
Returns dedicated connection. (Required for using temporary SQL tables.)
Definition: mythdbcon.cpp:596
vector< Source > SourceList
Definition: filldata.h:44
static MSqlQueryInfo InitCon(ConnectionReuse=kNormalConnection)
Only use this in combination with MSqlQuery constructor.
Definition: mythdbcon.cpp:547
uint myth_system(const QString &command, uint flags, uint timeout)
QString lineupid
Definition: filldata.h:37
uint Wait(time_t timeout=0)
vector< ChannelInfo > ChannelInfoList
Definition: channelinfo.h:119
QString name
Definition: filldata.h:33
QString createTempFile(QString name_template, bool dir)
bool prepare(const QString &query)
QSqlQuery::prepare() is not thread safe in Qt <= 3.3.2.
Definition: mythdbcon.cpp:819
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
QString xmltvgrabber
Definition: filldata.h:34
static void DataDirectProgramUpdate(void)
Definition: datadirect.cpp:833
bool updateLastRunStatus(QString &status)
Definition: filldata.cpp:57
bool exec(void)
Wrap QSqlQuery::exec() so we can display SQL.
Definition: mythdbcon.cpp:615
bool SaveSettingOnHost(const QString &key, const QString &newValue, const QString &host)
bool updateNextScheduledRun()
Definition: filldata.cpp:64
static void DBError(const QString &where, const MSqlQuery &query)
Definition: mythdb.cpp:179
static void UpdateProgramViewTable(uint sourceid)
Definition: datadirect.cpp:628
bool isNull(int field) const
Definition: mythdbcon.h:192
QString userid
Definition: filldata.h:35
static bool is_grabber_labs(const QString &grabber)
Definition: videosource.h:50
static int get_datadirect_provider(const QString &grabber)
Definition: videosource.h:40
QDateTime fromString(const QString &dtstr)
Converts kFilename && kISODate formats to QDateTime.
Definition: mythdate.cpp:30
void DataDirectStationUpdate(Source source)
Definition: filldata.cpp:90
Default UTC.
Definition: mythdate.h:14
static bool is_grabber_external(const QString &grabber)
Definition: videosource.h:27
bool GrabDDData(Source source, int poffset, QDate pdate, int ddSource)
Definition: filldata.cpp:170
bool DataDirectUpdateChannels(Source source)
Definition: filldata.cpp:133
bool updateLastRunStart(void)
Definition: filldata.cpp:48
void SetRefresh(int day, bool set)
Definition: filldata.cpp:72