MythTV  master
remotefile.cpp
Go to the documentation of this file.
1 #include <iostream>
2 using namespace std;
3 
4 #include <QUrl>
5 #include <QFile>
6 #include <QFileInfo>
7 
8 // POSIX C headers
9 #include <unistd.h>
10 #include <fcntl.h>
11 
12 #include "mythconfig.h"
13 
14 #ifndef O_STREAMING
15 #define O_STREAMING 0
16 #endif
17 
18 #ifndef O_LARGEFILE
19 #define O_LARGEFILE 0
20 #endif
21 
22 #ifndef O_BINARY
23 #define O_BINARY 0
24 #endif
25 
26 #include "mythdb.h"
27 #include "remotefile.h"
28 #include "mythcorecontext.h"
29 #include "mythsocket.h"
30 #include "compat.h"
31 #include "mythtimer.h"
32 #include "mythdate.h"
33 #include "mythmiscutil.h"
34 #include "threadedfilewriter.h"
35 #include "storagegroup.h"
36 
37 #define MAX_FILE_CHECK 500 // in ms
38 
39 static bool RemoteSendReceiveStringList(const QString &host, QStringList &strlist)
40 {
41  bool ok = false;
42 
44  {
45  // since the master backend cannot connect back around to
46  // itself, and the libraries do not have access to the list
47  // of connected slave backends to query an existing connection
48  // start up a new temporary connection directly to the slave
49  // backend to query the file list
50  QString ann = QString("ANN Playback %1 0")
51  .arg(gCoreContext->GetHostName());
52  QString addr = gCoreContext->GetBackendServerIP(host);
53  int port = gCoreContext->GetBackendServerPort(host);
54  bool mismatch = false;
55 
57  addr, port, ann, &mismatch);
58  if (sock)
59  {
60  ok = sock->SendReceiveStringList(strlist);
61  sock->DecrRef();
62  }
63  else
64  strlist.clear();
65  }
66  else
67  ok = gCoreContext->SendReceiveStringList(strlist);
68 
69  return ok;
70 }
71 
72 RemoteFile::RemoteFile(const QString &_path, bool write, bool useRA,
73  int _timeout_ms,
74  const QStringList *possibleAuxiliaryFiles) :
75  path(_path),
76  usereadahead(useRA), timeout_ms(_timeout_ms),
77  filesize(-1), timeoutisfast(false),
78  readposition(0LL), lastposition(0LL),
79  canresume(false), recordernum(0),
80  lock(QMutex::NonRecursive),
81  controlSock(nullptr), sock(nullptr),
82  query("QUERY_FILETRANSFER %1"),
83  writemode(write), completed(false),
84  localFile(-1), fileWriter(nullptr)
85 {
86  if (writemode)
87  {
88  usereadahead = false;
89  timeout_ms = -1;
90  }
91  else if (possibleAuxiliaryFiles)
92  possibleauxfiles = *possibleAuxiliaryFiles;
93 
94  if (!path.isEmpty())
95  Open();
96 
97  LOG(VB_FILE, LOG_DEBUG, QString("RemoteFile(%1)").arg(path));
98 }
99 
101 {
102  Close();
103  if (controlSock)
104  {
105  controlSock->DecrRef();
106  controlSock = nullptr;
107  }
108  if (sock)
109  {
110  sock->DecrRef();
111  sock = nullptr;
112  }
113 }
114 
115 bool RemoteFile::isLocal(const QString &lpath)
116 {
117  bool is_local = !lpath.isEmpty() &&
118  !lpath.startsWith("myth:") &&
119  (lpath.startsWith("/") || QFile::exists(lpath));
120  return is_local;
121 }
122 
123 bool RemoteFile::isLocal(void) const
124 {
125  return isLocal(path);
126 }
127 
129 {
130  QUrl qurl(path);
131  QString dir;
132 
133  QString host = qurl.host();
134  int port = qurl.port();
135 
136  dir = qurl.path();
137 
138  if (qurl.hasQuery())
139  dir += "?" + QUrl::fromPercentEncoding(
140  qurl.query(QUrl::FullyEncoded).toLocal8Bit());
141 
142  if (qurl.hasFragment())
143  dir += "#" + qurl.fragment();
144 
145  QString sgroup = qurl.userName();
146 
147  MythSocket *lsock = new MythSocket();
148  QString stype = (control) ? "control socket" : "file data socket";
149 
150  QString loc = QString("RemoteFile::openSocket(%1): ").arg(stype);
151 
152  if (port <= 0)
153  {
154  port = gCoreContext->GetBackendServerPort(host);
155  }
156 
157  if (!lsock->ConnectToHost(host, port))
158  {
159  LOG(VB_GENERAL, LOG_ERR, loc +
160  QString("Could not connect to server %1:%2") .arg(host).arg(port));
161  lsock->DecrRef();
162  return nullptr;
163  }
164 
165  QString hostname = GetMythDB()->GetHostName();
166 
167  QStringList strlist;
168 
169 #ifndef IGNORE_PROTO_VER_MISMATCH
170  if (!gCoreContext->CheckProtoVersion(lsock, 5000))
171  {
172  LOG(VB_GENERAL, LOG_ERR, loc +
173  QString("Failed validation to server %1:%2").arg(host).arg(port));
174  lsock->DecrRef();
175  return nullptr;
176  }
177 #endif
178 
179  if (control)
180  {
181  strlist.append(QString("ANN Playback %1 %2").arg(hostname).arg(false));
182  if (!lsock->SendReceiveStringList(strlist))
183  {
184  LOG(VB_GENERAL, LOG_ERR, loc +
185  QString("Could not read string list from server %1:%2")
186  .arg(host).arg(port));
187  lsock->DecrRef();
188  return nullptr;
189  }
190  }
191  else
192  {
193  strlist.push_back(QString("ANN FileTransfer %1 %2 %3 %4")
194  .arg(hostname).arg(writemode)
195  .arg(usereadahead).arg(timeout_ms));
196  strlist << QString("%1").arg(dir);
197  strlist << sgroup;
198 
199  QStringList::const_iterator it = possibleauxfiles.begin();
200  for (; it != possibleauxfiles.end(); ++it)
201  strlist << *it;
202 
203  if (!lsock->SendReceiveStringList(strlist))
204  {
205  LOG(VB_GENERAL, LOG_ERR, loc +
206  QString("Did not get proper response from %1:%2")
207  .arg(host).arg(port));
208  strlist.clear();
209  strlist.push_back("ERROR");
210  strlist.push_back("invalid response");
211  }
212 
213  if (strlist.size() >= 3)
214  {
215  it = strlist.begin(); ++it;
216  recordernum = (*it).toInt(); ++it;
217  filesize = (*(it)).toLongLong(); ++it;
218  for (; it != strlist.end(); ++it)
219  auxfiles << *it;
220  }
221  else if (!strlist.isEmpty() && strlist.size() < 3 &&
222  strlist[0] != "ERROR")
223  {
224  LOG(VB_GENERAL, LOG_ERR, loc +
225  QString("Did not get proper response from %1:%2")
226  .arg(host).arg(port));
227  strlist.clear();
228  strlist.push_back("ERROR");
229  strlist.push_back("invalid response");
230  }
231  }
232 
233  if (strlist.isEmpty() || strlist[0] == "ERROR")
234  {
235  lsock->DecrRef();
236  lsock = nullptr;
237  if (strlist.isEmpty())
238  {
239  LOG(VB_GENERAL, LOG_ERR, loc + "Failed to open socket, timeout");
240  }
241  else
242  {
243  LOG(VB_GENERAL, LOG_ERR, loc + "Failed to open socket" +
244  ((strlist.size() >= 2) ?
245  QString(", error was %1").arg(strlist[1]) :
246  QString(", remote error")));
247  }
248  }
249 
250  return lsock;
251 }
252 
253 bool RemoteFile::isOpen() const
254 {
255  if (isLocal())
256  {
257  return writemode ? (fileWriter != nullptr) : (localFile != -1);
258  }
259  return sock && controlSock;
260 }
261 
263 {
264  if (isOpen())
265  return true;
266 
267  QMutexLocker locker(&lock);
268  return OpenInternal();
269 }
270 
276 {
277  if (isLocal())
278  {
279  if (writemode)
280  {
281  // make sure the directories are created if necessary
282  QFileInfo fi(path);
283  QDir dir(fi.path());
284  if (!dir.exists())
285  {
286  LOG(VB_FILE, LOG_WARNING, QString("RemoteFile::Open(%1) creating directories")
287  .arg(path));
288 
289  if (!dir.mkpath(fi.path()))
290  {
291  LOG(VB_GENERAL, LOG_ERR, QString("RemoteFile::Open(%1) failed to create the directories")
292  .arg(path));
293  return false;
294  }
295  }
296 
298  O_WRONLY|O_TRUNC|O_CREAT|O_LARGEFILE,
299  0644);
300 
301  if (!fileWriter->Open())
302  {
303  delete fileWriter;
304  fileWriter = nullptr;
305  LOG(VB_FILE, LOG_ERR, QString("RemoteFile::Open(%1) write mode error")
306  .arg(path));
307  return false;
308  }
309  SetBlocking();
310  return true;
311  }
312 
313  // local mode, read only
314  if (!Exists(path))
315  {
316  LOG(VB_FILE, LOG_ERR,
317  QString("RemoteFile::Open(%1) Error: Does not exist").arg(path));
318  return false;
319  }
320 
321  localFile = ::open(path.toLocal8Bit().constData(), O_RDONLY);
322  if (localFile == -1)
323  {
324  LOG(VB_FILE, LOG_ERR, QString("RemoteFile::Open(%1) Error: %2")
325  .arg(path).arg(strerror(errno)));
326  return false;
327  }
328  return true;
329  }
330  controlSock = openSocket(true);
331  if (!controlSock)
332  return false;
333 
334  sock = openSocket(false);
335  if (!sock)
336  {
337  // Close the sockets if we received an error so that isOpen() will
338  // return false if the caller tries to use the RemoteFile.
339  Close(true);
340  return false;
341  }
342  canresume = true;
343 
344  return true;
345 }
346 
347 bool RemoteFile::ReOpen(QString newFilename)
348 {
349  if (isLocal())
350  {
351  if (isOpen())
352  {
353  Close();
354  }
355  path = newFilename;
356  return Open();
357  }
358 
359  QMutexLocker locker(&lock);
360 
361  if (!CheckConnection(false))
362  {
363  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::ReOpen(): Couldn't connect");
364  return false;
365  }
366 
367  QStringList strlist( query.arg(recordernum) );
368  strlist << "REOPEN";
369  strlist << newFilename;
370 
372 
373  lock.unlock();
374 
375  bool retval = false;
376  if (!strlist.isEmpty())
377  retval = strlist[0].toInt();
378 
379  return retval;
380 }
381 
382 void RemoteFile::Close(bool haslock)
383 {
384  if (isLocal())
385  {
387  localFile = -1;
388  delete fileWriter;
389  fileWriter = nullptr;
390  return;
391  }
392  if (!controlSock)
393  return;
394 
395  QStringList strlist( query.arg(recordernum) );
396  strlist << "DONE";
397 
398  if (!haslock)
399  {
400  lock.lock();
401  }
403  strlist, 0, MythSocket::kShortTimeout))
404  {
405  LOG(VB_GENERAL, LOG_ERR, "Remote file timeout.");
406  }
407 
408  if (sock)
409  {
410  sock->DecrRef();
411  sock = nullptr;
412  }
413  if (controlSock)
414  {
415  controlSock->DecrRef();
416  controlSock = nullptr;
417  }
418 
419  if (!haslock)
420  {
421  lock.unlock();
422  }
423 }
424 
425 bool RemoteFile::DeleteFile(const QString &url)
426 {
427  if (isLocal(url))
428  {
429  QFile file(url);
430  return file.remove();
431  }
432 
433  bool result = false;
434  QUrl qurl(url);
435  QString filename = qurl.path();
436  QString sgroup = qurl.userName();
437 
438  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
439  filename = filename + "#" + qurl.fragment();
440 
441  if (filename.startsWith("/"))
442  filename = filename.right(filename.length()-1);
443 
444  if (filename.isEmpty() || sgroup.isEmpty())
445  return false;
446 
447  QStringList strlist("DELETE_FILE");
448  strlist << filename;
449  strlist << sgroup;
450 
452 
453  if (!strlist.isEmpty() && strlist[0] == "1")
454  result = true;
455 
456  return result;
457 }
458 
459 bool RemoteFile::Exists(const QString &url)
460 {
461  if (url.isEmpty())
462  return false;
463 
464  struct stat fileinfo;
465  return Exists(url, &fileinfo);
466 }
467 
468 bool RemoteFile::Exists(const QString &url, struct stat *fileinfo)
469 {
470  if (url.isEmpty())
471  return false;
472 
473  QUrl qurl(url);
474  QString filename = qurl.path();
475  QString sgroup = qurl.userName();
476  QString host = qurl.host();
477 
478  if (isLocal(url) || gCoreContext->IsThisBackend(host))
479  {
480  LOG(VB_FILE, LOG_INFO,
481  QString("RemoteFile::Exists(): looking for local file: %1").arg(url));
482 
483  bool fileExists = false;
484  QString fullFilePath = "";
485 
486  if (url.startsWith("myth:"))
487  {
488  StorageGroup sGroup(sgroup, gCoreContext->GetHostName());
489  fullFilePath = sGroup.FindFile(filename);
490  if (!fullFilePath.isEmpty())
491  fileExists = true;
492  }
493  else
494  {
495  QFileInfo info(url);
496  fileExists = info.exists() /*&& info.isFile()*/;
497  fullFilePath = url;
498  }
499 
500  if (fileExists)
501  {
502  if (stat(fullFilePath.toLocal8Bit().constData(), fileinfo) == -1)
503  {
504  LOG(VB_FILE, LOG_ERR,
505  QString("RemoteFile::Exists(): failed to stat file: %1").arg(fullFilePath) + ENO);
506  }
507  }
508 
509  return fileExists;
510  }
511 
512  LOG(VB_FILE, LOG_INFO,
513  QString("RemoteFile::Exists(): looking for remote file: %1").arg(url));
514 
515  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
516  filename = filename + "#" + qurl.fragment();
517 
518  if (filename.startsWith("/"))
519  filename = filename.right(filename.length()-1);
520 
521  if (filename.isEmpty())
522  return false;
523 
524  QStringList strlist("QUERY_FILE_EXISTS");
525  strlist << filename;
526  if (!sgroup.isEmpty())
527  strlist << sgroup;
528 
529  bool result = false;
530  if (RemoteSendReceiveStringList(host, strlist) && strlist[0] == "1")
531  {
532  if ((strlist.size() >= 15) && fileinfo)
533  {
534  fileinfo->st_dev = strlist[2].toLongLong();
535  fileinfo->st_ino = strlist[3].toLongLong();
536  fileinfo->st_mode = strlist[4].toLongLong();
537  fileinfo->st_nlink = strlist[5].toLongLong();
538  fileinfo->st_uid = strlist[6].toLongLong();
539  fileinfo->st_gid = strlist[7].toLongLong();
540  fileinfo->st_rdev = strlist[8].toLongLong();
541  fileinfo->st_size = strlist[9].toLongLong();
542 #ifndef _WIN32
543  fileinfo->st_blksize = strlist[10].toLongLong();
544  fileinfo->st_blocks = strlist[11].toLongLong();
545 #endif
546  fileinfo->st_atime = strlist[12].toLongLong();
547  fileinfo->st_mtime = strlist[13].toLongLong();
548  fileinfo->st_ctime = strlist[14].toLongLong();
549  result = true;
550  }
551  else if (!fileinfo)
552  {
553  result = true;
554  }
555  }
556 
557  return result;
558 }
559 
560 QString RemoteFile::GetFileHash(const QString &url)
561 {
562  if (isLocal(url))
563  {
564  return FileHash(url);
565  }
566  QString result;
567  QUrl qurl(url);
568  QString filename = qurl.path();
569  QString hostname = qurl.host();
570  QString sgroup = qurl.userName();
571 
572  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
573  filename = filename + "#" + qurl.fragment();
574 
575  if (filename.startsWith("/"))
576  filename = filename.right(filename.length()-1);
577 
578  if (filename.isEmpty() || sgroup.isEmpty())
579  return QString();
580 
581  QStringList strlist("QUERY_FILE_HASH");
582  strlist << filename;
583  strlist << sgroup;
584  strlist << hostname;
585 
587  {
588  result = strlist[0];
589  }
590 
591  return result;
592 }
593 
594 bool RemoteFile::CopyFile (const QString& src, const QString& dst,
595  bool overwrite, bool verify)
596 {
597  LOG(VB_FILE, LOG_INFO,
598  QString("RemoteFile::CopyFile: Copying file from '%1' to '%2'").arg(src).arg(dst));
599 
600  // sanity check
601  if (src == dst)
602  {
603  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::CopyFile: Cannot copy a file to itself");
604  return false;
605  }
606 
607  RemoteFile srcFile(src, false);
608  if (!srcFile.isOpen())
609  {
610  LOG(VB_GENERAL, LOG_ERR,
611  QString("RemoteFile::CopyFile: Failed to open file (%1) for reading.").arg(src));
612  return false;
613  }
614 
615  const int readSize = 2 * 1024 * 1024;
616  char *buf = new char[readSize];
617  if (!buf)
618  {
619  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::CopyFile: ERROR, unable to allocate copy buffer");
620  return false;
621  }
622 
623  if (overwrite)
624  {
625  DeleteFile(dst);
626  }
627  else if (Exists(dst))
628  {
629  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::CopyFile: File already exists");
630  delete[] buf;
631  return false;
632  }
633 
634  RemoteFile dstFile(dst, true);
635  if (!dstFile.isOpen())
636  {
637  LOG(VB_GENERAL, LOG_ERR,
638  QString("RemoteFile::CopyFile: Failed to open file (%1) for writing.").arg(dst));
639  srcFile.Close();
640  delete[] buf;
641  return false;
642  }
643 
644  dstFile.SetBlocking(true);
645 
646  bool success = true;
647  int srcLen;
648 
649  while ((srcLen = srcFile.Read(buf, readSize)) > 0)
650  {
651  int dstLen = dstFile.Write(buf, srcLen);
652 
653  if (dstLen == -1 || srcLen != dstLen)
654  {
655  LOG(VB_GENERAL, LOG_ERR,
656  "RemoteFile::CopyFile: Error while trying to write to destination file.");
657  success = false;
658  }
659  }
660 
661  srcFile.Close();
662  dstFile.Close();
663  delete[] buf;
664 
665  if (success && verify)
666  {
667  // Check written file is correct size
668  struct stat fileinfo;
669  long long dstSize = Exists(dst, &fileinfo) ? fileinfo.st_size : -1;
670  long long srcSize = srcFile.GetFileSize();
671  if (dstSize != srcSize)
672  {
673  LOG(VB_GENERAL, LOG_ERR,
674  QString("RemoteFile::CopyFile: Copied file is wrong size (%1 rather than %2)")
675  .arg(dstSize).arg(srcSize));
676  success = false;
677  DeleteFile(dst);
678  }
679  }
680 
681  return success;
682 }
683 
684 bool RemoteFile::MoveFile (const QString& src, const QString& dst, bool overwrite)
685 {
686  LOG(VB_FILE, LOG_INFO,
687  QString("RemoteFile::MoveFile: Moving file from '%1' to '%2'").arg(src).arg(dst));
688 
689  // sanity check
690  if (src == dst)
691  {
692  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::MoveFile: Cannot move a file to itself");
693  return false;
694  }
695 
696  if (isLocal(src) != isLocal(dst))
697  {
698  // Moving between local & remote requires a copy & delete
699  bool ok = CopyFile(src, dst, overwrite, true);
700  if (ok)
701  {
702  if (!DeleteFile(src))
703  LOG(VB_FILE, LOG_ERR,
704  "RemoteFile::MoveFile: Failed to delete file after successful copy");
705  }
706  return ok;
707  }
708 
709  if (overwrite)
710  {
711  DeleteFile(dst);
712  }
713  else if (Exists(dst))
714  {
715  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::MoveFile: File already exists");
716  return false;
717  }
718 
719  if (isLocal(src))
720  {
721  // Moving local -> local
722  QFileInfo fi(dst);
723  if (QDir().mkpath(fi.path()) && QFile::rename(src, dst))
724  return true;
725 
726  LOG(VB_FILE, LOG_ERR, "RemoteFile::MoveFile: Rename failed");
727  return false;
728  }
729 
730  // Moving remote -> remote
731  QUrl srcUrl(src);
732  QUrl dstUrl(dst);
733 
734  if (srcUrl.userName() != dstUrl.userName())
735  {
736  LOG(VB_FILE, LOG_ERR, "RemoteFile::MoveFile: Cannot change a file's Storage Group");
737  return false;
738  }
739 
740  QStringList strlist("MOVE_FILE");
741  strlist << srcUrl.userName() << srcUrl.path() << dstUrl.path();
742 
744 
745  if (!strlist.isEmpty() && strlist[0] == "1")
746  return true;
747 
748  LOG(VB_FILE, LOG_ERR, QString("RemoteFile::MoveFile: MOVE_FILE failed with: %1")
749  .arg(strlist.join(",")));
750  return false;
751 }
752 
754 {
755  if (isLocal())
756  return;
757  QMutexLocker locker(&lock);
758  if (!sock)
759  {
760  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Reset(): Called with no socket");
761  return;
762  }
763  sock->Reset();
764 }
765 
766 long long RemoteFile::Seek(long long pos, int whence, long long curpos)
767 {
768  QMutexLocker locker(&lock);
769 
770  return SeekInternal(pos, whence, curpos);
771 }
772 
773 long long RemoteFile::SeekInternal(long long pos, int whence, long long curpos)
774 {
775  if (isLocal())
776  {
777  if (!isOpen())
778  {
779  LOG(VB_FILE, LOG_ERR, "RemoteFile::Seek(): Called with no file opened");
780  return -1;
781  }
782  if (writemode)
783  return fileWriter->Seek(pos, whence);
784 
785  long long offset = 0LL;
786  if (whence == SEEK_SET)
787  {
788  QFileInfo info(path);
789  offset = min(pos, info.size());
790  }
791  else if (whence == SEEK_END)
792  {
793  QFileInfo info(path);
794  offset = info.size() + pos;
795  }
796  else if (whence == SEEK_CUR)
797  {
798  offset = ((curpos > 0) ? curpos : ::lseek64(localFile, 0, SEEK_CUR)) + pos;
799  }
800  else
801  return -1;
802 
803  off64_t localpos = ::lseek64(localFile, pos, whence);
804  if (localpos != pos)
805  {
806  LOG(VB_FILE, LOG_ERR,
807  QString("RemoteFile::Seek(): Couldn't seek to offset %1")
808  .arg(offset));
809  return -1;
810  }
811  return localpos;
812  }
813 
814  if (!CheckConnection(false))
815  {
816  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Seek(): Couldn't connect");
817  return -1;
818  }
819 
820  QStringList strlist( query.arg(recordernum) );
821  strlist << "SEEK";
822  strlist << QString::number(pos);
823  strlist << QString::number(whence);
824  if (curpos > 0)
825  strlist << QString::number(curpos);
826  else
827  strlist << QString::number(readposition);
828 
829  bool ok = controlSock->SendReceiveStringList(strlist);
830 
831  if (ok && !strlist.isEmpty())
832  {
833  lastposition = readposition = strlist[0].toLongLong();
834  sock->Reset();
835  return strlist[0].toLongLong();
836  }
837  else
838  {
839  lastposition = 0LL;
840  }
841 
842  return -1;
843 }
844 
845 int RemoteFile::Write(const void *data, int size)
846 {
847  int recv = 0;
848  int sent = 0;
849  unsigned zerocnt = 0;
850  bool error = false;
851  bool response = false;
852 
853  if (!writemode)
854  {
855  LOG(VB_NETWORK, LOG_ERR,
856  "RemoteFile::Write(): Called when not in write mode");
857  return -1;
858  }
859  if (isLocal())
860  {
861  if (!isOpen())
862  {
863  LOG(VB_FILE, LOG_ERR,
864  "RemoteFile::Write(): File not opened");
865  return -1;
866  }
867  return fileWriter->Write(data, size);
868  }
869 
870  QMutexLocker locker(&lock);
871 
872  if (!CheckConnection())
873  {
874  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Write(): Couldn't connect");
875  return -1;
876  }
877 
878  QStringList strlist( query.arg(recordernum) );
879  strlist << "WRITE_BLOCK";
880  strlist << QString::number(size);
881  bool ok = controlSock->WriteStringList(strlist);
882  if (!ok)
883  {
884  LOG(VB_NETWORK, LOG_ERR,
885  "RemoteFile::Write(): Block notification failed");
886  return -1;
887  }
888 
889  recv = size;
890  while (sent < recv && !error && zerocnt++ < 50)
891  {
892  int ret = sock->Write((char*)data + sent, recv - sent);
893  if (ret > 0)
894  {
895  sent += ret;
896  }
897  else
898  {
899  LOG(VB_GENERAL, LOG_ERR, "RemoteFile::Write(): socket error");
900  error = true;
901  break;
902  }
903 
904  if (controlSock->IsDataAvailable() &&
906  !strlist.isEmpty())
907  {
908  recv = strlist[0].toInt(); // -1 on backend error
909  response = true;
910  }
911  }
912 
913  if (!error && !response)
914  {
916  !strlist.isEmpty())
917  {
918  recv = strlist[0].toInt(); // -1 on backend error
919  }
920  else
921  {
922  LOG(VB_GENERAL, LOG_ERR,
923  "RemoteFile::Write(): No response from control socket.");
924  recv = -1;
925  }
926  }
927 
928  LOG(VB_NETWORK, LOG_DEBUG,
929  QString("RemoteFile::Write(): reqd=%1, sent=%2, rept=%3, error=%4")
930  .arg(size).arg(sent).arg(recv).arg(error));
931 
932  if (recv < 0)
933  return recv;
934 
935  if (error || recv != sent)
936  {
937  sent = -1;
938  }
939  else
940  {
941  lastposition += sent;
942  }
943 
944  return sent;
945 }
946 
947 int RemoteFile::Read(void *data, int size)
948 {
949  int recv = 0;
950  int sent = 0;
951  bool error = false;
952  bool response = false;
953 
954  QMutexLocker locker(&lock);
955 
956  if (isLocal())
957  {
958  if (writemode)
959  {
960  LOG(VB_FILE, LOG_ERR, "RemoteFile:Read() called in writing mode");
961  return -1;
962  }
963  if (isOpen())
964  {
965  return ::read(localFile, data, size);
966  }
967  LOG(VB_FILE, LOG_ERR, "RemoteFile:Read() called when local file not opened");
968  return -1;
969  }
970 
971  if (!CheckConnection())
972  {
973  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Read(): Couldn't connect");
974  return -1;
975  }
976 
977  if (sock->IsDataAvailable())
978  {
979  LOG(VB_NETWORK, LOG_ERR,
980  "RemoteFile::Read(): Read socket not empty to start!");
981  sock->Reset();
982  }
983 
984  while (controlSock->IsDataAvailable())
985  {
986  LOG(VB_NETWORK, LOG_WARNING,
987  "RemoteFile::Read(): Control socket not empty to start!");
988  controlSock->Reset();
989  }
990 
991  QStringList strlist( query.arg(recordernum) );
992  strlist << "REQUEST_BLOCK";
993  strlist << QString::number(size);
994  bool ok = controlSock->WriteStringList(strlist);
995  if (!ok)
996  {
997  LOG(VB_NETWORK, LOG_ERR, "RemoteFile::Read(): Block request failed");
998  return -1;
999  }
1000 
1001  sent = size;
1002 
1003  int waitms = 30;
1004  MythTimer mtimer;
1005  mtimer.start();
1006 
1007  while (recv < sent && !error && mtimer.elapsed() < 10000)
1008  {
1009  int ret = sock->Read(((char *)data) + recv, sent - recv, waitms);
1010 
1011  if (ret > 0)
1012  recv += ret;
1013  else if (ret < 0)
1014  error = true;
1015 
1016  waitms += (waitms < 200) ? 20 : 0;
1017 
1018  if (controlSock->IsDataAvailable() &&
1020  !strlist.isEmpty())
1021  {
1022  sent = strlist[0].toInt(); // -1 on backend error
1023  response = true;
1024  if (ret < sent)
1025  {
1026  // We have received less than what the server sent, retry immediately
1027  ret = sock->Read(((char *)data) + recv, sent - recv, waitms);
1028  if (ret > 0)
1029  recv += ret;
1030  else if (ret < 0)
1031  error = true;
1032  }
1033  }
1034  }
1035 
1036  if (!error && !response)
1037  {
1038  // Wait up to 1.5s for the backend to send the size
1039  // MythSocket::ReadString will drop the connection
1040  if (controlSock->ReadStringList(strlist, 1500) &&
1041  !strlist.isEmpty())
1042  {
1043  sent = strlist[0].toInt(); // -1 on backend error
1044  }
1045  else
1046  {
1047  LOG(VB_GENERAL, LOG_ERR,
1048  "RemoteFile::Read(): No response from control socket.");
1049  // If no data was received from control socket, and we got what we asked for
1050  // assume everything is okay
1051  if (recv == size)
1052  {
1053  sent = recv;
1054  }
1055  else
1056  {
1057  sent = -1;
1058  }
1059  // The TCP socket is dropped if there's a timeout, so we reconnect
1060  if (!Resume())
1061  {
1062  sent = -1;
1063  }
1064  }
1065  }
1066 
1067  LOG(VB_NETWORK, LOG_DEBUG,
1068  QString("Read(): reqd=%1, rcvd=%2, rept=%3, error=%4")
1069  .arg(size).arg(recv).arg(sent).arg(error));
1070 
1071  if (sent < 0)
1072  return sent;
1073 
1074  if (error || sent != recv)
1075  {
1076  LOG(VB_GENERAL, LOG_WARNING,
1077  QString("RemoteFile::Read(): sent %1 != recv %2")
1078  .arg(sent).arg(recv));
1079  recv = -1;
1080 
1081  // The TCP socket is dropped if there's a timeout, so we reconnect
1082  if (!Resume())
1083  {
1084  LOG(VB_GENERAL, LOG_WARNING, "RemoteFile::Read(): Resume failed.");
1085  sent = -1;
1086  }
1087  else
1088  LOG(VB_GENERAL, LOG_NOTICE, "RemoteFile::Read(): Resume success.");
1089  }
1090  else
1091  {
1092  lastposition += recv;
1093  }
1094 
1095  return recv;
1096 }
1097 
1103 long long RemoteFile::GetFileSize(void) const
1104 {
1105  if (isLocal())
1106  {
1107  if (isOpen() && writemode)
1108  {
1109  fileWriter->Flush();
1110  }
1111  if (Exists(path))
1112  {
1113  QFileInfo info(path);
1114  return info.size();
1115  }
1116  return -1;
1117  }
1118 
1119  QMutexLocker locker(&lock);
1120  return filesize;
1121 }
1122 
1132 {
1133  if (isLocal())
1134  {
1135  return GetFileSize();
1136  }
1137 
1138  QMutexLocker locker(&lock);
1139 
1140  if (completed ||
1142  {
1143  return filesize;
1144  }
1145 
1146  if (!CheckConnection())
1147  {
1148  // Can't establish a new connection, using system one
1149  struct stat fileinfo;
1150 
1151  if (Exists(path, &fileinfo))
1152  {
1153  filesize = fileinfo.st_size;
1154  }
1155  return filesize;
1156  }
1157 
1158  QStringList strlist(query.arg(recordernum));
1159  strlist << "REQUEST_SIZE";
1160 
1161  bool ok = controlSock->SendReceiveStringList(strlist);
1162 
1163  if (ok && !strlist.isEmpty())
1164  {
1165  bool validate;
1166  long long size = strlist[0].toLongLong(&validate);
1167 
1168  if (validate)
1169  {
1170  if (strlist.count() >= 2)
1171  {
1172  completed = strlist[1].toInt();
1173  }
1174  filesize = size;
1175  }
1176  else
1177  {
1178  struct stat fileinfo;
1179 
1180  if (Exists(path, &fileinfo))
1181  {
1182  filesize = fileinfo.st_size;
1183  }
1184  }
1186  return filesize;
1187  }
1188 
1189  return -1;
1190 }
1191 
1192 bool RemoteFile::SaveAs(QByteArray &data)
1193 {
1194  long long fs = GetRealFileSize();
1195 
1196  if (fs < 0)
1197  return false;
1198 
1199  data.resize(fs);
1200  Read(data.data(), fs);
1201 
1202  return true;
1203 }
1204 
1205 void RemoteFile::SetTimeout(bool fast)
1206 {
1207  if (isLocal())
1208  {
1209  // not much we can do with local accesses
1210  return;
1211  }
1212  if (timeoutisfast == fast)
1213  return;
1214 
1215  QMutexLocker locker(&lock);
1216 
1217  if (!CheckConnection())
1218  {
1219  LOG(VB_NETWORK, LOG_ERR,
1220  "RemoteFile::SetTimeout(): Couldn't connect");
1221  return;
1222  }
1223 
1224  QStringList strlist( query.arg(recordernum) );
1225  strlist << "SET_TIMEOUT";
1226  strlist << QString::number((int)fast);
1227 
1229 
1230  timeoutisfast = fast;
1231 }
1232 
1233 QDateTime RemoteFile::LastModified(const QString &url)
1234 {
1235  if (isLocal(url))
1236  {
1237  QFileInfo info(url);
1238  return info.lastModified();
1239  }
1240  QDateTime result;
1241  QUrl qurl(url);
1242  QString filename = qurl.path();
1243  QString sgroup = qurl.userName();
1244 
1245  if (!qurl.fragment().isEmpty() || url.endsWith("#"))
1246  filename = filename + "#" + qurl.fragment();
1247 
1248  if (filename.startsWith("/"))
1249  filename = filename.right(filename.length()-1);
1250 
1251  if (filename.isEmpty() || sgroup.isEmpty())
1252  return result;
1253 
1254  QStringList strlist("QUERY_SG_FILEQUERY");
1255  strlist << qurl.host();
1256  strlist << sgroup;
1257  strlist << filename;
1258 
1260 
1261  if (strlist.size() > 1) {
1262 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
1263  result = MythDate::fromTime_t(strlist[1].toUInt());
1264 #else
1265  if (!strlist[1].isEmpty() && (strlist[1].toInt() != -1))
1266  result = MythDate::fromSecsSinceEpoch(strlist[1].toLongLong());
1267  else
1268  result = QDateTime();;
1269 #endif
1270  }
1271 
1272  return result;
1273 }
1274 
1275 QDateTime RemoteFile::LastModified(void) const
1276 {
1277  return LastModified(path);
1278 }
1279 
1289 QString RemoteFile::FindFile(const QString& filename, const QString& host,
1290  const QString& storageGroup, bool useRegex,
1291  bool allowFallback)
1292 {
1293  QStringList files = RemoteFile::FindFileList(filename, host, storageGroup, useRegex, allowFallback);
1294 
1295  if (!files.isEmpty())
1296  return files[0];
1297 
1298  return QString();
1299 }
1300 
1310 QStringList RemoteFile::FindFileList(const QString& filename, const QString& host,
1311  const QString& storageGroup, bool useRegex,
1312  bool allowFallback)
1313 {
1314  LOG(VB_FILE, LOG_INFO, QString("RemoteFile::FindFile(): looking for '%1' on '%2' in group '%3' "
1315  "(useregex: %4, allowfallback: %5)")
1316  .arg(filename).arg(host).arg(storageGroup)
1317  .arg(useRegex).arg(allowFallback));
1318 
1319  if (filename.isEmpty() || storageGroup.isEmpty())
1320  return QStringList();
1321 
1322  QStringList strList;
1323  QString hostName = host;
1324 
1325  if (hostName.isEmpty())
1326  hostName = gCoreContext->GetMasterHostName();
1327 
1328  // if we are looking for the file on this host just search the local storage group first
1329  if (gCoreContext->IsThisBackend(hostName))
1330  {
1331  // We could have made it this far with an IP when we really want
1332  // a hostname
1333  hostName = gCoreContext->GetHostName();
1334  StorageGroup sgroup(storageGroup, hostName);
1335 
1336  if (useRegex)
1337  {
1338  QFileInfo fi(filename);
1339  QStringList files = sgroup.GetFileList('/' + fi.path());
1340 
1341  LOG(VB_FILE, LOG_INFO, QString("RemoteFile::FindFileList: Looking in dir '%1' for '%2'")
1342  .arg(fi.path()).arg(fi.fileName()));
1343 
1344  for (int x = 0; x < files.size(); x++)
1345  {
1346  LOG(VB_FILE, LOG_INFO, QString("RemoteFile::FindFileList: Found '%1 - %2'")
1347  .arg(x).arg(files[x]));
1348  }
1349 
1350  QStringList filteredFiles = files.filter(QRegExp(fi.fileName()));
1351  for (int x = 0; x < filteredFiles.size(); x++)
1352  {
1355  fi.path() + '/' + filteredFiles[x],
1356  storageGroup);
1357  }
1358  }
1359  else
1360  {
1361  if (!sgroup.FindFile(filename).isEmpty())
1362  strList << gCoreContext->GenMythURL(hostName,
1364  filename, storageGroup);
1365  }
1366 
1367  if (!strList.isEmpty() || !allowFallback)
1368  return strList;
1369  }
1370 
1371  // if we didn't find any files ask the master BE to find it
1372  if (strList.isEmpty() && !gCoreContext->IsMasterBackend())
1373  {
1374  strList << "QUERY_FINDFILE" << hostName << storageGroup << filename
1375  << (useRegex ? "1" : "0")
1376  << "1";
1377 
1378  if (gCoreContext->SendReceiveStringList(strList))
1379  {
1380  if (strList.size() > 0 && !strList[0].isEmpty() &&
1381  strList[0] != "NOT FOUND" && !strList[0].startsWith("ERROR: "))
1382  return strList;
1383  }
1384  }
1385 
1386  return QStringList();
1387 }
1388 
1394 bool RemoteFile::SetBlocking(bool block)
1395 {
1396  if (fileWriter)
1397  {
1398  return fileWriter->SetBlocking(block);
1399  }
1400  return true;
1401 }
1402 
1410 {
1411  if (IsConnected())
1412  {
1413  return true;
1414  }
1415  if (!canresume)
1416  {
1417  return false;
1418  }
1419  return Resume(repos);
1420 }
1421 
1427 {
1428  return sock && controlSock &&
1430 }
1431 
1437 bool RemoteFile::Resume(bool repos)
1438 {
1439  Close(true);
1440  if (!OpenInternal())
1441  return false;
1442 
1443  if (repos)
1444  {
1446  if (SeekInternal(lastposition, SEEK_SET) < 0)
1447  {
1448  Close(true);
1449  LOG(VB_FILE, LOG_ERR,
1450  QString("RemoteFile::Resume: Enable to re-seek into last known "
1451  "position (%1").arg(lastposition));
1452  return false;
1453  }
1454  }
1455  readposition = lastposition = 0;
1456  return true;
1457 }
1458 
1459 /* vim: set expandtab tabstop=4 shiftwidth=4: */
#define MAX_FILE_CHECK
Definition: remotefile.cpp:37
RemoteFile(const QString &url="", bool write=false, bool usereadahead=true, int timeout_ms=2000, const QStringList *possibleAuxiliaryFiles=nullptr)
Definition: remotefile.cpp:72
long long filesize
Definition: remotefile.h:81
def write(text, progress=True)
Definition: mythburn.py:279
static bool DeleteFile(const QString &url)
Definition: remotefile.cpp:425
int restart(void)
Returns milliseconds elapsed since last start() or restart() and resets the count.
Definition: mythtimer.cpp:62
void SetTimeout(bool fast)
bool usereadahead
Definition: remotefile.h:79
QString FileHash(QString filename)
A QElapsedTimer based timer to replace use of QTime as a timer.
Definition: mythtimer.h:13
void Reset(void)
Definition: remotefile.cpp:753
bool SetBlocking(bool block=true)
Set write blocking mode for the ThreadedFileWriter instance.
QString GenMythURL(QString host=QString(), QString port=QString(), QString path=QString(), QString storageGroup=QString())
long long readposition
Definition: remotefile.h:83
bool timeoutisfast
Definition: remotefile.h:82
bool completed
Definition: remotefile.h:94
int recordernum
Definition: remotefile.h:86
long long SeekInternal(long long pos, int whence, long long curpos=-1)
Definition: remotefile.cpp:773
static void error(const char *str,...)
Definition: vbi.c:41
QString GetBackendServerIP(void)
Returns the IP address of the locally defined backend IP.
static QString FindFile(const QString &filename, const QString &host, const QString &storageGroup, bool useRegex=false, bool allowFallback=false)
Search all BE's for a file in the give storage group.
QMutex lock
Definition: remotefile.h:88
int localFile
Definition: remotefile.h:99
int Read(char *, int size, int max_wait_ms)
Definition: mythsocket.cpp:546
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
bool IsThisBackend(const QString &addr)
is this address mapped to this backend host
long long Seek(long long pos, int whence)
Seek to a position within stream; May be unsafe.
bool writemode
Definition: remotefile.h:93
#define O_LARGEFILE
Definition: remotefile.cpp:19
void Close(bool haslock=false)
Definition: remotefile.cpp:382
bool IsConnected(void)
Check if both the control and data sockets are currently connected.
bool isRunning(void) const
Returns true if start() or restart() has been called at least once since construction and since any c...
Definition: mythtimer.cpp:134
MythSocket * openSocket(bool control)
Definition: remotefile.cpp:128
def read(device=None, features=[])
Definition: disc.py:35
long long GetRealFileSize(void)
GetRealFileSize: returns the current remote file's size.
int GetBackendServerPort(void)
Returns the locally defined backend control port.
bool OpenInternal(void)
Attempts to resume from a disconnected step.
Definition: remotefile.cpp:275
long long lastposition
Definition: remotefile.h:84
static bool RemoteSendReceiveStringList(const QString &host, QStringList &strlist)
Definition: remotefile.cpp:39
MythSocket * ConnectCommandSocket(const QString &hostname, int port, const QString &announcement, bool *proto_mismatch=nullptr, int maxConnTry=-1, int setup_timeout=-1)
int Write(const void *data, uint count)
Writes data to the end of the write buffer.
bool SendReceiveStringList(QStringList &strlist, bool quickTimeout=false, bool block=true)
Send a message to the backend and wait for a response.
static bool MoveFile(const QString &src, const QString &dest, bool overwrite=false)
Definition: remotefile.cpp:684
MBASE_PUBLIC QDateTime fromSecsSinceEpoch(uint seconds)
This function takes the number of seconds since the start of the epoch and returns a QDateTime with t...
Definition: mythdate.cpp:88
VERBOSE_PREAMBLE false
Definition: verbosedefs.h:85
#define close
Definition: compat.h:16
bool isLocal(void) const
Definition: remotefile.cpp:123
int Write(const char *, int size)
Definition: mythsocket.cpp:533
QStringList auxfiles
Definition: remotefile.h:98
bool IsDataAvailable(void) const
Definition: mythsocket.cpp:576
void Flush(void)
Allow DiskLoop() to flush buffer completely ignoring low watermark.
virtual int DecrRef(void)
Decrements reference count and deletes on 0.
MythSocket * controlSock
Definition: remotefile.h:89
int Read(void *data, int size)
Definition: remotefile.cpp:947
QString GetMasterHostName(void)
string hostname
Definition: caa.py:17
bool canresume
Definition: remotefile.h:85
MythTimer lastSizeCheck
Definition: remotefile.h:95
QDateTime LastModified(void) const
static bool CopyFile(const QString &src, const QString &dest, bool overwrite=false, bool verify=false)
Definition: remotefile.cpp:594
#define ENO
This can be appended to the LOG args with "+".
Definition: mythlogging.h:99
bool CheckConnection(bool repos=true)
Check current connection and re-establish it if lost.
MythSocket * sock
Definition: remotefile.h:90
long long GetFileSize(void) const
GetFileSize: returns the remote file's size at the time it was first opened Will query the server in ...
void Reset(void)
Definition: mythsocket.cpp:560
int elapsed(void) const
Returns milliseconds elapsed since last start() or restart()
Definition: mythtimer.cpp:90
bool Resume(bool repos=true)
Attempts to resume from a disconnected step.
bool ReadStringList(QStringList &list, uint timeoutMS=kShortTimeout)
Definition: mythsocket.cpp:332
static bool Exists(const QString &url, struct stat *fileinfo)
Definition: remotefile.cpp:468
bool SaveAs(QByteArray &data)
bool Open(void)
Opens the file we will be writing to.
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
ThreadedFileWriter * fileWriter
Definition: remotefile.h:100
bool isOpen(void) const
Definition: remotefile.cpp:253
bool ConnectToHost(const QString &hostname, quint16 port)
connect to host
Definition: mythsocket.cpp:393
QString FindFile(const QString &filename)
static QStringList FindFileList(const QString &filename, const QString &host, const QString &storageGroup, bool useRegex=false, bool allowFallback=false)
Search all BE's for files in the give storage group.
bool CheckProtoVersion(MythSocket *socket, uint timeout_ms=kMythSocketLongTimeout, bool error_dialog_desired=false)
bool Open(void)
Definition: remotefile.cpp:262
static const uint kShortTimeout
Definition: mythsocket.h:71
QString query
Definition: remotefile.h:91
bool ReOpen(QString newFilename)
Definition: remotefile.cpp:347
bool IsConnected(void) const
Definition: mythsocket.cpp:570
QString path
Definition: remotefile.h:78
This class supports the writing of recordings to disk.
Class for communcating between myth backends and frontends.
Definition: mythsocket.h:26
QStringList GetFileList(const QString &Path, bool recursive=false)
bool SendReceiveStringList(QStringList &list, uint min_reply_length=0, uint timeoutMS=kLongTimeout)
Definition: mythsocket.cpp:345
long long Seek(long long pos, int whence, long long curpos=-1)
Definition: remotefile.cpp:766
bool WriteStringList(const QStringList &list)
Definition: mythsocket.cpp:320
MythDB * GetMythDB(void)
Definition: mythdb.cpp:46
QStringList possibleauxfiles
Definition: remotefile.h:97
int timeout_ms
Definition: remotefile.h:80
bool IsMasterBackend(void)
is this the actual MBE process
int Write(const void *data, int size)
Definition: remotefile.cpp:845
QString GetHostName(void)
void start(void)
starts measuring elapsed time.
Definition: mythtimer.cpp:47
static QString GetFileHash(const QString &url)
Definition: remotefile.cpp:560
bool SetBlocking(bool block=true)
Set write blocking mode While in blocking mode, ThreadedFileWriter::Write will wait for buffers to be...