Let files be scanned one at a time as they arrive

include/par2/libpar2.h

@@ -25,6 +25,7 @@
 #include <memory>
 #include <mutex>
 #include <ostream>
+#include <set>
 #include <string>
 #include <utility>
 #include <vector>
@@ -277,16 +278,16 @@ class Par2Verifier
 
   // Look for blocks which are not where the set says they should be, the -N
   // option, with leaway the distance either side to search, the -S option.
-  // Zero leaway selects the default. Applies to Verify.
+  // Zero leaway selects the default. Applies to Verify and to VerifyFile.
   void SetDataSkipping(const bool enabled, const u64 leaway = 0);
 
   // Hash the whole of each file as well as its blocks, the --full-hash option.
-  // Applies to Verify.
+  // Applies to Verify and to VerifyFile.
   void SetFullHash(const bool enabled);
 
   // Threads for the main processing and for hashing files in parallel, the -t
   // and -T options. Either left zero stays at the default. They are read by
-  // the next Verify or Repair.
+  // the next Verify, VerifyFile or Repair.
   void SetThreadCounts(const u32 nthreads, const u32 filethreads);
 
   // Check the files described by the set against the data on disk. Returns
@@ -297,6 +298,27 @@ class Par2Verifier
   // the results of the Verify that preceded it, so call them in that order.
   Result Verify(const std::vector<std::string> &extrafiles = {});
 
+  // Scan one file that has become available, matching it against the set the
+  // way a Verify would, without reading anything else. Use it to feed files in
+  // as they arrive rather than waiting for all of them.
+  //
+  // Safe to call again for the same file: whatever an earlier scan found for it
+  // is discarded first, so a file which was incomplete when it was first
+  // scanned can be scanned again once it is finished. Blocks another file
+  // supplied are left alone.
+  //
+  // The name may be one the set describes or one it does not; an unrecognised
+  // file is matched by content, as an extra file is.
+  //
+  // Returns what Verify would return for the set as it stands, so a file which
+  // has not been scanned yet still counts as missing. A later Verify replaces
+  // everything the individual scans found, and after a Repair they start again
+  // from nothing.
+  //
+  // May be called before any PAR2 file has been added: the result is then
+  // eInsufficientCriticalData, and the file is scanned once one arrives.
+  Result VerifyFile(const std::string &filename);
+
   // The numbers behind the last Verify or Reassess. A repair is possible when
   // recoveryblockcount is at least missingblockcount, and needs
   // missingblockcount - recoveryblockcount more blocks when it is not.
@@ -375,6 +397,7 @@ class Par2Verifier
   u64 skipleaway;
   bool fullhash;
   std::vector<std::string> par2files;
+  std::set<std::string> scannedfiles;
   std::map<std::string, std::vector<bool> > knownblocks;
   bool verified;
   bool scanned;

src/diskfile.cpp

@@ -1216,6 +1216,16 @@ DiskFile* DiskFileMap::Find(std::string filename) const
   return (f != diskfilemap.end()) ?  f->second : 0;
 }
 
+std::vector<DiskFile*> DiskFileMap::Files(void) const
+{
+  std::vector<DiskFile*> files;
+
+  for (const auto &f : diskfilemap)
+    files.push_back(f.second);
+
+  return files;
+}
+
 
 FileSizeCache::FileSizeCache()
 {

src/diskfile.h

@@ -159,6 +159,9 @@ class DiskFileMap
   void Remove(DiskFile *diskfile);
   DiskFile* Find(std::string filename) const;
 
+  // Every file in the map
+  std::vector<DiskFile*> Files(void) const;
+
 protected:
   std::map<std::string, DiskFile*>    diskfilemap;             // Map from filename to DiskFile
 };

src/libpar2.cpp

@@ -166,6 +166,41 @@ class Par2Verifier::Impl : public Par2Repairer
     noiselevel = _noiselevel;
   }
 
+  // Forget every scan, keeping what the PAR2 files hold
+  void DiscardScans(void)
+  {
+    for (auto *sourcefile : sourcefiles)
+    {
+      if (0 == sourcefile)
+        continue;
+
+      sourcefile->SetTargetFile(0);
+      sourcefile->SetTargetExists(false);
+      sourcefile->SetCompleteFile(0);
+
+      if (sourcefile->GetDescriptionPacket() != 0)
+      {
+        auto block = sourcefile->SourceBlocks();
+        for (u32 i = 0; i < sourcefile->BlockCount(); ++i, ++block)
+        {
+          if (block->IsSet())
+            block->ClearLocation();
+        }
+      }
+    }
+
+    renamedlist.clear();
+
+    for (auto *diskfile : diskFileMap.Files())
+    {
+      if (0 == packetfiles.count(diskfile))
+      {
+        diskFileMap.Remove(diskfile);
+        delete diskfile;
+      }
+    }
+  }
+
   void SetBasePath(const std::string &_basepath)
   {
     basepath = _basepath;
@@ -189,15 +224,19 @@ class Par2Verifier::Impl : public Par2Repairer
     if (prepared != eSuccess)
       return prepared;
 
-    UpdateVerificationResults();
+    return ScanOutcome();
+  }
 
-    if (!CheckVerificationResults())
-      return eRepairNotPossible;
+  Result Scan(const std::string &filename, const size_t memorylimit,
+              const u32 _nthreads, const u32 _filethreads)
+  {
+    if (prepared != eSuccess)
+      return prepared;
 
-    if (completefilecount < mainpacket->RecoverableFileCount())
-      return eRepairPossible;
+    ApplyThreadCounts(_nthreads, _filethreads);
+    ApplyMemoryLimit(memorylimit);
 
-    return eSuccess;
+    return ScanFile(filename, basepath);
   }
 
   void SetDataSkipping(const bool _skipdata, const u64 _skipleaway)
@@ -278,13 +317,13 @@ std::string BasePathFor(const std::string &parfilename)
   return basepath;
 }
 
-// Verifying leaves a Par2Repairer with the results of that one pass, so a
-// second pass has to start from a new one. The PAR2 files that were added are
-// read again, which is cheap next to scanning the data files.
+// A repair, or a set which changes shape after files were scanned, leaves a
+// Par2Repairer with results which no longer hold, so the work starts again from
+// a new one. The PAR2 files that were added are read again.
 void Par2Verifier::Restart(void)
 {
-  // A cancel which arrives while the engine is replaced and replayed is kept
-  // for the new one rather than given to either
+  // A cancel which arrives while the engine is replaced and the PAR2 files
+  // are read again is kept for the new one rather than given to either
   {
     std::lock_guard<std::mutex> lock(cancelmutex);
     restarting = true;
@@ -293,8 +332,7 @@ void Par2Verifier::Restart(void)
 
   // The replay repeats work the observer has already been told about, so it is
   // told none of it, and nothing is written to the streams again. The observer
-  // is attached once the handle is back where it was, and the cancel with it, so
-  // that neither affects the replay itself.
+  // is attached once the handle is back where it was.
   impl->SetDataSkipping(skipdata, skipleaway);
   impl->SetFullHash(fullhash);
 
@@ -310,18 +348,42 @@ void Par2Verifier::Restart(void)
     impl->Add(par2file, 0);
   }
 
-  impl->SetNoiseLevel(noiselevel);
-  impl->SetObserver(observer);
-
-  std::lock_guard<std::mutex> lock(cancelmutex);
-  restarting = false;
-
-  if (cancelled)
-    impl->Cancel();
-
   verified = false;
   scanned = false;
   repaired = false;
+
+  // A cancel reaches the rescan, which stops where it got to
+  {
+    std::lock_guard<std::mutex> lock(cancelmutex);
+    restarting = false;
+
+    if (cancelled)
+      impl->Cancel();
+  }
+
+  const std::set<std::string> rescan = scannedfiles;
+  scannedfiles.clear();
+
+  bool cutshort = false;
+
+  for (const auto &f : rescan)
+  {
+    if (impl->IsCancelled() || eCancelled == VerifyFile(f))
+    {
+      cutshort = true;
+      break;
+    }
+  }
+
+  // What the rescan did not reach is kept for the next one
+  if (cutshort)
+  {
+    scannedfiles = rescan;
+    verified = false;
+  }
+
+  impl->SetNoiseLevel(noiselevel);
+  impl->SetObserver(observer);
 }
 
 Par2Verifier::Par2Verifier(std::ostream &sout, std::ostream &serr, NoiseLevel noiselevel,
@@ -338,6 +400,7 @@ Par2Verifier::Par2Verifier(std::ostream &sout, std::ostream &serr, NoiseLevel no
 , skipleaway(DEFAULT_SKIP_LEAWAY)
 , fullhash(false)
 , par2files()
+, scannedfiles()
 , knownblocks()
 , verified(false)
 , scanned(false)
@@ -417,8 +480,10 @@ Result Par2Verifier::AddPar2File(const std::string &_parfilename)
   }
 
   // Extra recovery data leaves what the scan found still true, so it is kept
-  // and Reassess can use it. A set of a different shape does not.
-  if (scanned && setchanged)
+  // and Reassess can use it. A set of a different shape does not, even when the
+  // scan was cancelled. Files scanned before the set was known are replayed by
+  // the same restart.
+  if (setchanged && (scanned || !scannedfiles.empty()))
     Restart();
 
   return result;
@@ -476,8 +541,17 @@ bool Par2Verifier::SetKnownBlocks(const std::string &filename,
 
 Result Par2Verifier::Verify(const std::vector<std::string> &extrafiles)
 {
-  if (scanned)
+  // A full pass covers everything the individual scans did, so they are dropped
+  // rather than replayed into it. Whatever was scanned before, by a pass that
+  // was cancelled too, is started afresh.
+  scannedfiles.clear();
+
+  if (repaired)
     Restart();
+  else
+    impl->DiscardScans();
+
+  verified = false;
 
   const Result result = impl->Check(extrafiles, memorylimit, nthreads, filethreads);
 
@@ -490,6 +564,35 @@ Result Par2Verifier::Verify(const std::vector<std::string> &extrafiles)
   return result;
 }
 
+Result Par2Verifier::VerifyFile(const std::string &filename)
+{
+  // After a repair a new engine starts from nothing, and each file is scanned
+  // again as it is fed in
+  if (repaired)
+  {
+    scannedfiles.clear();
+    Restart();
+  }
+
+  const Result result = impl->Scan(filename, memorylimit, nthreads, filethreads);
+
+  if (result != eInsufficientCriticalData)
+    scanned = true;
+
+  if (result == eCancelled)
+    return result;
+
+  // Remembered even when the set is not known yet, so that adding the PAR2 file
+  // which describes it replays the scan rather than losing it. Scanning one
+  // again replaces what the last scan of it found, so it is remembered once.
+  scannedfiles.insert(filename);
+
+  if (result == eSuccess || result == eRepairPossible || result == eRepairNotPossible)
+    verified = true;
+
+  return result;
+}
+
 Result Par2Verifier::Repair(const bool verifyafter)
 {
   if (!verified)