From 1e01f35af1a35c7015e67143d3a36716ea28a360 Mon Sep 17 00:00:00 2001 From: mnightingale <9887246+mnightingale@users.noreply.github.com> Date: Sun, 23 Aug 2026 13:55:57 +0100 Subject: [PATCH] Let files be scanned one at a time as they arrive --- include/par2/libpar2.h | 29 ++++- src/diskfile.cpp | 10 ++ src/diskfile.h | 3 + src/libpar2.cpp | 153 ++++++++++++++++++++----- src/par2repairer.cpp | 197 ++++++++++++++++++++++++++++---- src/par2repairer.h | 18 ++- tests/consumer.cpp | 250 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 611 insertions(+), 49 deletions(-) diff --git a/include/par2/libpar2.h b/include/par2/libpar2.h index 49b4b2a8..7f5027b6 100644 --- a/include/par2/libpar2.h +++ b/include/par2/libpar2.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -277,16 +278,16 @@ class Par2Verifier // Look for blocks which are not where the set says they should be, the -N // option, with leaway the distance either side to search, the -S option. - // Zero leaway selects the default. Applies to Verify. + // Zero leaway selects the default. Applies to Verify and to VerifyFile. void SetDataSkipping(const bool enabled, const u64 leaway = 0); // Hash the whole of each file as well as its blocks, the --full-hash option. - // Applies to Verify. + // Applies to Verify and to VerifyFile. void SetFullHash(const bool enabled); // Threads for the main processing and for hashing files in parallel, the -t // and -T options. Either left zero stays at the default. They are read by - // the next Verify or Repair. + // the next Verify, VerifyFile or Repair. void SetThreadCounts(const u32 nthreads, const u32 filethreads); // Check the files described by the set against the data on disk. Returns @@ -297,6 +298,27 @@ class Par2Verifier // the results of the Verify that preceded it, so call them in that order. Result Verify(const std::vector &extrafiles = {}); + // Scan one file that has become available, matching it against the set the + // way a Verify would, without reading anything else. Use it to feed files in + // as they arrive rather than waiting for all of them. + // + // Safe to call again for the same file: whatever an earlier scan found for it + // is discarded first, so a file which was incomplete when it was first + // scanned can be scanned again once it is finished. Blocks another file + // supplied are left alone. + // + // The name may be one the set describes or one it does not; an unrecognised + // file is matched by content, as an extra file is. + // + // Returns what Verify would return for the set as it stands, so a file which + // has not been scanned yet still counts as missing. A later Verify replaces + // everything the individual scans found, and after a Repair they start again + // from nothing. + // + // May be called before any PAR2 file has been added: the result is then + // eInsufficientCriticalData, and the file is scanned once one arrives. + Result VerifyFile(const std::string &filename); + // The numbers behind the last Verify or Reassess. A repair is possible when // recoveryblockcount is at least missingblockcount, and needs // missingblockcount - recoveryblockcount more blocks when it is not. @@ -375,6 +397,7 @@ class Par2Verifier u64 skipleaway; bool fullhash; std::vector par2files; + std::set scannedfiles; std::map > knownblocks; bool verified; bool scanned; diff --git a/src/diskfile.cpp b/src/diskfile.cpp index 853251bb..923fd931 100644 --- a/src/diskfile.cpp +++ b/src/diskfile.cpp @@ -1216,6 +1216,16 @@ DiskFile* DiskFileMap::Find(std::string filename) const return (f != diskfilemap.end()) ? f->second : 0; } +std::vector DiskFileMap::Files(void) const +{ + std::vector files; + + for (const auto &f : diskfilemap) + files.push_back(f.second); + + return files; +} + FileSizeCache::FileSizeCache() { diff --git a/src/diskfile.h b/src/diskfile.h index 8f12a2d1..82668f18 100644 --- a/src/diskfile.h +++ b/src/diskfile.h @@ -159,6 +159,9 @@ class DiskFileMap void Remove(DiskFile *diskfile); DiskFile* Find(std::string filename) const; + // Every file in the map + std::vector Files(void) const; + protected: std::map diskfilemap; // Map from filename to DiskFile }; diff --git a/src/libpar2.cpp b/src/libpar2.cpp index af250020..913d77a2 100644 --- a/src/libpar2.cpp +++ b/src/libpar2.cpp @@ -166,6 +166,41 @@ class Par2Verifier::Impl : public Par2Repairer noiselevel = _noiselevel; } + // Forget every scan, keeping what the PAR2 files hold + void DiscardScans(void) + { + for (auto *sourcefile : sourcefiles) + { + if (0 == sourcefile) + continue; + + sourcefile->SetTargetFile(0); + sourcefile->SetTargetExists(false); + sourcefile->SetCompleteFile(0); + + if (sourcefile->GetDescriptionPacket() != 0) + { + auto block = sourcefile->SourceBlocks(); + for (u32 i = 0; i < sourcefile->BlockCount(); ++i, ++block) + { + if (block->IsSet()) + block->ClearLocation(); + } + } + } + + renamedlist.clear(); + + for (auto *diskfile : diskFileMap.Files()) + { + if (0 == packetfiles.count(diskfile)) + { + diskFileMap.Remove(diskfile); + delete diskfile; + } + } + } + void SetBasePath(const std::string &_basepath) { basepath = _basepath; @@ -189,15 +224,19 @@ class Par2Verifier::Impl : public Par2Repairer if (prepared != eSuccess) return prepared; - UpdateVerificationResults(); + return ScanOutcome(); + } - if (!CheckVerificationResults()) - return eRepairNotPossible; + Result Scan(const std::string &filename, const size_t memorylimit, + const u32 _nthreads, const u32 _filethreads) + { + if (prepared != eSuccess) + return prepared; - if (completefilecount < mainpacket->RecoverableFileCount()) - return eRepairPossible; + ApplyThreadCounts(_nthreads, _filethreads); + ApplyMemoryLimit(memorylimit); - return eSuccess; + return ScanFile(filename, basepath); } void SetDataSkipping(const bool _skipdata, const u64 _skipleaway) @@ -278,13 +317,13 @@ std::string BasePathFor(const std::string &parfilename) return basepath; } -// Verifying leaves a Par2Repairer with the results of that one pass, so a -// second pass has to start from a new one. The PAR2 files that were added are -// read again, which is cheap next to scanning the data files. +// A repair, or a set which changes shape after files were scanned, leaves a +// Par2Repairer with results which no longer hold, so the work starts again from +// a new one. The PAR2 files that were added are read again. void Par2Verifier::Restart(void) { - // A cancel which arrives while the engine is replaced and replayed is kept - // for the new one rather than given to either + // A cancel which arrives while the engine is replaced and the PAR2 files + // are read again is kept for the new one rather than given to either { std::lock_guard lock(cancelmutex); restarting = true; @@ -293,8 +332,7 @@ void Par2Verifier::Restart(void) // The replay repeats work the observer has already been told about, so it is // told none of it, and nothing is written to the streams again. The observer - // is attached once the handle is back where it was, and the cancel with it, so - // that neither affects the replay itself. + // is attached once the handle is back where it was. impl->SetDataSkipping(skipdata, skipleaway); impl->SetFullHash(fullhash); @@ -310,18 +348,42 @@ void Par2Verifier::Restart(void) impl->Add(par2file, 0); } - impl->SetNoiseLevel(noiselevel); - impl->SetObserver(observer); - - std::lock_guard lock(cancelmutex); - restarting = false; - - if (cancelled) - impl->Cancel(); - verified = false; scanned = false; repaired = false; + + // A cancel reaches the rescan, which stops where it got to + { + std::lock_guard lock(cancelmutex); + restarting = false; + + if (cancelled) + impl->Cancel(); + } + + const std::set rescan = scannedfiles; + scannedfiles.clear(); + + bool cutshort = false; + + for (const auto &f : rescan) + { + if (impl->IsCancelled() || eCancelled == VerifyFile(f)) + { + cutshort = true; + break; + } + } + + // What the rescan did not reach is kept for the next one + if (cutshort) + { + scannedfiles = rescan; + verified = false; + } + + impl->SetNoiseLevel(noiselevel); + impl->SetObserver(observer); } Par2Verifier::Par2Verifier(std::ostream &sout, std::ostream &serr, NoiseLevel noiselevel, @@ -338,6 +400,7 @@ Par2Verifier::Par2Verifier(std::ostream &sout, std::ostream &serr, NoiseLevel no , skipleaway(DEFAULT_SKIP_LEAWAY) , fullhash(false) , par2files() +, scannedfiles() , knownblocks() , verified(false) , scanned(false) @@ -417,8 +480,10 @@ Result Par2Verifier::AddPar2File(const std::string &_parfilename) } // Extra recovery data leaves what the scan found still true, so it is kept - // and Reassess can use it. A set of a different shape does not. - if (scanned && setchanged) + // and Reassess can use it. A set of a different shape does not, even when the + // scan was cancelled. Files scanned before the set was known are replayed by + // the same restart. + if (setchanged && (scanned || !scannedfiles.empty())) Restart(); return result; @@ -476,8 +541,17 @@ bool Par2Verifier::SetKnownBlocks(const std::string &filename, Result Par2Verifier::Verify(const std::vector &extrafiles) { - if (scanned) + // A full pass covers everything the individual scans did, so they are dropped + // rather than replayed into it. Whatever was scanned before, by a pass that + // was cancelled too, is started afresh. + scannedfiles.clear(); + + if (repaired) Restart(); + else + impl->DiscardScans(); + + verified = false; const Result result = impl->Check(extrafiles, memorylimit, nthreads, filethreads); @@ -490,6 +564,35 @@ Result Par2Verifier::Verify(const std::vector &extrafiles) return result; } +Result Par2Verifier::VerifyFile(const std::string &filename) +{ + // After a repair a new engine starts from nothing, and each file is scanned + // again as it is fed in + if (repaired) + { + scannedfiles.clear(); + Restart(); + } + + const Result result = impl->Scan(filename, memorylimit, nthreads, filethreads); + + if (result != eInsufficientCriticalData) + scanned = true; + + if (result == eCancelled) + return result; + + // Remembered even when the set is not known yet, so that adding the PAR2 file + // which describes it replays the scan rather than losing it. Scanning one + // again replaces what the last scan of it found, so it is remembered once. + scannedfiles.insert(filename); + + if (result == eSuccess || result == eRepairPossible || result == eRepairNotPossible) + verified = true; + + return result; +} + Result Par2Verifier::Repair(const bool verifyafter) { if (!verified) diff --git a/src/par2repairer.cpp b/src/par2repairer.cpp index 35e8657c..37e52f0a 100644 --- a/src/par2repairer.cpp +++ b/src/par2repairer.cpp @@ -79,6 +79,7 @@ Par2Repairer::Par2Repairer(std::ostream &sout, std::ostream &serr, const NoiseLe , par2list() , sourceblocks() , targetblocks() +, scanningprepared(false) , blockverifiable(false) , verificationhashtable() , unverifiablesourcefiles() @@ -221,6 +222,163 @@ void Par2Repairer::ApplyThreadCounts(const u32 _nthreads, const u32 _filethreads filethreads = std::max(1u, std::min(_filethreads != 0 ? _filethreads : filethreads, totalthreads)); } +// The hash table and window table are built from the packets loaded so far and +// are not rebuilt per scan: PrepareVerificationHashTable appends to +// unverifiablesourcefiles and loads the hash table, so calling it twice would +// duplicate both. +bool Par2Repairer::PrepareForScanning(void) +{ + if (scanningprepared) + return true; + + if (!PrepareVerificationHashTable()) + return false; + + if (!ComputeWindowTable()) + return false; + + scanningprepared = true; + + return true; +} + +void Par2Repairer::DiscardScannedFile(DiskFile *diskfile) +{ + for (auto *sourcefile : sourcefiles) + { + if (0 == sourcefile) + continue; + + if (sourcefile->GetTargetFile() == diskfile) + { + sourcefile->SetTargetFile(0); + sourcefile->SetTargetExists(false); + } + + if (sourcefile->GetCompleteFile() == diskfile) + { + sourcefile->SetCompleteFile(0); + renamedlist.erase(sourcefile->TargetFileName()); + } + + // Only the blocks this file supplied: another file may hold the rest + if (sourcefile->GetDescriptionPacket() != 0) + { + auto block = sourcefile->SourceBlocks(); + for (u32 i = 0; i < sourcefile->BlockCount(); ++i, ++block) + { + if (block->IsSet() && block->GetDiskFile() == diskfile) + block->ClearLocation(); + } + } + } + + diskFileMap.Remove(diskfile); + delete diskfile; +} + +Result Par2Repairer::ScanFile(const std::string &filename, const std::string &basepath) +{ + if (0 == mainpacket) + return eInsufficientCriticalData; + + if (!PrepareForScanning()) + return eLogicError; + + const std::string pathname = DiskFile::GetCanonicalPathname(filename); + + DiskFile *previous = diskFileMap.Find(pathname); + + // Or under the name the set records, as a Verify opens it + if (0 == previous) + { + auto expected = sourcefilesbytarget.find(pathname); + if (expected != sourcefilesbytarget.end()) + previous = expected->second->GetTargetFile(); + } + + // A PAR2 file whose packets have been read is not scanned as data, and the + // packets still hold its DiskFile + if (previous != 0 && packetfiles.count(previous) != 0) + { + return ScanOutcome(false); + } + + if (previous != 0) + DiscardScannedFile(previous); + + // Which source file the set expects at this path, if any + auto expected = sourcefilesbytarget.find(pathname); + Par2RepairerSourceFile *sourcefile = (expected == sourcefilesbytarget.end()) ? 0 : expected->second; + + DiskFile *diskfile = new DiskFile(sout, serr); + if (!diskfile->Open(pathname)) + { + delete diskfile; + return ScanOutcome(false); + } + + if (!diskFileMap.Insert(diskfile)) + { + diskfile->Close(); + delete diskfile; + + return eLogicError; + } + + // The file at the name the set records is matched in its own right, as a + // Verify matches it before a copy found under another name + DiskFile *copy = 0; + if (0 != sourcefile) + { + sourcefile->SetTargetExists(true); + sourcefile->SetTargetFile(diskfile); + + copy = sourcefile->GetCompleteFile(); + sourcefile->SetCompleteFile(0); + } + + ProgressMeter progress(sout, "Scanning: ", diskfile->FileSize(), noiselevel, observer); + + ResetScanBuffers(1, diskfile->FileSize()); + + VerifyDataFile(diskfile, sourcefile, basepath, progress); + + diskfile->Close(); + + if (0 != sourcefile && 0 == sourcefile->GetCompleteFile()) + sourcefile->SetCompleteFile(copy); + + // Nothing is scanned again until the next file arrives, so the buffers are + // given up rather than held against the memory a repair needs + scanbuffers.Reset(0, 0); + + // A cancelled scan is discarded, leaving the file as if it had not been scanned + if (IsCancelled()) + { + DiscardScannedFile(diskfile); + return eCancelled; + } + + return ScanOutcome(false); +} + +Result Par2Repairer::ScanOutcome(const bool report) +{ + UpdateVerificationResults(); + + // CheckVerificationResults writes the summary out and comes to the same answer + const bool possible = report ? CheckVerificationResults() + : recoverypacketmap.size() >= missingblockcount; + if (!possible) + return eRepairNotPossible; + + if (completefilecount < mainpacket->RecoverableFileCount()) + return eRepairPossible; + + return eSuccess; +} + // Verify the source files and work out whether a repair is needed or possible Result Par2Repairer::VerifyFiles(const std::string &basepath, std::vector &extrafiles, @@ -228,14 +386,7 @@ Result Par2Repairer::VerifyFiles(const std::string &basepath, { renamedlist.clear(); - // Create a verification hash table for all files for which we have not - // found a complete version of the file and for which we have - // a verification packet - if (!PrepareVerificationHashTable()) - return eLogicError; - - // Compute the table for the sliding CRC computation - if (!ComputeWindowTable()) + if (!PrepareForScanning()) return eLogicError; ResetScanBuffers(std::max(sourcefiles.size(), extrafiles.size())); @@ -251,9 +402,6 @@ Result Par2Repairer::VerifyFiles(const std::string &basepath, return IsCancelled() ? eCancelled : eLogicError; } - // Find out how much data we have found - UpdateVerificationResults(); - // Nothing is scanned again until the repaired files are verified, so the // buffers are given up rather than held against the memory a repair needs scanbuffers.Reset(0, 0); @@ -262,13 +410,7 @@ Result Par2Repairer::VerifyFiles(const std::string &basepath, sout << '\n'; // Check the verification results and report the results - if (!CheckVerificationResults()) - return eRepairNotPossible; - // Are any of the files incomplete - if (completefilecount < mainpacket->RecoverableFileCount()) - return eRepairPossible; - - return eSuccess; + return ScanOutcome(); } // Rebuild whatever is missing or damaged @@ -658,6 +800,7 @@ Result Par2Repairer::PreparePackets(void) { sourcefiles.clear(); sourcefilesbyname.clear(); + sourcefilesbytarget.clear(); // Check that the packets are consistent and discard any that are not if (!CheckPacketConsistency()) @@ -682,6 +825,16 @@ Result Par2Repairer::PreparePackets(void) sourcefilesbyname.insert(std::make_pair(sourcefile->GetDescriptionPacket()->FileName(), sourcefile)); } + // The source file each canonical path belongs to, for finding the one a + // scanned file is expected to be + for (auto *sourcefile : sourcefiles) + { + if (0 == sourcefile || 0 == sourcefile->GetDescriptionPacket()) + continue; + + sourcefilesbytarget.insert(std::make_pair(DiskFile::GetCanonicalPathname(sourcefile->TargetFileName()), sourcefile)); + } + if (observer) { Par2SetInfo info; @@ -3298,7 +3451,7 @@ bool Par2Repairer::ComputeRSmatrix(void) // take more than the memory limit between them, and never fewer than one each. // The buffers are given up before a repair allocates the ones it works // through, so the two never hold that memory at the same time. -void Par2Repairer::ResetScanBuffers(const size_t filecount) +void Par2Repairer::ResetScanBuffers(const size_t filecount, const u64 filesize) { // The blocks of a file are only checked where they are expected to be when // there are verification packets to check them against and more than one @@ -3322,7 +3475,11 @@ void Par2Repairer::ResetScanBuffers(const size_t filecount) const u32 readers = FileThreads(filecount); const u32 workers = std::max(1u, totalthreads / readers); - const size_t affordable = std::max(1, scanmemorylimit / (2 * readers) / (size_t)blocksize); + size_t affordable = std::max(1, scanmemorylimit / (2 * readers) / (size_t)blocksize); + + // A file needs no more than its own blocks + if (filesize > 0) + affordable = std::min(affordable, (size_t)((filesize + blocksize - 1) / blocksize)); const u32 blocksperbatch = (u32)std::min((size_t)workers * SCAN_BATCH_PER_THREAD, affordable); diff --git a/src/par2repairer.h b/src/par2repairer.h index af2480e1..7e01dcf0 100644 --- a/src/par2repairer.h +++ b/src/par2repairer.h @@ -115,6 +115,9 @@ class Par2Repairer // Apply the -m memory limit, which bounds the buffers a scan reads into void ApplyMemoryLimit(const size_t _memorylimit) {scanmemorylimit = _memorylimit;} + // Scan one file, replacing whatever an earlier scan of it found + Result ScanFile(const std::string &filename, const std::string &basepath); + // Verify the source files and work out whether a repair is needed Result VerifyFiles(const std::string &basepath, std::vector &extrafiles, @@ -170,6 +173,17 @@ class Par2Repairer // Scan any extra files specified on the command line bool VerifyExtraFiles(const std::vector &extrafiles, const std::string &basepath, const bool renameonly); + // Set up the tables a scan needs, once + bool PrepareForScanning(void); + + // Forget what a scan of this file found: the blocks it supplied and its + // place as a target or complete file + void DiscardScannedFile(DiskFile *diskfile); + + // What the files scanned so far add up to, as a verify of the set says it, + // with the summary written out when report is set + Result ScanOutcome(const bool report = true); + // Attempt to match the data in the DiskFile with the source file, reporting // the file to the observer for as long as the match takes bool VerifyDataFile(DiskFile *diskfile, Par2RepairerSourceFile *sourcefile, const std::string &basepath, ProgressMeter &progress, const bool renameonly = false); @@ -242,7 +256,7 @@ class Par2Repairer // Make the buffers the files being scanned read into, or give them up when // no file will have its blocks checked where they are expected to be - void ResetScanBuffers(const size_t filecount); + void ResetScanBuffers(const size_t filecount, const u64 filesize = 0); // The number of files to read at once, which is what limits how many are // open at a time rather than how much of the work they get @@ -306,6 +320,7 @@ class Par2Repairer std::map sourcefilemap;// Map from FileId to SourceFile std::vector sourcefiles; // The source files + std::map sourcefilesbytarget; // The source file expected at each canonical path std::vector verifylist; // Those source files that are being repaired std::vector backuplist; // Those source files backups std::map backupnames; // What each backup was called before it was renamed @@ -325,6 +340,7 @@ class Par2Repairer u32 windowtable[256]; // Table for sliding CRCs + bool scanningprepared; // Whether the tables a scan needs have been built bool blockverifiable; // Whether and files can be verified at the block level VerificationHashTable verificationhashtable; // Hash table for block verification std::list unverifiablesourcefiles; // Files that are not block verifiable diff --git a/tests/consumer.cpp b/tests/consumer.cpp index 5f1f3679..8cd36398 100644 --- a/tests/consumer.cpp +++ b/tests/consumer.cpp @@ -796,6 +796,48 @@ int main() std::remove(data); } + // A file scanned again once it no longer holds the set's data stops being + // reported as that file under another name + { + Check(MakeDirectory("restaledir"), "mkdir for the rescanned rename check"); + + const char *const data = "restaledir/proper.data"; + const char *const obfuscated = "restaledir/c4d2e19a.dat"; + + WriteData(data, 53, 12000); + + std::vector files; + files.emplace_back(data); + Check(par2::eSuccess == par2::par2create(quiet, quiet, par2::nlSilent, + 64 * 1024 * 1024, "restaledir/", 0, 2, + "restaledir/stale", files, BLOCKSIZE, 0, + par2::scUniform, 1, 4), + "par2create for the rescanned rename check"); + + std::rename(data, obfuscated); + + par2::Par2Verifier verifier(quiet, quiet, par2::nlSilent, "restaledir/"); + Check(par2::eSuccess == verifier.AddPar2File("restaledir/stale.par2"), + "AddPar2File for the rescanned rename check"); + Check(par2::eRepairPossible == verifier.VerifyFile(obfuscated), + "the file is found under its other name"); + + std::vector > renamed; + Check(verifier.GetRenamedFiles(&renamed) && renamed.size() == 1, + "and reported as renamed"); + + // Its contents change, and it is scanned again + WriteData(obfuscated, 99, 12000); + verifier.VerifyFile(obfuscated); + + Check(verifier.GetRenamedFiles(&renamed), "GetRenamedFiles after scanning it again"); + Check(renamed.empty(), "it is no longer reported as renamed"); + + std::remove(obfuscated); + std::remove("restaledir/stale.par2"); + std::remove("restaledir/stale.vol0+4.par2"); + } + // With no basepath the set is resolved beside its PAR2 files, as the tool // does, rather than against the working directory { @@ -986,6 +1028,214 @@ int main() WriteData(DATA[i], (unsigned)i, 20000 + i * 9000); } + // Files fed in one at a time as they arrive, including one which is the right + // size but has not finished downloading when it is first scanned + { + Check(MakeDirectory("arrivedir"), "mkdir for the incremental check"); + + const char *const arriving[] = {"arrivedir/arrive-0.data", + "arrivedir/arrive-1.data", + "arrivedir/arrive-2.data"}; + const size_t arrivingcount = sizeof(arriving) / sizeof(arriving[0]); + + std::vector files; + for (size_t i = 0; i < arrivingcount; ++i) + { + WriteData(arriving[i], (unsigned)(91 + i), 30000); + files.emplace_back(arriving[i]); + } + Check(par2::eSuccess == par2::par2create(quiet, quiet, par2::nlSilent, + 64 * 1024 * 1024, "arrivedir/", 0, 2, + "arrivedir/arrive", files, BLOCKSIZE, 0, + par2::scUniform, 1, 20), + "par2create for the incremental check"); + + // Only the third file is on disk, and it is the right size with a hole in + // the middle, which is what a download in progress looks like + Corrupt(arriving[2], 12000, 6000); + std::remove(arriving[0]); + std::remove(arriving[1]); + + par2::Par2Verifier verifier(quiet, quiet, par2::nlSilent, "arrivedir/"); + Check(par2::eSuccess == verifier.AddPar2File("arrivedir/arrive.par2"), + "AddPar2File for the incremental check"); + + par2::Par2VerifyResult r{}; + + // The incomplete one, scanned while it is still a hole + verifier.VerifyFile(arriving[2]); + Check(verifier.GetVerifyResult(&r), "GetVerifyResult after the first scan"); + Check(r.completefilecount == 0, "nothing is complete yet"); + Check(r.damagedfilecount == 1, "the file on disk is damaged"); + Check(r.availableblockcount > 0, "but some of its blocks are usable"); + + const par2::u32 partial = r.availableblockcount; + + // It finishes downloading, and is scanned again + WriteData(arriving[2], (unsigned)93, 30000); + verifier.VerifyFile(arriving[2]); + Check(verifier.GetVerifyResult(&r), "GetVerifyResult after the rescan"); + Check(r.completefilecount == 1, "the finished file is now complete"); + Check(r.damagedfilecount == 0, "and no longer counts as damaged"); + Check(r.availableblockcount > partial, "the blocks it was missing are there"); + + // The other two arrive + WriteData(arriving[0], (unsigned)91, 30000); + verifier.VerifyFile(arriving[0]); + WriteData(arriving[1], (unsigned)92, 30000); + Check(par2::eSuccess == verifier.VerifyFile(arriving[1]), + "the set is complete once the last file is scanned"); + + Check(verifier.GetVerifyResult(&r), "GetVerifyResult at the end"); + Check(r.completefilecount == arrivingcount, "every file is complete"); + Check(r.missingblockcount == 0, "and nothing is missing"); + + for (const char *name : arriving) + std::remove(name); + } + + // A Verify after files were fed in one at a time starts again from what is on + // disk, rather than adding to what the scans found + { + Check(MakeDirectory("rescandir"), "mkdir for the verify after scans check"); + + const char *const names[] = {"rescandir/rescan-0.data", "rescandir/rescan-1.data"}; + + std::vector files; + for (size_t i = 0; i < 2; ++i) + { + WriteData(names[i], (unsigned)(71 + i), 30000); + files.emplace_back(names[i]); + } + Check(par2::eSuccess == par2::par2create(quiet, quiet, par2::nlSilent, + 64 * 1024 * 1024, "rescandir/", 0, 2, + "rescandir/rescan", files, BLOCKSIZE, 0, + par2::scUniform, 1, 20), + "par2create for the verify after scans check"); + + par2::Par2Verifier verifier(quiet, quiet, par2::nlSilent, "rescandir/"); + Check(par2::eSuccess == verifier.AddPar2File("rescandir/rescan.par2"), + "AddPar2File for the verify after scans check"); + verifier.VerifyFile(names[0]); + Check(par2::eSuccess == verifier.VerifyFile(names[1]), + "both files are intact when they are fed in"); + + Corrupt(names[0], 5000, 2000); + + par2::Par2VerifyResult r{}; + Check(par2::eRepairPossible == verifier.Verify(), + "a Verify after the scans finds the damage done since"); + Check(verifier.GetVerifyResult(&r), "GetVerifyResult after the verify"); + Check(r.completefilecount == 1 && r.damagedfilecount == 1, + "one file is intact and one damaged, each counted once"); + + for (const char *name : names) + std::remove(name); + } + + // A PAR2 file fed in along with the data is left alone, since a repair + // reads the recovery data through the packets read from it + { + Check(MakeDirectory("feeddir"), "mkdir for the fed PAR2 file check"); + + const char *const data = "feeddir/feed.data"; + + WriteData(data, 27, 40000); + + std::vector files; + files.emplace_back(data); + Check(par2::eSuccess == par2::par2create(quiet, quiet, par2::nlSilent, + 64 * 1024 * 1024, "feeddir/", 0, 2, + "feeddir/feed", files, BLOCKSIZE, 0, + par2::scUniform, 1, 8), + "par2create for the fed PAR2 file check"); + + // Named in full, as an application working in absolute paths names them, + // which is also how the volumes found beside the set are recorded + std::string dir; + { + par2::Par2Verifier probe(quiet, quiet, par2::nlSilent, "feeddir/"); + std::vector info; + Check(par2::eSuccess == probe.AddPar2File("feeddir/feed.par2") && + probe.GetFileInfo(&info) && info.size() == 1, + "GetFileInfo for the fed PAR2 file check"); + if (info.size() == 1) + dir = info[0].localfilename.substr(0, info[0].localfilename.size() - + std::string("feed.data").size()); + } + + Corrupt(data, 1000, 5000); + + par2::Par2Verifier verifier(quiet, quiet, par2::nlSilent, dir); + Check(par2::eSuccess == verifier.AddPar2File(dir + "feed.par2"), + "AddPar2File for the fed PAR2 file check"); + Check(par2::eRepairPossible == verifier.VerifyFile(dir + "feed.data"), + "the damaged data is found"); + Check(par2::eRepairPossible == verifier.VerifyFile(dir + "feed.vol0+8.par2"), + "a volume fed in too is not taken for data"); + Check(par2::eRepairPossible == verifier.VerifyFile(dir + "feed.par2"), + "nor is the index file"); + Check(par2::eSuccess == verifier.Repair(), + "and the repair still reads its recovery data through them"); + + // With the set whole, a file which adds nothing says so as a verify would + Check(par2::eSuccess == verifier.VerifyFile(dir + "feed.data"), + "the repaired data fed in again is whole"); + Check(par2::eSuccess == verifier.VerifyFile(dir + "feed.par2"), + "a PAR2 file fed in once the set is whole reports it whole"); + Check(par2::eSuccess == verifier.VerifyFile(dir + "absent.data"), + "and so does a file which is not there"); + + par2::Par2Verifier after(quiet, quiet, par2::nlSilent, "feeddir/"); + Check(par2::eSuccess == after.AddPar2File("feeddir/feed.par2"), + "AddPar2File after the fed repair"); + Check(par2::eSuccess == after.Verify(), "the data is whole again"); + + std::remove(data); + std::remove("feeddir/feed.data.1"); + std::remove("feeddir/feed.par2"); + std::remove("feeddir/feed.vol0+8.par2"); + } + + // A data file which arrives before the PAR2 file describing it + { + Check(MakeDirectory("firstdir"), "mkdir for the ordering check"); + + const char *const early = "firstdir/early.data"; + const char *const late = "firstdir/late.data"; + + WriteData(early, 95, 30000); + WriteData(late, 96, 30000); + + std::vector files; + files.emplace_back(early); + files.emplace_back(late); + Check(par2::eSuccess == par2::par2create(quiet, quiet, par2::nlSilent, + 64 * 1024 * 1024, "firstdir/", 0, 2, + "firstdir/first", files, BLOCKSIZE, 0, + par2::scUniform, 1, 20), + "par2create for the ordering check"); + + std::remove(late); + + par2::Par2Verifier verifier(quiet, quiet, par2::nlSilent, "firstdir/"); + + // Nothing describes it yet, so it cannot be scanned + Check(par2::eInsufficientCriticalData == verifier.VerifyFile(early), + "a scan before the set is known says so"); + + // The PAR2 file arrives and the earlier scan is replayed against it + Check(par2::eSuccess == verifier.AddPar2File("firstdir/first.par2"), + "AddPar2File for the ordering check"); + + par2::Par2VerifyResult r{}; + Check(verifier.GetVerifyResult(&r), "the replayed scan counts as a verify"); + Check(r.completefilecount == 1, "the file scanned first was found"); + Check(r.missingfilecount == 1, "and the one never scanned is missing"); + + std::remove(early); + } + // The implementations an application supplies reach the work the handle does, // rather than being dropped in favour of the ones built in {