diff --git a/Cargo.lock b/Cargo.lock index 92026cb5b5d..f468853a69c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -576,7 +576,7 @@ dependencies = [ "bitflags 2.13.0", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "regex", @@ -1229,7 +1229,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2493,7 +2493,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2554,7 +2554,7 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2973,7 +2973,7 @@ dependencies = [ [[package]] name = "grovedb" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "axum 0.8.9", "bincode", @@ -2987,6 +2987,7 @@ dependencies = [ "grovedb-merk", "grovedb-merkle-mountain-range", "grovedb-path", + "grovedb-private-document-store", "grovedb-query", "grovedb-storage", "grovedb-version", @@ -3011,7 +3012,7 @@ dependencies = [ [[package]] name = "grovedb-bulk-append-tree" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "bincode", "blake3", @@ -3020,19 +3021,22 @@ dependencies = [ "grovedb-merkle-mountain-range", "grovedb-query", "grovedb-storage", + "grovedb-version", "hex", + "integer-encoding", "thiserror 2.0.18", ] [[package]] name = "grovedb-commitment-tree" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "blake3", "grovedb-bulk-append-tree", "grovedb-costs", "grovedb-storage", + "grovedb-version", "incrementalmerkletree", "orchard", "rusqlite", @@ -3043,7 +3047,7 @@ dependencies = [ [[package]] name = "grovedb-costs" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "integer-encoding", "intmap", @@ -3053,24 +3057,26 @@ dependencies = [ [[package]] name = "grovedb-dense-fixed-sized-merkle-tree" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "bincode", "blake3", "grovedb-costs", "grovedb-query", "grovedb-storage", + "grovedb-version", "thiserror 2.0.18", ] [[package]] name = "grovedb-element" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "bincode", "bincode_derive", "grovedb-path", + "grovedb-query", "grovedb-version", "grovedb-visualize", "hex", @@ -3081,7 +3087,7 @@ dependencies = [ [[package]] name = "grovedb-epoch-based-storage-flags" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "grovedb-costs", "hex", @@ -3093,7 +3099,7 @@ dependencies = [ [[package]] name = "grovedb-merk" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "bincode", "bincode_derive", @@ -3119,26 +3125,41 @@ dependencies = [ [[package]] name = "grovedb-merkle-mountain-range" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "bincode", "blake3", "grovedb-costs", "grovedb-storage", + "grovedb-version", + "integer-encoding", ] [[package]] name = "grovedb-path" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "hex", ] +[[package]] +name = "grovedb-private-document-store" +version = "5.0.1" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" +dependencies = [ + "blake3", + "grovedb-bulk-append-tree", + "grovedb-costs", + "grovedb-storage", + "grovedb-version", + "thiserror 2.0.18", +] + [[package]] name = "grovedb-query" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "bincode", "byteorder", @@ -3154,7 +3175,7 @@ dependencies = [ [[package]] name = "grovedb-storage" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "blake3", "grovedb-costs", @@ -3173,7 +3194,7 @@ dependencies = [ [[package]] name = "grovedb-version" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "thiserror 2.0.18", "versioned-feature-core 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3182,7 +3203,7 @@ dependencies = [ [[package]] name = "grovedb-visualize" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "hex", "itertools 0.14.0", @@ -3191,7 +3212,7 @@ dependencies = [ [[package]] name = "grovedbg-types" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=a2791bbdca756d6a6113024aec48f09f7a33faa9#a2791bbdca756d6a6113024aec48f09f7a33faa9" +source = "git+https://github.com/dashpay/grovedb?rev=753a11f14c9a4bc72bf2d5302751dd43d174621e#753a11f14c9a4bc72bf2d5302751dd43d174621e" dependencies = [ "serde", "serde_with 3.21.0", @@ -3607,7 +3628,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "system-configuration", "tokio", "tower-service", @@ -3858,7 +3879,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5560,7 +5581,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck 0.4.1", - "itertools 0.10.5", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -5581,7 +5602,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.117", @@ -5594,7 +5615,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.117", @@ -5730,7 +5751,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.2", "rustls", - "socket2 0.5.10", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -5768,9 +5789,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.4", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6578,7 +6599,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6591,7 +6612,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6650,7 +6671,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -7510,7 +7531,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -8959,7 +8980,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/book/src/drive/ranked-index-examples.md b/book/src/drive/ranked-index-examples.md index 20798741803..c702e51fa28 100644 --- a/book/src/drive/ranked-index-examples.md +++ b/book/src/drive/ranked-index-examples.md @@ -321,11 +321,11 @@ The response carries the skip back in `RankedEntries.skipped` (see [The Response Three properties worth stating plainly: -- **The skip is attested, not walked.** grovedb proves the skipped region from the counted subtree commitments (`HashWithCount` / `HashWithCountAndSum`) rather than by traversing it. Both the prover's work and the proof's size stay `O(log n + k)` **at any offset**. -- **There is therefore no offset ceiling.** An offset of 4 and an offset of four billion cost the same, so there is no denial-of-service lever a cap would close — and a cap would only stop honest deep pagination. -- **An offset past the end is a positive answer.** `entries` comes back empty and `skipped` is the ranking's *entire attested population*. "There are only 12 groups" is more information than a bare empty list. +- **The skip is counted, not walked.** grovedb descends the secondary reading each subtree's aggregate count and collapses any subtree that fits entirely inside the remaining offset, instead of stepping through it. Both paths do this: the prover attests the skipped region from the counted subtree commitments (`HashWithCount` / `HashWithCountAndSum`), and the unproven read performs the same counted descent without building a proof. Work and proof size stay `O(log n + k)` **at any offset**. +- **There is therefore no offset ceiling.** An offset of 4 and an offset of four billion cost the same order of work — on either path, the deeper one in fact cheaper, since a tree that fits entirely inside the offset collapses at the root. There is no denial-of-service lever a cap would close, and a cap would only stop honest deep pagination. +- **An offset past the end is a positive answer.** `entries` comes back empty and `skipped` is the ranking's *entire reported population*. "There are only 12 groups" is more information than a bare empty list. -On the **unproven** read there is nothing to attest and grovedb's read API does not report a short walk, so `skipped` simply echoes the requested offset. The proved and unproven paths therefore disagree in exactly one case — an offset past the end, where the unproven read reports the request and the proved one reports the truth. **Callers who need the population must prove.** +Both paths report the same `skipped`: the offset you asked for when the skip succeeded, and the ranking's total population when the walk ran out of groups first. What differs is the warrant, not the value. On the proved path it is cryptographically attested, re-derived by the verifier from the counted commitments. On the unproven path it is an **unverified claim, exactly like the entries beside it** — equal to the attested value on an honest node, with nothing forcing a node to be honest. **Callers who need to trust the population, rather than merely receive it, must still prove.** ## The Response diff --git a/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h b/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h index c4e66dadcb9..c37388a2691 100644 --- a/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h +++ b/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h @@ -3155,13 +3155,17 @@ GPB_FINAL @interface GetDocumentsRequest_GetDocumentsRequestV1 : GPBMessage * routes to the ranked executor (`group_by` + a single `order_by` * naming the selected aggregate), `offset` skips that many ranks * before the returned page, so `ORDER BY avg(grade) DESC LIMIT 1 - * OFFSET 4` is the 5th-best group. The skip is **count-attested**, - * not walked: grovedb proves it from the counted subtree - * commitments, so the proof stays `O(log n + k)` at any offset and - * the response echoes the attested number in - * `RankedEntries.skipped`. There is deliberately no ceiling — an - * offset of 4 and an offset of four billion cost the same, so - * there is no denial-of-service lever a cap would close. An offset + * OFFSET 4` is the 5th-best group. The skip is **counted, not + * walked**: grovedb descends on each subtree's aggregate count and + * collapses whole subtrees that fit inside the remaining offset, so + * the work stays `O(log n + k)` at any offset and the response + * reports the skip it performed in `RankedEntries.skipped`. On a + * proved request that count is additionally *attested* — committed + * to by the proof and re-derived by the verifier; on an unproved + * one it is the node's own report. See `RankedEntries.skipped`. There is deliberately no ceiling — an + * offset of 4 and an offset of four billion cost the same *order* + * of work — neither walks the region it skips — so there is no + * denial-of-service lever a cap would close. An offset * past the end of the ranking is a provable answer rather than an * error: `entries` comes back empty and `skipped` is the ranking's * whole population. @@ -3842,20 +3846,31 @@ GPB_FINAL @interface GetDocumentsResponse_GetDocumentsResponseV1_RankedEntries : * group rather than the best. * * **When a requested offset exceeds the population**, `entries` - * is empty and `skipped` is the ranking's attested *total* + * is empty and `skipped` is the ranking's *total* reported * population — a positive, useful answer ("there are only 12 * groups") rather than a bare empty list. * - * On the proved path the number is grovedb's cryptographically - * attested count, re-derived by the verifier from the counted - * subtree commitments in the proof bytes rather than trusted - * from this field; a proving client should use the verified - * value. On the unproven read there is nothing to attest and - * grovedb's read API does not report a short walk, so the server - * echoes the requested offset. The two therefore disagree in - * exactly one case — an offset past the end, where the unproven - * read reports the request and the proved one reports the truth. - * Callers who need the population must prove. + * Both paths report the same quantity: the offset you asked for + * when the skip succeeded, and the ranking's total population + * when the walk ran out of groups first. They no longer disagree + * anywhere, including past the end. + * + * What differs is the *warrant*, not the value. On the proved + * path the number is cryptographically attested — re-derived by + * the verifier from the counted subtree commitments in the proof + * bytes rather than trusted from this field — so a proving client + * should use the verified value and ignore this one. On the + * unproven path it is an **unverified claim**, exactly like the + * entries beside it: it equals the attested value on an honest + * node, and nothing forces a node to be honest. Read "the true + * population" as "what this node says the population is". + * Callers who need to trust it, rather than merely receive it, + * must still prove. + * + * Do not assume this field equals the offset you requested. It + * equals the offset only when the skip succeeded; when the walk + * ran out of groups first it is smaller, and that is the answer + * rather than an inconsistency. **/ @property(nonatomic, readwrite) uint64_t skipped; diff --git a/packages/dapi-grpc/protos/platform/v0/platform.proto b/packages/dapi-grpc/protos/platform/v0/platform.proto index 937962817c3..0ad3a197fd0 100644 --- a/packages/dapi-grpc/protos/platform/v0/platform.proto +++ b/packages/dapi-grpc/protos/platform/v0/platform.proto @@ -1125,13 +1125,17 @@ message GetDocumentsRequest { // routes to the ranked executor (`group_by` + a single `order_by` // naming the selected aggregate), `offset` skips that many ranks // before the returned page, so `ORDER BY avg(grade) DESC LIMIT 1 - // OFFSET 4` is the 5th-best group. The skip is **count-attested**, - // not walked: grovedb proves it from the counted subtree - // commitments, so the proof stays `O(log n + k)` at any offset and - // the response echoes the attested number in - // `RankedEntries.skipped`. There is deliberately no ceiling — an - // offset of 4 and an offset of four billion cost the same, so - // there is no denial-of-service lever a cap would close. An offset + // OFFSET 4` is the 5th-best group. The skip is **counted, not + // walked**: grovedb descends on each subtree's aggregate count and + // collapses whole subtrees that fit inside the remaining offset, so + // the work stays `O(log n + k)` at any offset and the response + // reports the skip it performed in `RankedEntries.skipped`. On a + // proved request that count is additionally *attested* — committed + // to by the proof and re-derived by the verifier; on an unproved + // one it is the node's own report. See `RankedEntries.skipped`. There is deliberately no ceiling — an + // offset of 4 and an offset of four billion cost the same *order* + // of work — neither walks the region it skips — so there is no + // denial-of-service lever a cap would close. An offset // past the end of the ranking is a provable answer rather than an // error: `entries` comes back empty and `skipped` is the ranking's // whole population. @@ -1417,20 +1421,31 @@ message GetDocumentsResponse { // group rather than the best. // // **When a requested offset exceeds the population**, `entries` - // is empty and `skipped` is the ranking's attested *total* + // is empty and `skipped` is the ranking's *total* reported // population — a positive, useful answer ("there are only 12 // groups") rather than a bare empty list. // - // On the proved path the number is grovedb's cryptographically - // attested count, re-derived by the verifier from the counted - // subtree commitments in the proof bytes rather than trusted - // from this field; a proving client should use the verified - // value. On the unproven read there is nothing to attest and - // grovedb's read API does not report a short walk, so the server - // echoes the requested offset. The two therefore disagree in - // exactly one case — an offset past the end, where the unproven - // read reports the request and the proved one reports the truth. - // Callers who need the population must prove. + // Both paths report the same quantity: the offset you asked for + // when the skip succeeded, and the ranking's total population + // when the walk ran out of groups first. They no longer disagree + // anywhere, including past the end. + // + // What differs is the *warrant*, not the value. On the proved + // path the number is cryptographically attested — re-derived by + // the verifier from the counted subtree commitments in the proof + // bytes rather than trusted from this field — so a proving client + // should use the verified value and ignore this one. On the + // unproven path it is an **unverified claim**, exactly like the + // entries beside it: it equals the attested value on an honest + // node, and nothing forces a node to be honest. Read "the true + // population" as "what this node says the population is". + // Callers who need to trust it, rather than merely receive it, + // must still prove. + // + // Do not assume this field equals the offset you requested. It + // equals the offset only when the skip succeeded; when the walk + // ran out of groups first it is smaller, and that is the answer + // rather than an inconsistency. optional uint64 skipped = 2 [jstype = JS_STRING]; } diff --git a/packages/dash-platform-queries/src/documents/document_query.rs b/packages/dash-platform-queries/src/documents/document_query.rs index 113e6e1e478..d7c7bf95908 100644 --- a/packages/dash-platform-queries/src/documents/document_query.rs +++ b/packages/dash-platform-queries/src/documents/document_query.rs @@ -344,9 +344,15 @@ impl DocumentQuery { /// Only the ranked surface honours it (see /// [`Self::order_by_selected_aggregate`]); on every other path the /// server rejects a set offset with `Unsupported`. There is no - /// ceiling: grovedb attests the skipped region from counted - /// subtree commitments instead of walking it, so a deep offset - /// costs exactly what a shallow one does. + /// ceiling: grovedb counts the skipped region from the subtree + /// aggregates instead of walking it, on both `prove` settings, so + /// the cost of a deep offset does not scale with the offset. It is + /// not identical to a shallow one — `offset = 0` keeps a sequential + /// fast path, a positive offset descends the tree in `O(log n)`, + /// and an offset at or past the population is answered from the + /// root without descending at all — but nothing here grows with how + /// far you page, which is why there is no ceiling. Only a proved + /// response additionally attests the count. /// /// An offset past the end of the ranking is a legitimate answer /// rather than an error — the page comes back empty, and on a diff --git a/packages/dash-platform-queries/src/documents/document_ranked_entries.rs b/packages/dash-platform-queries/src/documents/document_ranked_entries.rs index 5f9461e1ea3..1f351f2c4b9 100644 --- a/packages/dash-platform-queries/src/documents/document_ranked_entries.rs +++ b/packages/dash-platform-queries/src/documents/document_ranked_entries.rs @@ -340,9 +340,11 @@ mod tests { } /// An offset far past any plausible population is **not** capped: - /// grovedb attests the skipped region from counted commitments - /// rather than walking it, so a deep page costs what a shallow one - /// does and there is nothing for a cap to protect. + /// grovedb counts the skipped region from the subtree aggregates + /// rather than walking it, on both `prove` settings, so the cost of + /// a deep page does not grow with the offset — `O(log n)` in the + /// size of the ranking, not in how far you page — and there is + /// nothing for a cap to protect. #[test] fn a_very_deep_offset_is_not_capped() { let query = top_five_by_avg_grade().with_limit(1).with_offset(u32::MAX); diff --git a/packages/rs-dpp/Cargo.toml b/packages/rs-dpp/Cargo.toml index 7a1c3bf85f7..0759b188069 100644 --- a/packages/rs-dpp/Cargo.toml +++ b/packages/rs-dpp/Cargo.toml @@ -71,7 +71,7 @@ strum = { version = "0.26", features = ["derive"] } json-schema-compatibility-validator = { path = '../rs-json-schema-compatibility-validator', optional = true } once_cell = "1.19.0" tracing = { version = "0.1.41" } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "a2791bbdca756d6a6113024aec48f09f7a33faa9", optional = true } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "753a11f14c9a4bc72bf2d5302751dd43d174621e", optional = true } [dev-dependencies] tokio = { version = "1.40", features = ["full"] } diff --git a/packages/rs-drive-abci/Cargo.toml b/packages/rs-drive-abci/Cargo.toml index 23069f0158e..6d8a854e32d 100644 --- a/packages/rs-drive-abci/Cargo.toml +++ b/packages/rs-drive-abci/Cargo.toml @@ -82,7 +82,7 @@ derive_more = { version = "1.0", features = ["from", "deref", "deref_mut"] } async-trait = "0.1.77" console-subscriber = { version = "0.4", optional = true } bls-signatures = { git = "https://github.com/dashpay/bls-signatures", rev = "0842b17583888e8f46c252a4ee84cdfd58e0546f", optional = true } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "a2791bbdca756d6a6113024aec48f09f7a33faa9" } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "753a11f14c9a4bc72bf2d5302751dd43d174621e" } nonempty = "0.11" # Shielded-pool snapshot needs raw RocksDB SstFileWriter + ingest_external_file_cf # bindings, and blake3 for the snapshot-file checksum. @@ -107,7 +107,7 @@ dpp = { path = "../rs-dpp", default-features = false, features = [ drive = { path = "../rs-drive", features = ["fixtures-and-mocks"] } drive-proof-verifier = { path = "../rs-drive-proof-verifier" } strategy-tests = { path = "../strategy-tests" } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "a2791bbdca756d6a6113024aec48f09f7a33faa9", features = ["client"] } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "753a11f14c9a4bc72bf2d5302751dd43d174621e", features = ["client"] } assert_matches = "1.5.0" drive-abci = { path = ".", features = ["testing-config", "mocks", "shielded_test_data"] } bls-signatures = { git = "https://github.com/dashpay/bls-signatures", rev = "0842b17583888e8f46c252a4ee84cdfd58e0546f" } @@ -121,8 +121,8 @@ integer-encoding = { version = "4.0.0" } # For dump_only_default_and_aux_cfs_under_shielded_subtree_prefix — same # subtree-prefix algorithm grovedb uses internally. -grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "a2791bbdca756d6a6113024aec48f09f7a33faa9" } -grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "a2791bbdca756d6a6113024aec48f09f7a33faa9" } +grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "753a11f14c9a4bc72bf2d5302751dd43d174621e" } +grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "753a11f14c9a4bc72bf2d5302751dd43d174621e" } [features] default = ["bls-signatures"] diff --git a/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/shielded.rs b/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/shielded.rs index 76ec156b84f..969d2cfc875 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/shielded.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/shielded.rs @@ -310,13 +310,18 @@ impl Platform { .raw_storage() .get_transactional_storage_context(subtree_path, Some(&data_batch), tx) .unwrap(); - let mut ct = CommitmentTree::<_, DashMemo>::open(0, chunk_power, storage_ctx) - .value - .map_err(|e| { - Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( - format!("seed: CommitmentTree::open: {e}").into_boxed_str(), - ))) - })?; + let mut ct = CommitmentTree::<_, DashMemo>::open( + 0, + chunk_power, + storage_ctx, + &platform_version.drive.grove_version, + ) + .value + .map_err(|e| { + Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( + format!("seed: CommitmentTree::open: {e}").into_boxed_str(), + ))) + })?; // Batched seed via repeated `append_many_raw` (grovedb PR #751). // Each batch: @@ -364,12 +369,15 @@ impl Platform { cv_net: [0u8; 32], payload: n.encrypted_note, }); - let append_result = ct.append_many_raw(iter).value.map_err(|e| { - Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( - format!("seed: append_many_raw (batch {batch_index}): {e}") - .into_boxed_str(), - ))) - })?; + let append_result = ct + .append_many_raw(iter, &platform_version.drive.grove_version) + .value + .map_err(|e| { + Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( + format!("seed: append_many_raw (batch {batch_index}): {e}") + .into_boxed_str(), + ))) + })?; // Persist the Sinsemilla frontier per batch — cheap and // gives durable mid-bake checkpoints if we ever want to // resume from a crash. Mid-bake MMR `commit_mmr` is @@ -381,11 +389,13 @@ impl Platform { // between batches corrupts the in-memory overlay (manifests // as "MMR get_root failed: Inconsistent store" on the next // call). One final `commit_mmr` follows the loop below. - ct.save().value.map_err(|e| { - Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( - format!("seed: ct.save (batch {batch_index}): {e}").into_boxed_str(), - ))) - })?; + ct.save(&platform_version.drive.grove_version) + .value + .map_err(|e| { + Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( + format!("seed: ct.save (batch {batch_index}): {e}").into_boxed_str(), + ))) + })?; last_sinsemilla_root = append_result.sinsemilla_root; last_bulk_state_root = append_result.bulk_state_root; @@ -424,11 +434,12 @@ impl Platform { // overlay accumulates across `append_many_raw` calls and is // persisted only here. See the in-loop comment above and the // upstream fix `1340db71` for the rationale. - ct.commit_mmr().map_err(|e| { - Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( - format!("seed: ct.commit_mmr (final): {e}").into_boxed_str(), - ))) - })?; + ct.commit_mmr(&platform_version.drive.grove_version) + .map_err(|e| { + Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( + format!("seed: ct.commit_mmr (final): {e}").into_boxed_str(), + ))) + })?; let combined_root = grovedb_commitment_tree::compute_commitment_tree_state_root( &last_sinsemilla_root, diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/ranked_group_drain.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/ranked_group_drain.rs index ac11d80cc67..10ada3042f7 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/ranked_group_drain.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/batch/tests/document/ranked_group_drain.rs @@ -89,6 +89,7 @@ fn ranked_count_groups( .unwrap() .expect("the ranked count read must succeed") .into_iter() + .map(|entry| entry.key_pair()) .map(|(count, key)| { ( count, diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/shield/tests.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/shield/tests.rs index d9dc23a97bb..d0b03339f40 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/shield/tests.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/shield/tests.rs @@ -1896,23 +1896,26 @@ mod tests { } } - /// Binary-search both edges of the funding range to measure the band where - /// `validate_fees_of_event` accepts a shield that execution then rejects. + /// Measure the lower edge of the funding range — the least headroom + /// `validate_fees_of_event` accepts — and require that exact headroom to EXECUTE. /// - /// * lower edge = the ESTIMATED fee (below it, validation rejects cleanly) - /// * upper edge = the ACTUAL metered fee (at or above it, the shield executes) - /// - /// If the two cost models agreed, the edges would coincide and the band would be empty. - /// Any width is a range of funding levels where the transition is accepted by validation + /// If the two cost models agreed everywhere, the least accepted headroom would execute; + /// any gap is a band of funding levels where the transition is accepted by validation /// and then dropped at execution — no longer a chain halt since the proposer-side /// per-transition rollback (shipped in 4.1.1), but still a transition that can never - /// confirm despite paying the quoted fee. + /// confirm despite paying the quoted fee. Because execution's metered cost is a fixed + /// function of the state, "the least accepted headroom executes" is equivalent to the + /// band being empty at every funding level, so one edge search plus one execution probe + /// replaces the old two-edge search at half the cost. /// - /// Ignored until the estimation gap is closed: the estimated-cost path skips the keyless - /// commitment-tree append entirely (dashpay/grovedb#812), so the band is measurably open - /// (18,919,200 credits, 10.7% of the fee, at 494 notes). Re-enable with the grovedb pin - /// bump that fixes it. Also ~40 full Orchard proving runs, so keep it out of routine CI. - #[ignore = "open until the grovedb#812 estimator fix is pinned; ~40 Orchard proving runs"] + /// Under protocol version 13 (GROVE_V3) the estimated-cost path skips the keyless + /// commitment-tree append entirely (dashpay/grovedb#812, locked for replay), so the band + /// is measurably open there (18,919,200 credits, 10.7% of the fee, at 494 notes). From + /// protocol version 14 (GROVE_V4) the estimator prices the fixed per-append model + /// (grovedb #829/#830) and the band is closed EXACTLY: measured at 494 notes, the least + /// accepted headroom and the metered fee coincide (the estimate IS the metered fee, so + /// an address funds precisely what it is charged). ~30 fresh-platform runs; the Orchard + /// proof is built once. #[tokio::test] async fn shield_fee_estimate_and_actual_must_not_leave_a_halting_band() { let pv = PlatformVersion::latest(); @@ -1926,7 +1929,12 @@ mod tests { "sanity: the upper bound must comfortably fund the shield ({top_msg})" ); - // Upper edge: least headroom that actually executes == the ACTUAL metered fee. + // Least headroom that EXECUTES. Success is the one monotone + // predicate over the whole funding range — below it the outcome + // is some rejection whose error class varies with the gate that + // fires (the input-minimum gate at tiny headroom, the structural + // fee minimum, then the metered-affordability rejection), so + // searching on any single rejection class would not converge. let (mut lo, mut hi) = (0u64, CEILING); while lo + 1 < hi { let mid = lo + (hi - lo) / 2; @@ -1936,44 +1944,30 @@ mod tests { lo = mid; } } - let actual_fee = hi; - - // Lower edge: below the flat structural minimum shielded fee the transition is - // rejected in BASIC validation (`ShieldedInvalidValueBalanceError`) before any write, - // which is safe. The dangerous band starts where that gate stops rejecting. - let (mut lo2, mut hi2) = (0u64, actual_fee); - while lo2 + 1 < hi2 { - let mid = lo2 + (hi2 - lo2) / 2; - if run_at(mid, &b, pv).await.0 == Outcome::Internal { - hi2 = mid; - } else { - lo2 = mid; - } - } - let band_start = hi2; + let admission = hi; - let (edge_outcome, edge_msg) = run_at(band_start - 1, &b, pv).await; + let (below, below_msg) = run_at(admission - 1, &b, pv).await; - println!("shield_amount = {}", b.shield_amount); - println!("band start = {band_start} (first headroom that reaches execution)"); - println!("actual fee = {actual_fee} (execution, apply=true)"); - println!("just below band = {edge_outcome:?} :: {edge_msg}"); - println!( - "HALTING BAND = [{band_start}, {actual_fee}) width = {} credits ({:.1}% of the fee)", - actual_fee - band_start, - 100.0 * (actual_fee - band_start) as f64 / actual_fee as f64 - ); + println!("shield_amount = {}", b.shield_amount); + println!("least executing = {admission}"); + println!("just below = {below:?} :: {below_msg}"); - assert_eq!( - band_start, actual_fee, - "HALTING BAND: any shield whose fee headroom falls in [{band_start}, {actual_fee}) \ - clears both the structural minimum-fee gate and validate_fees_of_event (which \ - prices the batch with the synthetic apply=false cost model, and which SKIPS the \ - keyless commitment-tree append entirely), and is then REJECTED by \ - paid_from_address_inputs_and_outputs on the real apply=true cost — after its drive \ - operations were already written to the block transaction. Such a transition is \ - stripped from the block as TxAction::Removed while its writes remain in the \ - proposer's app hash, so no validator can reproduce that hash." + // One credit below the least headroom that executes must be a + // VALIDATION rejection. If it is instead the InternalError drop, + // validation accepted a transition execution then rejected — the + // mainnet-halt band is open at every funding level in the gap + // between the two thresholds. + assert_ne!( + below, + Outcome::Internal, + "HALTING BAND: headroom {} is accepted by validation and dropped at \ + execution ({below_msg})", + admission - 1 + ); + assert_ne!( + below, + Outcome::Success, + "binary search must have found the least executing headroom" ); } @@ -1994,6 +1988,9 @@ mod tests { /// binary-searches the least headroom validation lets through and the least headroom /// that executes, asserts the band between them is still open, and drops a shield /// funded at the midpoint. + /// From protocol version 14 (GROVE_V4) the estimator prices the fixed per-append + /// model exactly (grovedb #829/#830), so the mid-band shield this test needs cannot be + /// built there — see `shield_fee_estimate_and_actual_must_not_leave_a_halting_band`. #[tokio::test] async fn dropped_shield_must_not_mutate_state() { let pv = PlatformVersion::get(13).expect("protocol version 13 should exist"); diff --git a/packages/rs-drive-abci/src/query/document_query/v1/routing.rs b/packages/rs-drive-abci/src/query/document_query/v1/routing.rs index 3677b6bf0c3..f686ffb47f4 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/routing.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/routing.rs @@ -279,9 +279,11 @@ pub(super) fn validate_and_route( /// The `OFFSET` gate, applied **after** routing. /// /// Offset pagination exists on exactly one path: the ranked executor, -/// where `OFFSET m` is the rank the returned page starts at and costs -/// nothing to prove (grovedb attests the skipped region from counted -/// subtree commitments rather than walking it). Every other v1 shape — +/// where `OFFSET m` is the rank the returned page starts at; skipping +/// is a counted tree descent on either `prove` setting — grovedb counts +/// the skipped region from the subtree aggregates rather than walking +/// it, so the work is bounded by tree depth and does not grow with `m`. Only the proved result +/// additionally *attests* the count. Every other v1 shape — /// documents, and the grouped count / sum / average modes — has no /// offset primitive behind it and keeps the rejection it has always /// had, **message for message**: those callers paginate with diff --git a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs index 475419587a2..50fadc2e293 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs @@ -2504,9 +2504,11 @@ mod ranked_tests { assert_eq!(tail.skipped, Some(3)); // A window entirely past the end is an empty page, not an - // error. On this *unproven* path grovedb's read API doesn't - // report the short walk, so `skipped` echoes the request; the - // proved path is where it becomes the attested population. + // error, and `skipped` collapses to the population the walk + // actually reached. That reaches the wire on this *unproven* + // path too: grovedb's counted descent tracks how far the skip + // got and returns it on the page, so the response carries a + // population rather than the offset that was requested. let past_end = ranked_page(&platform, &state, paged(2, 9), version); assert!( past_end.entries.is_empty(), @@ -2514,7 +2516,12 @@ mod ranked_tests { error — got {:?}", group_keys(&past_end.entries) ); - assert_eq!(past_end.skipped, Some(9)); + assert_eq!( + past_end.skipped, + Some(5), + "the response reports the five groups the ranking holds, not the offset that \ + was asked for" + ); // And the same page proves. let result = platform diff --git a/packages/rs-drive-abci/src/query/shielded/encrypted_notes/v0/mod.rs b/packages/rs-drive-abci/src/query/shielded/encrypted_notes/v0/mod.rs index 808d0e49051..86214e5b5f0 100644 --- a/packages/rs-drive-abci/src/query/shielded/encrypted_notes/v0/mod.rs +++ b/packages/rs-drive-abci/src/query/shielded/encrypted_notes/v0/mod.rs @@ -81,6 +81,7 @@ impl Platform { path: shielded_credit_pool_path_vec(), query: SizedQuery { query: Query { + read_mode: None, items: vec![QueryItem::Key(vec![SHIELDED_NOTES_KEY])], default_subquery_branch: SubqueryBranch { subquery_path: None, diff --git a/packages/rs-drive-abci/src/shielded_snapshot/mod.rs b/packages/rs-drive-abci/src/shielded_snapshot/mod.rs index e7b64a893b5..a75dc3f0ad3 100644 --- a/packages/rs-drive-abci/src/shielded_snapshot/mod.rs +++ b/packages/rs-drive-abci/src/shielded_snapshot/mod.rs @@ -322,11 +322,16 @@ pub fn dump_shielded_subtree( // 4. Compute `combined_root` for the header by reloading CommitmentTree // from the same storage. Apply side recomputes independently and // cross-validates — drift surfaces as CombinedRootMismatch. - let ct = CommitmentTree::<_, DashMemo>::open(total_count, chunk_power, storage_ctx) - .value - .map_err(|e| ShieldedSnapshotError::GroveDb(format!("CommitmentTree::open: {e}")))?; + let ct = CommitmentTree::<_, DashMemo>::open( + total_count, + chunk_power, + storage_ctx, + &platform_version.drive.grove_version, + ) + .value + .map_err(|e| ShieldedSnapshotError::GroveDb(format!("CommitmentTree::open: {e}")))?; let combined_root = ct - .compute_current_state_root() + .compute_current_state_root(&platform_version.drive.grove_version) .map_err(|e| ShieldedSnapshotError::GroveDb(format!("compute_current_state_root: {e}")))?; // The CommitmentTree owns the storage_ctx. We need to drop it to free @@ -559,14 +564,18 @@ pub fn apply_shielded_snapshot( .get_transactional_storage_context(subtree_path, None, tx_ref) .unwrap(); - let ct = - CommitmentTree::<_, DashMemo>::open(header.total_count, header.chunk_power, storage_ctx) - .value - .map_err(|e| { - ShieldedSnapshotError::GroveDb(format!("CommitmentTree::open after ingest: {e}")) - })?; + let ct = CommitmentTree::<_, DashMemo>::open( + header.total_count, + header.chunk_power, + storage_ctx, + &platform_version.drive.grove_version, + ) + .value + .map_err(|e| { + ShieldedSnapshotError::GroveDb(format!("CommitmentTree::open after ingest: {e}")) + })?; let recomputed = ct - .compute_current_state_root() + .compute_current_state_root(&platform_version.drive.grove_version) .map_err(|e| ShieldedSnapshotError::GroveDb(format!("compute_current_state_root: {e}")))?; drop(ct); diff --git a/packages/rs-drive-proof-verifier/src/proof/document_ranked.rs b/packages/rs-drive-proof-verifier/src/proof/document_ranked.rs index 63bd8d1e01e..24f06524e7b 100644 --- a/packages/rs-drive-proof-verifier/src/proof/document_ranked.rs +++ b/packages/rs-drive-proof-verifier/src/proof/document_ranked.rs @@ -7,9 +7,10 @@ //! the per-axis *secondary* Merk of an indexed tree (grovedb PR #657), //! so it costs `O(log n + k)` and comes with a proof that commits to //! exactly the `k` returned `(aggregate, group key)` pairs — plus the -//! `OFFSET`, which grovedb attests from counted subtree commitments -//! rather than by walking the skipped region, so deep pages cost the -//! same as the first one. +//! `OFFSET`, which grovedb counts from the subtree aggregates rather +//! than by walking the skipped region — and additionally attests, on +//! this proved path — so a deep page costs `O(log n + k)` like any +//! other rather than growing with the offset. //! //! This module holds the client-facing result type //! ([`DocumentRankedEntries`]), the tenderdash-composition wrapper diff --git a/packages/rs-drive/Cargo.toml b/packages/rs-drive/Cargo.toml index f04f8410396..54012ec1f1f 100644 --- a/packages/rs-drive/Cargo.toml +++ b/packages/rs-drive/Cargo.toml @@ -52,12 +52,12 @@ enum-map = { version = "2.0.3", optional = true } intmap = { version = "3.0.1", features = ["serde"], optional = true } chrono = { version = "0.4.35", optional = true } itertools = { version = "0.13", optional = true } -grovedb = { git = "https://github.com/dashpay/grovedb", rev = "a2791bbdca756d6a6113024aec48f09f7a33faa9", optional = true, default-features = false } -grovedb-costs = { git = "https://github.com/dashpay/grovedb", rev = "a2791bbdca756d6a6113024aec48f09f7a33faa9", optional = true } -grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "a2791bbdca756d6a6113024aec48f09f7a33faa9" } -grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "a2791bbdca756d6a6113024aec48f09f7a33faa9", optional = true } -grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "a2791bbdca756d6a6113024aec48f09f7a33faa9" } -grovedb-epoch-based-storage-flags = { git = "https://github.com/dashpay/grovedb", rev = "a2791bbdca756d6a6113024aec48f09f7a33faa9" } +grovedb = { git = "https://github.com/dashpay/grovedb", rev = "753a11f14c9a4bc72bf2d5302751dd43d174621e", optional = true, default-features = false } +grovedb-costs = { git = "https://github.com/dashpay/grovedb", rev = "753a11f14c9a4bc72bf2d5302751dd43d174621e", optional = true } +grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "753a11f14c9a4bc72bf2d5302751dd43d174621e" } +grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "753a11f14c9a4bc72bf2d5302751dd43d174621e", optional = true } +grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "753a11f14c9a4bc72bf2d5302751dd43d174621e" } +grovedb-epoch-based-storage-flags = { git = "https://github.com/dashpay/grovedb", rev = "753a11f14c9a4bc72bf2d5302751dd43d174621e" } [dev-dependencies] criterion = "0.5" diff --git a/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/ranked_index_e2e_tests.rs b/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/ranked_index_e2e_tests.rs index c4e30fd9927..390621a9886 100644 --- a/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/ranked_index_e2e_tests.rs +++ b/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/ranked_index_e2e_tests.rs @@ -275,7 +275,7 @@ fn avg_top_k(drive: &Drive, path: &[Vec], k: u16, descending: bool) -> Vec<( let path_refs: Vec<&[u8]> = path.iter().map(|v| v.as_slice()).collect(); drive .grove - .indexed_avg_top_k( + .indexed_avg_top_k_keys( path_refs.as_slice(), k, descending, @@ -283,14 +283,14 @@ fn avg_top_k(drive: &Drive, path: &[Vec], k: u16, descending: bool) -> Vec<( &platform_version().drive.grove_version, ) .unwrap() - .expect("indexed_avg_top_k must succeed") + .expect("indexed_avg_top_k_keys must succeed") } fn count_top_k(drive: &Drive, path: &[Vec], k: u16, descending: bool) -> Vec<(u64, Vec)> { let path_refs: Vec<&[u8]> = path.iter().map(|v| v.as_slice()).collect(); drive .grove - .indexed_count_top_k( + .indexed_count_top_k_keys( path_refs.as_slice(), k, descending, @@ -298,14 +298,14 @@ fn count_top_k(drive: &Drive, path: &[Vec], k: u16, descending: bool) -> Vec &platform_version().drive.grove_version, ) .unwrap() - .expect("indexed_count_top_k must succeed") + .expect("indexed_count_top_k_keys must succeed") } fn sum_top_k(drive: &Drive, path: &[Vec], k: u16, descending: bool) -> Vec<(i64, Vec)> { let path_refs: Vec<&[u8]> = path.iter().map(|v| v.as_slice()).collect(); drive .grove - .indexed_sum_top_k( + .indexed_sum_top_k_keys( path_refs.as_slice(), k, descending, @@ -313,7 +313,7 @@ fn sum_top_k(drive: &Drive, path: &[Vec], k: u16, descending: bool) -> Vec<( &platform_version().drive.grove_version, ) .unwrap() - .expect("indexed_sum_top_k must succeed") + .expect("indexed_sum_top_k_keys must succeed") } // --------------------------------------------------------------------------- @@ -1295,6 +1295,136 @@ fn estimated_mode_insert_on_ranked_indexes_produces_fees_and_does_not_undercharg } } +/// Run one document update twice — estimated (`apply: false`) then applied — +/// and return `(estimated, actual)` fees. The document must already be +/// inserted; the estimate runs against the same committed state the applied +/// run will see. +fn estimated_and_actual_update_fees( + drive: &Drive, + contract: &DataContract, + document_type_name: &str, + updated: &Document, +) -> ( + dpp::fee::fee_result::FeeResult, + dpp::fee::fee_result::FeeResult, +) { + use std::borrow::Cow; + + let pv = platform_version(); + let document_type = contract + .document_type_for_name(document_type_name) + .unwrap_or_else(|_| panic!("{document_type_name} doctype exists")); + let storage_flags = Some(Cow::Owned(StorageFlags::SingleEpoch(0))); + + let run = |apply: bool| { + drive + .update_document_for_contract( + updated, + contract, + document_type, + None, + BlockInfo::default(), + apply, + storage_flags.clone(), + None, + pv, + None, + ) + .unwrap_or_else(|e| { + panic!("expected the {document_type_name} update (apply={apply}) to succeed: {e}") + }) + }; + + let estimated = run(false); + let actual = run(true); + (estimated, actual) +} + +/// Updates under a ranked index are write-amplified by design: an axis row +/// binds the immediate primary node's committed value hash (grovedb #817), so +/// ANY update of an indexed document refreshes every configured axis row — +/// an in-place payload change rewrites the rows where they stand, and a +/// group move deletes the old row and inserts a new one. Both must be priced +/// by the dry-run estimate at least as high as the applied run meters, or the +/// fee gate admits transitions that execution rejects. Both update shapes are +/// exercised per axis family, and the canonical-row integrity walk must be +/// clean afterwards (no stale rows left by the refresh paths). +#[test] +fn estimated_mode_update_on_ranked_indexes_does_not_undercharge() { + for (document_type_name, aggregated_property) in [ + ("review", "grade"), // PCPSIT, Avg axis + ("visit", "guests"), // PCIT, Count axis + ("tip", "amount"), // PSIT, Sum axis + ] { + let (drive, contract) = setup_restaurants(); + let doc = build_doc( + &contract, + document_type_name, + aggregated_property, + "alpha", + 42, + 7, + ); + insert_doc(&drive, &contract, document_type_name, &doc); + + // Phase 1: in-place update — same group, new aggregated payload. + let mut in_place = doc.clone(); + let mut props = std::collections::BTreeMap::new(); + props.insert(GROUP_PROPERTY.to_string(), Value::Text("alpha".to_string())); + props.insert(aggregated_property.to_string(), Value::I64(43)); + in_place.set_properties(props); + let (estimated, actual) = + estimated_and_actual_update_fees(&drive, &contract, document_type_name, &in_place); + for (kind, est, act) in [ + ("storage", estimated.storage_fee, actual.storage_fee), + ( + "processing", + estimated.processing_fee, + actual.processing_fee, + ), + ] { + assert!( + est >= act, + "{document_type_name}: in-place update estimated {kind} fee {est} is BELOW the \ + applied {kind} fee {act} (short by {}) — the axis row refresh under-charges", + act.saturating_sub(est), + ); + } + assert!( + actual.processing_fee > 0, + "{document_type_name}: the applied in-place update must meter work" + ); + + // Phase 2: group move — the sort key changes, so the old row is + // deleted and a new one inserted under the new prefix. + let mut moved = doc.clone(); + let mut props = std::collections::BTreeMap::new(); + props.insert(GROUP_PROPERTY.to_string(), Value::Text("beta".to_string())); + props.insert(aggregated_property.to_string(), Value::I64(43)); + moved.set_properties(props); + let (estimated, actual) = + estimated_and_actual_update_fees(&drive, &contract, document_type_name, &moved); + for (kind, est, act) in [ + ("storage", estimated.storage_fee, actual.storage_fee), + ( + "processing", + estimated.processing_fee, + actual.processing_fee, + ), + ] { + assert!( + est >= act, + "{document_type_name}: group-move update estimated {kind} fee {est} is BELOW \ + the applied {kind} fee {act} (short by {}) — the row move under-charges", + act.saturating_sub(est), + ); + } + + // No stale or non-canonical rows may survive either refresh path. + assert_grovedb_is_consistent(&drive); + } +} + /// The PSIT arm: a sum-only ranked index ranks its groups by running sum. #[test] fn sum_axis_ranks_groups_by_running_sum() { @@ -1557,10 +1687,10 @@ fn an_offset_window_spanning_the_end_returns_the_short_tail() { /// The page comes back empty and `skipped` collapses below the requested /// offset — and *that shape* is the proof that the ranking holds exactly /// `skipped` groups in total, because the counted commitments cover the whole -/// walk. It is the only way this surface reports a population, and the one -/// place the proved and unproven paths differ: the unproven read cannot see -/// the short walk (grovedb's read API returns an empty vector either way) and -/// reports the requested offset. +/// walk. It is the only way this surface reports a population, and both paths +/// report it: grovedb's counted descent tracks how far the skip got and returns +/// it on the page, so an unproven read reports the population rather than the +/// offset it was asked for. What proving adds is that the number is attested. #[test] fn an_offset_past_the_end_returns_an_empty_page_whose_skip_attests_the_population() { let (drive, contract) = setup_restaurants(); @@ -1572,8 +1702,9 @@ fn an_offset_past_the_end_returns_an_empty_page_whose_skip_attests_the_populatio "there is no rank 12 in a five-group ranking" ); assert_eq!( - page.skipped, 12, - "the unproven read echoes the requested offset — it has nothing to attest with" + page.skipped, 5, + "the unproven read reports the five groups the ranking holds, not the requested \ + offset of 12" ); let verified = verified_ranked_avg_page(&drive, &contract, 3, 12); diff --git a/packages/rs-drive/src/fees/op.rs b/packages/rs-drive/src/fees/op.rs index 6dcc2fc1cba..7e55a751bd4 100644 --- a/packages/rs-drive/src/fees/op.rs +++ b/packages/rs-drive/src/fees/op.rs @@ -1568,6 +1568,17 @@ impl LowLevelDriveOperationTreeTypeConverter for TreeType { batch_insert_empty_provable_count_provable_sum_indexed_tree) instead.", ))) } + // A private document store's entry size lives only on the + // `Element` (it does not affect Merk node layout), so `TreeType` + // cannot describe the element to insert. Drive has no private + // document store surface yet; when it does, creation must go + // through a dedicated helper that takes the entry size. + TreeType::PrivateDocumentStore(_) => { + return Err(Error::Drive(DriveError::NotSupported( + "empty_tree_operation_for_known_path_key cannot create a \ + PrivateDocumentStore — the entry size is not carried by TreeType", + ))) + } }; Ok(LowLevelDriveOperation::insert_for_known_path_key_element( diff --git a/packages/rs-drive/src/query/drive_document_having_query/execute_range.rs b/packages/rs-drive/src/query/drive_document_having_query/execute_range.rs index c9188d91f93..6707461d8d5 100644 --- a/packages/rs-drive/src/query/drive_document_having_query/execute_range.rs +++ b/packages/rs-drive/src/query/drive_document_having_query/execute_range.rs @@ -48,7 +48,7 @@ impl DriveDocumentHavingQuery<'_> { // but reads like a panicking unwrap at the call site. let entries = match self.bounds { AxisRangeBounds::Count { lo, hi } => { - let CostContext { value, cost: _ } = drive.grove.indexed_count_range( + let CostContext { value, cost: _ } = drive.grove.indexed_count_range_keys( path_refs.as_slice(), lo, hi, @@ -67,7 +67,7 @@ impl DriveDocumentHavingQuery<'_> { .collect::>() } AxisRangeBounds::Sum { lo, hi } => { - let CostContext { value, cost: _ } = drive.grove.indexed_sum_range( + let CostContext { value, cost: _ } = drive.grove.indexed_sum_range_keys( path_refs.as_slice(), lo, hi, @@ -86,7 +86,7 @@ impl DriveDocumentHavingQuery<'_> { .collect::>() } AxisRangeBounds::Avg { lo, hi } => { - let CostContext { value, cost: _ } = drive.grove.indexed_avg_range( + let CostContext { value, cost: _ } = drive.grove.indexed_avg_range_keys( path_refs.as_slice(), lo, hi, diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/execute_top_k.rs b/packages/rs-drive/src/query/drive_document_ranked_query/execute_top_k.rs index fb31a6635ff..9cda62c9234 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/execute_top_k.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/execute_top_k.rs @@ -15,7 +15,7 @@ use crate::drive::Drive; use crate::error::drive::DriveError; use crate::error::Error; use dpp::version::PlatformVersion; -use grovedb::TransactionArg; +use grovedb::{IndexedTopKKeysPage, TransactionArg}; use grovedb_costs::CostContext; impl DriveDocumentRankedQuery<'_> { @@ -38,12 +38,33 @@ impl DriveDocumentRankedQuery<'_> { /// no-proof and prove paths read the same code path in grovedb and /// cannot drift on the walk's semantics for offset-free queries. /// - /// [`RankedPage::skipped`] on this path is the *requested* offset: - /// grovedb's read API returns an empty vector when the walk runs out - /// during the skip and does not report how far it got, so an - /// unproven read cannot distinguish "skipped exactly `offset`" from - /// "the secondary holds fewer than `offset` groups". Only the proved - /// path attests the true value — see [`RankedPage::skipped`]. + /// # The offset is counted, not walked + /// + /// grovedb descends the secondary reading each subtree's aggregate + /// count off its link, and collapses any subtree that fits entirely + /// inside the remaining offset instead of stepping through it. The + /// skip therefore costs `O(log n)` at any offset rather than one + /// iterator step and one decode per skipped entry, and an offset at + /// or past the population is answered from the root's own count with + /// no descent at all — the cheapest request on this surface rather + /// than the most expensive. `offset = 0` keeps the plain iterator + /// path and never touches the tree, so the common unpaginated + /// request costs exactly what it always did. + /// + /// That is what makes an uncapped `OFFSET` safe rather than merely + /// tolerated. Ranked queries carry no fee, cannot be cancelled once + /// dispatched, and share their rate budget with state transitions + /// rather than having one of their own, so a skip whose cost grew + /// with the offset would be an unmetered lever for any + /// unauthenticated caller. It does not grow. + /// + /// [`RankedPage::skipped`] comes back from grovedb rather than being + /// echoed from the request: it is the requested offset when the skip + /// succeeded, and the secondary's whole population when the walk ran + /// out of groups first. That is the same quantity the proved path + /// attests, so the two no longer disagree — though on this path it is + /// the node's unverified claim rather than an attested value, exactly + /// like the entries beside it. See [`RankedPage::skipped`]. pub fn execute_top_k_no_proof( &self, drive: &Drive, @@ -55,33 +76,40 @@ impl DriveDocumentRankedQuery<'_> { let path_refs: Vec<&[u8]> = path.iter().map(|segment| segment.as_slice()).collect(); let offset = self.offset as u64; - // Costs are destructured away rather than `.unwrap()`-ed: + // The cost is dropped rather than `.unwrap()`-ed: // `CostContext::unwrap` is infallible (it drops the cost field) - // but reads like a panicking unwrap at the call site. The - // dispatcher wraps these executors with its own fee accounting, - // exactly as the count surface's `execute_range_count_no_proof` - // does. - let entries = match self.axis { + // but reads like a panicking unwrap at the call site. Dropping it + // is all there is to do with it — nothing meters a query on this + // surface: neither this executor's caller nor the dispatcher + // above it accumulates or charges the cost, and no credit is + // debited for a read. grovedb computes the `OperationCost` + // because its API always does, and it ends here. + let (entries, skipped) = match self.axis { RankedAxis::Count => { - let CostContext { value, cost: _ } = drive.grove.indexed_count_top_k_paginated( - path_refs.as_slice(), - self.k, - offset, - self.descending, - transaction, - grove_version, - ); - value - .map_err(|e| Error::GroveDB(Box::new(e)))? - .into_iter() - .map(|(count, key)| RankedEntry { - key, - value: RankedEntryValue::Count(count), - }) - .collect::>() + let CostContext { value, cost: _ } = + drive.grove.indexed_count_top_k_paginated_keys( + path_refs.as_slice(), + self.k, + offset, + self.descending, + transaction, + grove_version, + ); + let IndexedTopKKeysPage { entries, skipped } = + value.map_err(|e| Error::GroveDB(Box::new(e)))?; + ( + entries + .into_iter() + .map(|(count, key)| RankedEntry { + key, + value: RankedEntryValue::Count(count), + }) + .collect::>(), + skipped, + ) } RankedAxis::Sum => { - let CostContext { value, cost: _ } = drive.grove.indexed_sum_top_k_paginated( + let CostContext { value, cost: _ } = drive.grove.indexed_sum_top_k_paginated_keys( path_refs.as_slice(), self.k, offset, @@ -89,17 +117,21 @@ impl DriveDocumentRankedQuery<'_> { transaction, grove_version, ); - value - .map_err(|e| Error::GroveDB(Box::new(e)))? - .into_iter() - .map(|(sum, key)| RankedEntry { - key, - value: RankedEntryValue::Sum(sum), - }) - .collect::>() + let IndexedTopKKeysPage { entries, skipped } = + value.map_err(|e| Error::GroveDB(Box::new(e)))?; + ( + entries + .into_iter() + .map(|(sum, key)| RankedEntry { + key, + value: RankedEntryValue::Sum(sum), + }) + .collect::>(), + skipped, + ) } RankedAxis::Avg => { - let CostContext { value, cost: _ } = drive.grove.indexed_avg_top_k_paginated( + let CostContext { value, cost: _ } = drive.grove.indexed_avg_top_k_paginated_keys( path_refs.as_slice(), self.k, offset, @@ -107,14 +139,18 @@ impl DriveDocumentRankedQuery<'_> { transaction, grove_version, ); - value - .map_err(|e| Error::GroveDB(Box::new(e)))? - .into_iter() - .map(|(avg, key)| RankedEntry { - key, - value: RankedEntryValue::AvgFixedPoint(avg), - }) - .collect::>() + let IndexedTopKKeysPage { entries, skipped } = + value.map_err(|e| Error::GroveDB(Box::new(e)))?; + ( + entries + .into_iter() + .map(|(avg, key)| RankedEntry { + key, + value: RankedEntryValue::AvgFixedPoint(avg), + }) + .collect::>(), + skipped, + ) } }; @@ -131,10 +167,7 @@ impl DriveDocumentRankedQuery<'_> { self.k )))); } - Ok(RankedPage { - skipped: offset, - entries, - }) + Ok(RankedPage { skipped, entries }) } /// Generate the grovedb indexed-axis paginated top-k proof for this diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/executors/top_k_no_proof.rs b/packages/rs-drive/src/query/drive_document_ranked_query/executors/top_k_no_proof.rs index 281fdfe2c2a..4345618e5f3 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/executors/top_k_no_proof.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/executors/top_k_no_proof.rs @@ -1,6 +1,11 @@ -//! Ranked executor for `prove = false` — reads one page of `k` groups -//! straight out of the axis secondary, starting at rank `offset`, and -//! returns them in ranking order. +//! Ranked executor for `prove = false` — one page of `k` groups starting +//! at rank `offset`, in ranking order. +//! +//! The page is read from the axis secondary directly, with no proof +//! built. The `offset` is skipped by a counted descent rather than by +//! stepping through the skipped entries, so a deep offset costs +//! `O(log n)` — see +//! [`crate::query::DriveDocumentRankedQuery::execute_top_k_no_proof`]. use super::super::{DocumentRankedMode, RankedPage}; use super::ranked_query_for_mode; diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/mod.rs b/packages/rs-drive/src/query/drive_document_ranked_query/mod.rs index fed2b5dffbd..a72ccd7b7f4 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/mod.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/mod.rs @@ -52,14 +52,16 @@ //! by aggregate, not by group key — and is rejected rather than //! silently ignored, as is any non-equality prefix clause (`IN` //! included: one walk per element is a future multi-`IN` capability). -//! 2. **`limit` is mandatory, `offset` is free, `start_at` is refused.** +//! 2. **`limit` is mandatory, `offset` is depth-bounded, `start_at` is +//! refused.** //! `limit` is the `k` of the walk and the ranked surface has no //! server default for it, so it must be supplied. `offset` is the -//! rank the page starts at and is unbounded above: grovedb's -//! paginated prover is `O(log n + k)` *regardless of offset* (the -//! skipped region is attested by counted subtree commitments, never -//! walked entry by entry), so a large offset is not a cost lever and -//! needs no ceiling. `start_at` / `start_after` name a document id, +//! rank the page starts at and is unbounded above: grovedb counts +//! the skipped region from the subtree aggregates rather than +//! walking it entry by entry, so both executors are `O(log n + k)` +//! *regardless of offset* and a large offset is not a cost lever on +//! either. Only the proved result additionally attests the count. +//! So the offset needs no ceiling. `start_at` / `start_after` name a document id, //! which does not appear anywhere in an aggregate-ordered keyspace. //! 3. **Entry order IS the ranking order.** The executor returns entries //! in the order grovedb walked the secondary; callers must not @@ -312,14 +314,17 @@ pub struct DriveDocumentRankedQuery<'a> { /// How many ranks to skip before the returned page — the request's /// `OFFSET`. `0` for an unpaginated ranking. /// - /// Unbounded above (any `u32`), on purpose. grovedb attests the - /// skipped region through the counted subtree commitments - /// (`HashWithCount` / `HashWithCountAndSum`) rather than by walking - /// it, so both the prover's work and the proof's size stay - /// `O(log n + k)` **at any offset** — an offset of 4 and an offset - /// of four billion cost the same. There is therefore no - /// denial-of-service lever to cap, and capping would only stop - /// honest deep pagination. + /// Unbounded above (any `u32`), on purpose. grovedb skips by + /// counting rather than walking — descending the secondary on each + /// subtree's aggregate count (`HashWithCount` / + /// `HashWithCountAndSum`) and collapsing any subtree that fits + /// inside the remaining offset — so work and proof size stay + /// `O(log n + k)` **at any offset**, and an offset of 4 and an + /// offset of four billion cost the same order of work, the deeper + /// one in fact slightly less. Both executors go through that + /// descent, the unproved one without building a proof, so there is + /// no denial-of-service lever to cap on either path and capping + /// would only stop honest deep pagination. /// /// An offset past the end of the secondary is a provable answer, not /// an error: the page comes back empty and @@ -339,19 +344,28 @@ pub struct DriveDocumentRankedQuery<'a> { pub struct RankedPage { /// Number of secondary entries skipped before this page. /// - /// On the **proved** path this is grovedb's cryptographically - /// attested count, independently re-derived by the verifier from the - /// counted subtree commitments in the proof bytes: it equals the - /// requested offset unless the walk ran out of entries first, in - /// which case `entries` is empty and `skipped` is a proof that the - /// secondary holds exactly `skipped` groups in total. + /// Both paths report the same quantity, and it is never an echo of + /// the request: grovedb's counted descent tracks how far the skip + /// actually got, so this equals the requested offset when the skip + /// succeeded and the secondary's whole population when the walk ran + /// out of groups first (in which case `entries` is empty). /// - /// On the **unproven** read there is nothing to attest and grovedb's - /// read API does not report the short walk, so this is simply the - /// requested offset. The two paths therefore disagree in exactly one - /// case — an offset past the end — where the unproven read reports - /// the requested offset and the proved one reports the true - /// population. Callers that need the population must prove. + /// What differs between the paths is the warrant. On the **proved** + /// path the value is cryptographically attested — independently + /// re-derived by the verifier from the counted subtree commitments + /// in the proof bytes — so a verifying client uses its own + /// reconstruction rather than trusting the server's. On the + /// **unproven** read it is the node's unverified claim, exactly like + /// the entries beside it: equal to the attested value on an honest + /// node, with nothing forcing a node to be honest. + /// + /// One nuance worth knowing on the unproven path: the population is + /// read from the secondary's root aggregate, while grovedb's + /// per-node payload check only fires on nodes the descent visits. In + /// a *corrupt* secondary whose count violation lies outside the + /// visited region, this value can therefore disagree with the true + /// row count where the proved path's would not. On any valid + /// secondary the two are identical by construction. pub skipped: u64, /// The groups on this page, **in ranking order**. Never longer than /// the query's `k`. diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/mode_detection/v0/mod.rs b/packages/rs-drive/src/query/drive_document_ranked_query/mode_detection/v0/mod.rs index 8a96f033b47..abe883991e6 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/mode_detection/v0/mod.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/mode_detection/v0/mod.rs @@ -291,7 +291,8 @@ pub fn detect_ranked_mode_v0( proof commits one secondary entry per returned group, so its size grows \ linearly in the limit. Narrow the request — the ceiling is a hard limit, not \ a clamp, because `k` is echoed in the proof envelope and re-checked by the \ - verifier. Deep results are reached with `OFFSET`, which costs nothing." + verifier. Deep results are reached with `OFFSET`, whose skip work is bounded by \ + tree depth and does not grow with the offset." )))); } // Bounded by MAX_RANKED_LIMIT (a u16) immediately above. @@ -300,14 +301,20 @@ pub fn detect_ranked_mode_v0( // ---- OFFSET: optional, unbounded -------------------------------- // // No ceiling, and that is a deliberate statement about cost rather - // than an oversight: grovedb's paginated prover attests the skipped - // region from the counted subtree commitments instead of walking - // it, so proving `OFFSET 4` and `OFFSET 4_000_000_000` are the same - // O(log n + k) work and the same proof size. There is no - // denial-of-service lever here to cap, and an arbitrary cap would - // only break honest deep pagination. An offset past the end is a - // provable answer (empty page, `skipped` attesting the population), - // not an error. + // than an oversight. grovedb skips by *counting*, not by walking: + // it descends the secondary reading each subtree's aggregate count + // and collapses any subtree that fits inside the remaining offset, + // so `OFFSET 4` and `OFFSET 4_000_000_000` are the same order of + // O(log n + k) work — the deeper one in fact cheaper, since a tree + // that fits entirely inside the offset collapses at the root. + // + // Both executors get that: the prover attests the skipped region + // from the counted subtree commitments, and the unproved read + // performs the same counted descent without building a proof. So + // there is no denial-of-service lever here for a cap to close on + // either path, and an arbitrary cap would only break honest deep + // pagination. An offset past the end is a real answer (empty page, + // `skipped` reporting the population), not an error. let offset = pagination.offset.unwrap_or(0); // ---- START AT: must be absent ----------------------------------- diff --git a/packages/rs-drive/src/query/drive_document_ranked_query/tests.rs b/packages/rs-drive/src/query/drive_document_ranked_query/tests.rs index 4a477c98195..0052ab8388e 100644 --- a/packages/rs-drive/src/query/drive_document_ranked_query/tests.rs +++ b/packages/rs-drive/src/query/drive_document_ranked_query/tests.rs @@ -160,7 +160,8 @@ fn offset_is_optional_defaults_to_zero_and_is_uncapped() { assert!(fifth_best.descending); // Far past any plausible population, and far past MAX_RANKED_LIMIT: - // still accepted, because offset costs nothing to prove. + // still accepted, because the skip is counted from subtree + // aggregates — work bounded by tree depth, not by the offset. let deep = detect_avg(false, Some(10), Some(u32::MAX)).expect("a huge OFFSET is well-formed"); assert_eq!(deep.offset, u32::MAX); assert_eq!(deep.k, 10); @@ -1298,10 +1299,10 @@ fn top_k_larger_than_the_group_count_returns_every_group() { /// page is. /// 3. **A window entirely past the end** — the page is empty *and* /// `skipped` collapses to the secondary's true population, which is -/// the proof's way of saying "there is nothing here, and here is how -/// much there is in total". That is the one case where the proved -/// and unproven paths differ: the unproven read cannot see the short -/// walk and reports the requested offset. +/// the counted walk's way of saying "there is nothing here, and here +/// is how much there is in total". Both paths report it: the counted +/// descent tracks how far the skip got, so an unproven read reports a +/// population rather than the offset it was asked for. #[test] fn offset_pages_through_the_ranking_and_the_proof_attests_the_starting_rank() { let (drive, contract) = setup_restaurants(); @@ -1360,8 +1361,10 @@ fn offset_pages_through_the_ranking_and_the_proof_attests_the_starting_rank() { "there is no rank 9 in a five-group ranking" ); assert_eq!( - page.skipped, 9, - "the unproven read cannot see the short walk, so it echoes the requested offset" + page.skipped, 5, + "the unproven read reports the population it actually reached, not the requested \ + offset of 9: grovedb's counted descent knows how far the walk got and returns it \ + on the page" ); let verified = assert_proof_round_trips(&drive, &contract, &past_end, &page.entries); assert_eq!( diff --git a/packages/rs-drive/src/query/mod.rs b/packages/rs-drive/src/query/mod.rs index a601d420761..7d6e98f3c5b 100644 --- a/packages/rs-drive/src/query/mod.rs +++ b/packages/rs-drive/src/query/mod.rs @@ -1381,7 +1381,16 @@ impl<'a> DriveDocumentQuery<'a> { return Ok(main_path_query); } - if let Some(start_at_path_query) = start_at_path_query { + if let Some(mut start_at_path_query) = start_at_path_query { + // The cursor query selects exactly one key, so its walk + // direction carries no meaning — but grovedb's merge (V4+) + // requires every input to agree on direction and propagates + // the shared one to the merged root. Align it to the main + // query's `orderBy` direction so a descending page merges, + // and so the merged root keeps the direction the verifier + // will rebuild through this same path. + start_at_path_query.query.query.left_to_right = + main_path_query.query.query.left_to_right; let limit = main_path_query.query.limit.take(); let mut merged = PathQuery::merge( vec![&start_at_path_query, &main_path_query], diff --git a/packages/rs-drive/src/state_transition_action/action_convert_to_operations/shielded/mod.rs b/packages/rs-drive/src/state_transition_action/action_convert_to_operations/shielded/mod.rs index 4036adcb21f..2f76727173c 100644 --- a/packages/rs-drive/src/state_transition_action/action_convert_to_operations/shielded/mod.rs +++ b/packages/rs-drive/src/state_transition_action/action_convert_to_operations/shielded/mod.rs @@ -48,3 +48,141 @@ pub(super) fn update_balance<'a>(ops: &mut Vec>, new_total_ba ShieldedPoolOperationType::UpdateTotalBalance { new_total_balance }, )); } + +/// Measurement support for the pool-paid shielded fee-floor tests. +/// +/// The pool-paid shielded transitions (ShieldedTransfer, Unshield, +/// ShieldedWithdrawal) charge a FLAT fee and book +/// `storage_fee = min(actual_storage, flat)`, `processing = flat - +/// storage_fee`; they never validate affordability against a per-transition +/// estimate. The invariant such a fee has to satisfy is therefore an +/// **amortized** one: over a whole commitment-tree epoch — including the one +/// append per epoch that compacts the dense buffer into a chunk blob — the +/// flat fee must cover the average real write cost, and must stay above the +/// average real storage so the booking split never starves the proposer. +/// Under the GROVE_V4 fixed per-append model (grovedb #829/#830) the +/// compaction is amortized inside GroveDB itself, so the epoch average and +/// the boundary append coincide; measuring the full epoch keeps the +/// invariant honest under any model — whatever a boundary append meters, +/// the pool absorbs it, and a client cannot land on it more than once per +/// epoch. +/// +/// Measured once per test binary (an epoch is 2048 real appends). +#[cfg(test)] +pub(super) mod fee_floor_support { + use std::sync::OnceLock; + + use dpp::block::block_info::BlockInfo; + use dpp::block::epoch::Epoch; + use dpp::version::PlatformVersion; + + use crate::drive::shielded::paths::SHIELDED_NOTES_CHUNK_POWER; + use crate::state_transition_action::action_convert_to_operations::DriveHighLevelOperationConverter; + use crate::state_transition_action::shielded::shielded_transfer::v0::ShieldedTransferTransitionActionV0; + use crate::state_transition_action::shielded::shielded_transfer::ShieldedTransferTransitionAction; + use crate::state_transition_action::shielded::ShieldedActionNote; + use crate::util::test_helpers::setup::setup_drive_with_initial_state_structure; + + /// Real metered cost of one epoch of 1-action shielded transfers, + /// applied for real (`apply = true`) on a fresh pool. + pub(crate) struct TransferEpoch { + /// Appends measured (one epoch). + pub appends: u64, + /// Average total (storage + processing) per append. + pub avg_total: u64, + /// Average storage fee per append. + pub avg_storage: u64, + /// The compacting append's storage fee — the figure the pool-paid + /// booking split `min(actual_storage, flat)` must stay below. + pub boundary_storage: u64, + /// The compacting append's total. + pub boundary_total: u64, + /// An ordinary (non-compacting) append's total, late in the epoch. + pub ordinary_total: u64, + } + + /// A production-sized 1-action transfer, distinct per index. + pub(crate) fn transfer_action(i: u32, fee_amount: u64) -> ShieldedTransferTransitionAction { + ShieldedTransferTransitionAction::V0(ShieldedTransferTransitionActionV0 { + notes: vec![note(i)], + anchor: [0xAA; 32], + fee_amount, + current_total_balance: fee_amount + 1_000_000, + }) + } + + /// A production-sized note (216-byte ciphertext), distinct per index. + pub(crate) fn note(i: u32) -> ShieldedActionNote { + let b = i.to_be_bytes(); + let mut nf = [0u8; 32]; + nf[..4].copy_from_slice(&b); + nf[4] = 1; + let mut cmx = [0u8; 32]; + cmx[..4].copy_from_slice(&b); + cmx[4] = 2; + let mut cv = [0u8; 32]; + cv[..4].copy_from_slice(&b); + cv[4] = 3; + ShieldedActionNote { + nullifier: nf, + cmx, + cv_net: cv, + encrypted_note: vec![0x77; 216], + } + } + + static EPOCH: OnceLock = OnceLock::new(); + + /// One epoch of real 1-action transfers, measured once. + pub(crate) fn transfer_epoch() -> &'static TransferEpoch { + EPOCH.get_or_init(|| { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + let epoch = Epoch::new(0).unwrap(); + let tx = drive.grove.start_transaction(); + let appends: u64 = 1u64 << SHIELDED_NOTES_CHUNK_POWER; + let fee_amount = 1_000_000_000; + let mut total = 0u64; + let mut storage = 0u64; + let mut boundary_storage = 0; + let mut boundary_total = 0; + let mut ordinary_total = 0; + for i in 0..appends { + let ops = transfer_action(i as u32, fee_amount) + .into_high_level_drive_operations(&epoch, platform_version) + .expect("operations"); + let fr = drive + .apply_drive_operations( + ops, + true, + &BlockInfo::default(), + Some(&tx), + platform_version, + None, + ) + .expect("apply"); + let t = fr.total_base_fee(); + total += t; + storage += fr.storage_fee; + if i + 1 == appends { + boundary_storage = fr.storage_fee; + boundary_total = t; + } + if i + 8 == appends { + ordinary_total = t; + } + } + TransferEpoch { + appends, + // Ceiling division: a truncated average could let a flat fee + // marginally below the exact epoch-wide cost slip past the + // floor assertions. + avg_total: total.div_ceil(appends), + avg_storage: storage.div_ceil(appends), + boundary_storage, + boundary_total, + ordinary_total, + } + }) + } +} diff --git a/packages/rs-drive/src/state_transition_action/action_convert_to_operations/shielded/shielded_transfer_transition.rs b/packages/rs-drive/src/state_transition_action/action_convert_to_operations/shielded/shielded_transfer_transition.rs index b5ebf06f921..3db03db4424 100644 --- a/packages/rs-drive/src/state_transition_action/action_convert_to_operations/shielded/shielded_transfer_transition.rs +++ b/packages/rs-drive/src/state_transition_action/action_convert_to_operations/shielded/shielded_transfer_transition.rs @@ -152,94 +152,92 @@ mod tests { assert!(result.is_err()); } - /// Invariant: the flat `compute_minimum_shielded_fee` must cover the *actual* GroveDB write - /// cost (`Drive::calculate_fee`) of a shielded transition's operations. + /// Invariant: the flat `compute_minimum_shielded_fee` covers the **amortized** + /// real GroveDB write cost of a shielded transfer, and stays above the + /// amortized real storage so the pool-paid booking split + /// (`storage_fee = min(actual_storage, flat)`, `processing = flat - + /// storage_fee`) never starves the proposer. /// - /// A shielded transfer is the cleanest per-action case — insert nullifiers + notes + - /// pool-balance update, with no contract document — so it isolates the dominant variable - /// cost (note storage). We use production-sized notes (216-byte encrypted note → 280-byte - /// commitment-tree item) and measure the real cost in estimation mode (`apply = false`). + /// Amortized, because the fee is flat and pool-paid: whatever the + /// epoch-boundary append that compacts the dense buffer into a chunk + /// blob meters, the pool absorbs it by design, and a client cannot land + /// on it more than once per epoch. Under the GROVE_V4 fixed per-append + /// model (grovedb #829/#830) every append — the compacting one included — + /// meters the same figure, so the epoch average IS the per-append cost; + /// the measurement spans a full epoch anyway so the floor stays honest + /// under any cost model. Measured on a real pool, one epoch of real + /// appends including the compacting one (see `fee_floor_support`). /// - /// The flat fee covers it with large margin because it also bundles a flat 100M - /// proof-verification fee that `calculate_fee` never charges (Halo 2 verification is CPU, - /// not a GroveDB op). 16 is the max actions per bundle. + /// The booking split is checked at its worst point too: even the + /// compacting append's real storage must stay below the flat fee, so + /// `min()` never zeroes the proposer's processing share on any append. #[test] fn test_minimum_shielded_fee_covers_actual_grovedb_write_cost() { - use crate::util::test_helpers::setup::setup_drive_with_initial_state_structure; - use dpp::block::block_info::BlockInfo; + use super::super::fee_floor_support::transfer_epoch; use dpp::shielded::compute_minimum_shielded_fee; - let drive = setup_drive_with_initial_state_structure(None); let platform_version = PlatformVersion::latest(); - let epoch = Epoch::new(0).unwrap(); - - // Production-sized note: 216-byte encrypted note, distinct nullifier/cmx per action. - let realistic_note = |i: u8| ShieldedActionNote { - nullifier: [i.wrapping_add(1); 32], - cmx: [i.wrapping_add(101); 32], - cv_net: [i.wrapping_add(201); 32], - encrypted_note: vec![0x77; 216], - }; + let epoch = transfer_epoch(); - for num_actions in [1usize, 8, 16] { - let fee_amount = compute_minimum_shielded_fee(num_actions, platform_version) + for num_actions in [1u64, 8, 16] { + let fee_amount = compute_minimum_shielded_fee(num_actions as usize, platform_version) .expect("fee computation should not overflow"); - let notes: Vec<_> = (0..num_actions as u8).map(realistic_note).collect(); - let action = ShieldedTransferTransitionAction::V0(ShieldedTransferTransitionActionV0 { - notes, - anchor: [0xAA; 32], - fee_amount, - current_total_balance: fee_amount + 1_000_000, - }); - - let ops = action - .into_high_level_drive_operations(&epoch, platform_version) - .expect("operations"); - - // apply = false → estimation mode: no DB mutation, returns the real cost. - let fee_result = drive - .apply_drive_operations( - ops, - false, - &BlockInfo::default(), - None, - platform_version, - None, - ) - .expect("estimate write cost"); - let actual_cost = fee_result.total_base_fee(); - - // The fee must cover the real write cost. Measured margins over GroveDB cost - // (estimation mode, production-sized notes): ~10.9x at 1 action down to ~5.8x at - // the 16-action max. The margin is large and stays well above 1x because the - // per-action fee also prices the per-action Halo 2 verification CPU (which - // calculate_fee does not charge), so it exceeds the per-action GroveDB cost by - // design; see `shielded_per_action_processing_fee`. + // Each action is one note append + one nullifier; the amortized + // per-append cost scales linearly, and the flat fee also carries + // the fixed proof-verification term, so this holds with margin. assert!( - fee_amount >= actual_cost, - "compute_minimum_shielded_fee({num_actions}) = {fee_amount} must cover the actual \ - GroveDB write cost {actual_cost} (storage {} + processing {})", - fee_result.storage_fee, - fee_result.processing_fee + fee_amount >= num_actions * epoch.avg_total, + "compute_minimum_shielded_fee({num_actions}) = {fee_amount} must cover the \ + amortized real write cost {} x {} (epoch of {} appends; compacting append \ + total {}, ordinary append total {})", + num_actions, + epoch.avg_total, + epoch.appends, + epoch.boundary_total, + epoch.ordinary_total ); - - // Pin the booking-split invariant directly. The pool-paid booking in - // `execute_event/v0` splits the flat carved fee as - // storage_fee = min(real_metered_storage, flat_fee) - // processing_fee = flat_fee - storage_fee - // The `min()` only ever binds — zeroing the proposer's processing reward and - // undercharging storage — if the real metered storage EXCEEDS the flat fee. Asserting - // `flat_fee > real_metered_storage` here is exactly the condition that guarantees the - // `min()` is a no-op, so the proposer is always paid the processing remainder and - // storage is never undercharged. (Strict `>` because the flat fee also bundles the 100M - // proof-verification fee that GroveDB never meters.) assert!( - fee_amount > fee_result.storage_fee, - "compute_minimum_shielded_fee({num_actions}) = {fee_amount} must strictly exceed the \ - real metered storage {} so the booking split's min(real_storage, flat_fee) never \ - binds (proposer processing reward never zeroed, storage never undercharged)", - fee_result.storage_fee + fee_amount > num_actions * epoch.avg_storage, + "flat fee {fee_amount} must exceed the amortized real storage {} x {} so the \ + pool-paid booking split never starves the proposer", + num_actions, + epoch.avg_storage ); } + let fee_one = compute_minimum_shielded_fee(1, platform_version).expect("fee"); + assert!( + epoch.boundary_storage < fee_one, + "even the compacting append's real storage ({}) must stay below the flat fee \ + ({fee_one}), or `min(actual_storage, flat)` would zero the proposer's share", + epoch.boundary_storage + ); + + // Component independence: the fee's resource reservations must each + // cover their own cost, with nothing subsidized by the compute + // budget. The proof-verification fee is reserved for Halo 2 CPU and + // takes part in NO database assertion here. + let constants = &platform_version + .drive_abci + .validation_and_processing + .event_constants; + let storage = &platform_version.fee_version.storage; + let per_byte_rate = + storage.storage_disk_usage_credit_per_byte + storage.storage_processing_credit_per_byte; + let storage_component = constants.shielded_storage_bytes_per_action * per_byte_rate; + assert!( + storage_component >= epoch.avg_storage, + "the per-action storage allowance alone ({} bytes = {storage_component} credits) \ + must cover the amortized real storage per append ({}); raise \ + `shielded_storage_bytes_per_action`", + constants.shielded_storage_bytes_per_action, + epoch.avg_storage + ); + assert!( + constants.shielded_per_action_processing_fee >= epoch.avg_total - epoch.avg_storage, + "the per-action processing fee ({}) must cover the amortized metered GroveDB \ + processing per append ({}) on top of its verification work", + constants.shielded_per_action_processing_fee, + epoch.avg_total - epoch.avg_storage + ); } } diff --git a/packages/rs-drive/src/state_transition_action/action_convert_to_operations/shielded/unshield_transition.rs b/packages/rs-drive/src/state_transition_action/action_convert_to_operations/shielded/unshield_transition.rs index b8a4c770563..adae1927679 100644 --- a/packages/rs-drive/src/state_transition_action/action_convert_to_operations/shielded/unshield_transition.rs +++ b/packages/rs-drive/src/state_transition_action/action_convert_to_operations/shielded/unshield_transition.rs @@ -261,92 +261,97 @@ mod tests { assert!(result.is_err()); } - /// Invariant: the flat `compute_shielded_unshield_fee` must cover the *actual* GroveDB write - /// cost of an Unshield's operations, AND strictly exceed the metered storage so the pool-paid - /// booking split never undercharges. + /// Invariant: the flat `compute_shielded_unshield_fee` covers the **amortized** + /// real GroveDB write cost of an Unshield, and stays above the amortized + /// real storage so the pool-paid booking split never starves the + /// proposer — the same amortized reading as the ShieldedTransfer test + /// (see there, and `fee_floor_support`, for why a flat pool-paid fee is + /// held to the epoch average rather than the per-append worst case). /// - /// Unshield is pool-paid: `execute_event/v0` carves the flat fee from the shielded pool and - /// splits it as `storage_fee = min(real_metered_storage, flat_fee); processing_fee = flat_fee - - /// storage_fee`. The `min()` only binds (zeroing the proposer's processing reward and - /// undercharging storage) if real metered storage EXCEEDS the flat fee. Unshield also writes the - /// net to the output platform address (`AddBalanceToAddress`), so its fee - /// (`compute_shielded_unshield_fee`) prices that write as a flat storage component on top of the - /// base shielded fee — which is exactly why the `fee > storage` margin below holds with room to - /// spare. This test meters the real cost in estimation mode (`apply = false`, production-sized - /// 216-byte notes) and asserts both `fee >= total cost` and `fee > storage` — the latter being - /// exactly the condition that keeps the `min()` a no-op. Mirrors the ShieldedTransfer metering - /// test. + /// An Unshield is a transfer's note/nullifier writes plus the + /// `AddBalanceToAddress` output write, so its amortized cost is the + /// transfer epoch average plus the measured per-transition delta between + /// an ordinary unshield and an ordinary transfer. #[test] fn test_minimum_shielded_fee_covers_actual_grovedb_write_cost() { + use super::super::fee_floor_support::{note, transfer_action, transfer_epoch}; use crate::util::test_helpers::setup::setup_drive_with_initial_state_structure; use dpp::block::block_info::BlockInfo; use dpp::shielded::compute_shielded_unshield_fee; - let drive = setup_drive_with_initial_state_structure(None); let platform_version = PlatformVersion::latest(); - let epoch = Epoch::new(0).unwrap(); - - // Production-sized change note: 216-byte encrypted note, distinct nullifier/cmx per action. - let realistic_note = |i: u8| ShieldedActionNote { - nullifier: [i.wrapping_add(1); 32], - cmx: [i.wrapping_add(101); 32], - cv_net: [i.wrapping_add(201); 32], - encrypted_note: vec![0x77; 216], - }; - - for num_actions in [1usize, 8, 16] { - let fee_amount = compute_shielded_unshield_fee(num_actions, platform_version) - .expect("fee computation should not overflow"); - let notes: Vec<_> = (0..num_actions as u8).map(realistic_note).collect(); - // `amount` (unshielding_amount) must cover the fee with a positive net to the output - // address (so the AddBalanceToAddress write is exercised, the heaviest extra op). - let amount = fee_amount + 1_000_000; - let action = UnshieldTransitionAction::V0(UnshieldTransitionActionV0 { - output_address: PlatformAddress::P2pkh([0xBB; 20]), - amount, - notes, - anchor: [0xAA; 32], - fee_amount, - current_total_balance: amount + 1_000_000, - chargeable_failure: false, - }); - - let ops = action - .into_high_level_drive_operations(&epoch, platform_version) - .expect("operations"); - - // apply = false → estimation mode: no DB mutation, returns the real cost. - let fee_result = drive + let epoch_info = Epoch::new(0).unwrap(); + let epoch = transfer_epoch(); + + // The per-transition delta of an Unshield over a ShieldedTransfer at + // an ordinary (non-compacting) position: the output address write. + let measure = |ops: Vec| { + let drive = setup_drive_with_initial_state_structure(None); + drive .apply_drive_operations( ops, - false, + true, &BlockInfo::default(), None, platform_version, None, ) - .expect("estimate write cost"); - let actual_cost = fee_result.total_base_fee(); - + .expect("apply") + }; + let fee_one = compute_shielded_unshield_fee(1, platform_version).expect("fee"); + let amount = fee_one + 1_000_000; + let unshield_one = measure( + UnshieldTransitionAction::V0(UnshieldTransitionActionV0 { + output_address: PlatformAddress::P2pkh([0xBB; 20]), + amount, + notes: vec![note(7)], + anchor: [0xAA; 32], + fee_amount: fee_one, + current_total_balance: amount + 1_000_000, + chargeable_failure: false, + }) + .into_high_level_drive_operations(&epoch_info, platform_version) + .expect("operations"), + ); + let transfer_one = measure( + transfer_action(7, fee_one) + .into_high_level_drive_operations(&epoch_info, platform_version) + .expect("operations"), + ); + let delta_total = unshield_one + .total_base_fee() + .saturating_sub(transfer_one.total_base_fee()); + let delta_storage = unshield_one + .storage_fee + .saturating_sub(transfer_one.storage_fee); + + for num_actions in [1u64, 8, 16] { + let fee_amount = compute_shielded_unshield_fee(num_actions as usize, platform_version) + .expect("fee computation should not overflow"); assert!( - fee_amount >= actual_cost, - "compute_shielded_unshield_fee({num_actions}) = {fee_amount} must cover the actual \ - Unshield GroveDB write cost {actual_cost} (storage {} + processing {})", - fee_result.storage_fee, - fee_result.processing_fee + fee_amount >= num_actions * epoch.avg_total + delta_total, + "compute_shielded_unshield_fee({num_actions}) = {fee_amount} must cover the \ + amortized real Unshield write cost {} x {} + {} (output write)", + num_actions, + epoch.avg_total, + delta_total ); - - // The booking-split invariant: flat fee must strictly exceed real metered storage so - // `storage_fee = min(real_storage, flat_fee)` never binds (proposer processing reward - // never zeroed, storage never undercharged). See `execute_event/v0`. assert!( - fee_amount > fee_result.storage_fee, - "compute_shielded_unshield_fee({num_actions}) = {fee_amount} must strictly exceed the \ - real metered Unshield storage {} so the booking split's min(real_storage, flat_fee) \ - never binds", - fee_result.storage_fee + fee_amount > num_actions * epoch.avg_storage + delta_storage, + "flat fee {fee_amount} must exceed the amortized real storage {} x {} + {} so \ + the pool-paid booking split never starves the proposer", + num_actions, + epoch.avg_storage, + delta_storage ); } + assert!( + epoch.boundary_storage + delta_storage < fee_one, + "even the compacting append's real storage ({} + {}) must stay below the flat fee \ + ({fee_one}), or `min(actual_storage, flat)` would zero the proposer's share", + epoch.boundary_storage, + delta_storage + ); } #[test] diff --git a/packages/rs-drive/src/util/grove_operations/grove_insert_empty_tree/v0/mod.rs b/packages/rs-drive/src/util/grove_operations/grove_insert_empty_tree/v0/mod.rs index 0f14e32d071..4b3a2f065b8 100644 --- a/packages/rs-drive/src/util/grove_operations/grove_insert_empty_tree/v0/mod.rs +++ b/packages/rs-drive/src/util/grove_operations/grove_insert_empty_tree/v0/mod.rs @@ -56,6 +56,14 @@ impl Drive { batch_insert_empty_provable_count_provable_sum_indexed_tree instead.", ))) } + // The private document store's entry size is not carried by + // `TreeType` either (see `fees::op`); no generic creation path. + TreeType::PrivateDocumentStore(_) => { + return Err(Error::Drive(DriveError::NotSupported( + "grove_insert_empty_tree cannot create a PrivateDocumentStore — the entry \ + size is not carried by TreeType", + ))) + } }; let cost_context = self.grove.insert( path, diff --git a/packages/rs-drive/src/verify/document_having/verify_having_range_proof/mod.rs b/packages/rs-drive/src/verify/document_having/verify_having_range_proof/mod.rs index b4ba63ead28..893a2766bbb 100644 --- a/packages/rs-drive/src/verify/document_having/verify_having_range_proof/mod.rs +++ b/packages/rs-drive/src/verify/document_having/verify_having_range_proof/mod.rs @@ -41,7 +41,7 @@ impl DriveDocumentHavingQuery<'_> { .document_ranked .verify_having_range_proof { - 0 => self.verify_having_range_proof_v0(proof), + 0 => self.verify_having_range_proof_v0(proof, platform_version), version => Err(Error::Drive(DriveError::UnknownVersionMismatch { method: "DriveDocumentHavingQuery::verify_having_range_proof".to_string(), known_versions: vec![0], diff --git a/packages/rs-drive/src/verify/document_having/verify_having_range_proof/v0/mod.rs b/packages/rs-drive/src/verify/document_having/verify_having_range_proof/v0/mod.rs index fc9a0ab1727..9caf4f9df85 100644 --- a/packages/rs-drive/src/verify/document_having/verify_having_range_proof/v0/mod.rs +++ b/packages/rs-drive/src/verify/document_having/verify_having_range_proof/v0/mod.rs @@ -2,6 +2,7 @@ use crate::error::drive::DriveError; use crate::error::Error; use crate::query::{DriveDocumentHavingQuery, RankedAxis, RankedEntry, RankedEntryValue}; use crate::verify::RootHash; +use dpp::version::PlatformVersion; use grovedb::operations::proof::indexed_axis::AxisEntries; use grovedb::GroveDb; @@ -40,6 +41,7 @@ impl DriveDocumentHavingQuery<'_> { pub(super) fn verify_having_range_proof_v0( &self, proof: &[u8], + platform_version: &PlatformVersion, ) -> Result<(RootHash, Vec), Error> { let path = self.indexed_property_name_tree_path()?; let path_refs: Vec<&[u8]> = path.iter().map(|segment| segment.as_slice()).collect(); @@ -51,18 +53,21 @@ impl DriveDocumentHavingQuery<'_> { path_refs.as_slice(), secondary_query, Some(self.limit), + &platform_version.drive.grove_version, ), RankedAxis::Sum => GroveDb::verify_indexed_sum_query( proof, path_refs.as_slice(), secondary_query, Some(self.limit), + &platform_version.drive.grove_version, ), RankedAxis::Avg => GroveDb::verify_indexed_avg_query( proof, path_refs.as_slice(), secondary_query, Some(self.limit), + &platform_version.drive.grove_version, ), } .map_err(|e| Error::GroveDB(Box::new(e)))?; @@ -70,6 +75,7 @@ impl DriveDocumentHavingQuery<'_> { let entries = match (self.bounds.axis(), result.entries) { (RankedAxis::Count, AxisEntries::Count(entries)) => entries .into_iter() + .map(|entry| entry.key_pair()) .map(|(count, key)| RankedEntry { key, value: RankedEntryValue::Count(count), @@ -77,6 +83,7 @@ impl DriveDocumentHavingQuery<'_> { .collect::>(), (RankedAxis::Sum, AxisEntries::Sum(entries)) => entries .into_iter() + .map(|entry| entry.key_pair()) .map(|(sum, key)| RankedEntry { key, value: RankedEntryValue::Sum(sum), @@ -84,6 +91,7 @@ impl DriveDocumentHavingQuery<'_> { .collect::>(), (RankedAxis::Avg, AxisEntries::Avg(entries)) => entries .into_iter() + .map(|entry| entry.key_pair()) .map(|(avg, key)| RankedEntry { key, value: RankedEntryValue::AvgFixedPoint(avg), diff --git a/packages/rs-drive/src/verify/document_ranked/verify_ranked_top_k_proof/mod.rs b/packages/rs-drive/src/verify/document_ranked/verify_ranked_top_k_proof/mod.rs index 16d576a52f0..39bacf417f4 100644 --- a/packages/rs-drive/src/verify/document_ranked/verify_ranked_top_k_proof/mod.rs +++ b/packages/rs-drive/src/verify/document_ranked/verify_ranked_top_k_proof/mod.rs @@ -41,7 +41,7 @@ impl DriveDocumentRankedQuery<'_> { .document_ranked .verify_ranked_top_k_proof { - 0 => self.verify_ranked_top_k_proof_v0(proof), + 0 => self.verify_ranked_top_k_proof_v0(proof, platform_version), version => Err(Error::Drive(DriveError::UnknownVersionMismatch { method: "DriveDocumentRankedQuery::verify_ranked_top_k_proof".to_string(), known_versions: vec![0], diff --git a/packages/rs-drive/src/verify/document_ranked/verify_ranked_top_k_proof/v0/mod.rs b/packages/rs-drive/src/verify/document_ranked/verify_ranked_top_k_proof/v0/mod.rs index 668ab227770..82f5b0a2320 100644 --- a/packages/rs-drive/src/verify/document_ranked/verify_ranked_top_k_proof/v0/mod.rs +++ b/packages/rs-drive/src/verify/document_ranked/verify_ranked_top_k_proof/v0/mod.rs @@ -4,6 +4,7 @@ use crate::query::{ DriveDocumentRankedQuery, RankedAxis, RankedEntry, RankedEntryValue, RankedPage, }; use crate::verify::RootHash; +use dpp::version::PlatformVersion; use grovedb::operations::proof::indexed_axis::AxisEntries; use grovedb::GroveDb; @@ -44,14 +45,11 @@ impl DriveDocumentRankedQuery<'_> { /// is a useful answer, and the only place that knows whether it is /// acceptable is the caller. /// - /// No `platform_version` argument: the parent dispatcher already - /// consumed it to select this version, and nothing in the body needs - /// a grove version (verification derives everything from the proof - /// bytes plus the path). #[inline(always)] pub(super) fn verify_ranked_top_k_proof_v0( &self, proof: &[u8], + platform_version: &PlatformVersion, ) -> Result<(RootHash, RankedPage), Error> { let path = self.indexed_property_name_tree_path()?; let path_refs: Vec<&[u8]> = path.iter().map(|segment| segment.as_slice()).collect(); @@ -63,12 +61,14 @@ impl DriveDocumentRankedQuery<'_> { self.k, self.offset as u64, self.descending, + &platform_version.drive.grove_version, ) .map_err(|e| Error::GroveDB(Box::new(e)))?; let entries = match (self.axis, result.entries) { (RankedAxis::Count, AxisEntries::Count(entries)) => entries .into_iter() + .map(|entry| entry.key_pair()) .map(|(count, key)| RankedEntry { key, value: RankedEntryValue::Count(count), @@ -76,6 +76,7 @@ impl DriveDocumentRankedQuery<'_> { .collect::>(), (RankedAxis::Sum, AxisEntries::Sum(entries)) => entries .into_iter() + .map(|entry| entry.key_pair()) .map(|(sum, key)| RankedEntry { key, value: RankedEntryValue::Sum(sum), @@ -83,6 +84,7 @@ impl DriveDocumentRankedQuery<'_> { .collect::>(), (RankedAxis::Avg, AxisEntries::Avg(entries)) => entries .into_iter() + .map(|entry| entry.key_pair()) .map(|(avg, key)| RankedEntry { key, value: RankedEntryValue::AvgFixedPoint(avg), diff --git a/packages/rs-drive/src/verify/shielded/verify_shielded_encrypted_notes/v0/mod.rs b/packages/rs-drive/src/verify/shielded/verify_shielded_encrypted_notes/v0/mod.rs index 4129cb9447d..c3f6e582ffd 100644 --- a/packages/rs-drive/src/verify/shielded/verify_shielded_encrypted_notes/v0/mod.rs +++ b/packages/rs-drive/src/verify/shielded/verify_shielded_encrypted_notes/v0/mod.rs @@ -81,6 +81,7 @@ impl Drive { path: shielded_credit_pool_path_vec(), query: SizedQuery { query: Query { + read_mode: None, items: vec![QueryItem::Key(vec![SHIELDED_NOTES_KEY])], default_subquery_branch: SubqueryBranch { subquery_path: None, @@ -268,6 +269,7 @@ mod tests { path: shielded_credit_pool_path_vec(), query: SizedQuery { query: Query { + read_mode: None, items: vec![QueryItem::Key(vec![SHIELDED_NOTES_KEY])], default_subquery_branch: SubqueryBranch { subquery_path: None, diff --git a/packages/rs-platform-version/Cargo.toml b/packages/rs-platform-version/Cargo.toml index 27742bc04de..4ee353c872e 100644 --- a/packages/rs-platform-version/Cargo.toml +++ b/packages/rs-platform-version/Cargo.toml @@ -11,7 +11,7 @@ license = "MIT" thiserror = { version = "2.0.12" } bincode = { version = "=2.0.1" } versioned-feature-core = { git = "https://github.com/dashpay/versioned-feature-core", version = "1.0.0" } -grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "a2791bbdca756d6a6113024aec48f09f7a33faa9" } +grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "753a11f14c9a4bc72bf2d5302751dd43d174621e" } [features] mock-versions = [] diff --git a/packages/rs-platform-wallet/Cargo.toml b/packages/rs-platform-wallet/Cargo.toml index 7d404f6985a..755f7bb110b 100644 --- a/packages/rs-platform-wallet/Cargo.toml +++ b/packages/rs-platform-wallet/Cargo.toml @@ -69,7 +69,7 @@ zeroize = "1" log = "0.4" # Shielded pool (optional, behind `shielded` feature) -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "a2791bbdca756d6a6113024aec48f09f7a33faa9", optional = true } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "753a11f14c9a4bc72bf2d5302751dd43d174621e", optional = true } # Direct `rusqlite` access so `FileBackedShieldedStore::open_path` can set # WAL + synchronous=NORMAL pragmas before handing the connection to # `ClientPersistentCommitmentTree`. Version locked to match the rev grovedb diff --git a/packages/rs-sdk-ffi/src/system/queries/path_elements.rs b/packages/rs-sdk-ffi/src/system/queries/path_elements.rs index 0a54b9d8981..96466024445 100644 --- a/packages/rs-sdk-ffi/src/system/queries/path_elements.rs +++ b/packages/rs-sdk-ffi/src/system/queries/path_elements.rs @@ -203,6 +203,8 @@ fn format_element_data(element: &Element) -> String { Element::DenseAppendOnlyFixedSizeTree(_, _, _) => { "dense_append_only_fixed_size_tree".to_string() } + + Element::PrivateDocumentStore(_, _, _, _) => "private_document_store".to_string(), Element::NonCounted(inner) => format!("non_counted({})", format_element_data(inner)), Element::NotSummed(inner) => format!("not_summed({})", format_element_data(inner)), Element::NotCountedOrSummed(inner) => { @@ -241,6 +243,8 @@ fn format_element_type(element: &Element) -> String { Element::DenseAppendOnlyFixedSizeTree(_, _, _) => { "dense_append_only_fixed_size_tree".to_string() } + + Element::PrivateDocumentStore(_, _, _, _) => "private_document_store".to_string(), Element::NonCounted(inner) => format!("non_counted({})", format_element_type(inner)), Element::NotSummed(inner) => format!("not_summed({})", format_element_type(inner)), Element::NotCountedOrSummed(inner) => { diff --git a/packages/rs-sdk/Cargo.toml b/packages/rs-sdk/Cargo.toml index 651a7b6a11f..ee1bfca14ae 100644 --- a/packages/rs-sdk/Cargo.toml +++ b/packages/rs-sdk/Cargo.toml @@ -18,7 +18,7 @@ drive = { path = "../rs-drive", default-features = false, features = [ ] } drive-proof-verifier = { path = "../rs-drive-proof-verifier", default-features = false } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "a2791bbdca756d6a6113024aec48f09f7a33faa9", features = [ +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "753a11f14c9a4bc72bf2d5302751dd43d174621e", features = [ "client", "sqlite", ], optional = true } diff --git a/packages/wasm-sdk/src/queries/system.rs b/packages/wasm-sdk/src/queries/system.rs index 55865024b75..1051361908f 100644 --- a/packages/wasm-sdk/src/queries/system.rs +++ b/packages/wasm-sdk/src/queries/system.rs @@ -39,6 +39,7 @@ export type GroveElementType = | "mmrTree" | "bulkAppendTree" | "denseAppendOnlyFixedSizeTree" + | "privateDocumentStore" | "nonCountedItem" | "nonCountedReference" | "nonCountedTree" @@ -60,6 +61,7 @@ export type GroveElementType = | "nonCountedMmrTree" | "nonCountedBulkAppendTree" | "nonCountedDenseAppendOnlyFixedSizeTree" + | "nonCountedPrivateDocumentStore" | "notSummedSumTree" | "notSummedBigSumTree" | "notSummedCountSumTree" @@ -941,6 +943,8 @@ fn element_type_name(element: &Element) -> &'static str { Element::MmrTree(_, _) => "mmrTree", Element::BulkAppendTree(_, _, _) => "bulkAppendTree", Element::DenseAppendOnlyFixedSizeTree(_, _, _) => "denseAppendOnlyFixedSizeTree", + + Element::PrivateDocumentStore(_, _, _, _) => "privateDocumentStore", Element::NonCounted(inner) => non_counted_element_type_name(inner), Element::NotSummed(inner) => not_summed_element_type_name(inner), Element::NotCountedOrSummed(inner) => not_counted_or_summed_element_type_name(inner), @@ -974,6 +978,8 @@ fn non_counted_element_type_name(element: &Element) -> &'static str { Element::MmrTree(_, _) => "nonCountedMmrTree", Element::BulkAppendTree(_, _, _) => "nonCountedBulkAppendTree", Element::DenseAppendOnlyFixedSizeTree(_, _, _) => "nonCountedDenseAppendOnlyFixedSizeTree", + + Element::PrivateDocumentStore(_, _, _, _) => "nonCountedPrivateDocumentStore", Element::NonCounted(_) | Element::NotSummed(_) | Element::NotCountedOrSummed(_) => { element_type_name(element) } @@ -1939,6 +1945,14 @@ mod tests { Element::DenseAppendOnlyFixedSizeTree(1, 2, None), "denseAppendOnlyFixedSizeTree", ), + ( + Element::PrivateDocumentStore(0, 32, 2, None), + "privateDocumentStore", + ), + ( + Element::NonCounted(Box::new(Element::PrivateDocumentStore(0, 32, 2, None))), + "nonCountedPrivateDocumentStore", + ), ( Element::NonCounted(Box::new(Element::ReferenceWithSumItem( reference_path(),